From c6e0780f4b8933a50949cd10913b8c1be3ccfa82 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Thu, 22 Aug 2024 17:36:11 +0200 Subject: [PATCH] app/vmalert: update parsing for instant responses (#6859) This change is made in attempt to reduce memory usage by vmalert when parsing big instant responses from VM/Prometheus. In https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a5c427bac4102c53d1ea2aefe0b0e6872120ba9c vmalert switched from std json lib to fastjson lib in order to reduce amount of allocations, as according to highloaded profiles of vmalert the CPU is mostly spent on GC. But switching to fastjson resulted into excessive memory usage for cases when vmalert has to parse long json lines, which usually happens when instant response contains many `metric` objects. In this change we do a mixed parsing: 1. Slice of `metric` objects is parsed with std lib to keep mem low 2. Each `metric` object is parsed with fastjson to reduce allocs The benchmark results are the following: ``` pkg: github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op MBs allocated at heap: 493.078392 mallocs: 18655472 BenchmarkParsePrometheusResponse/Instant_fastjson-10 6109 198258 ns/op 172839 B/op 5548 allocs/op MBs allocated at heap: 1056.384464 mallocs: 34457184 BenchmarkParsePrometheusResponse/Instant_std-10 1287 950987 ns/op 451677 B/op 9619 allocs/op MBs allocated at heap: 580.802976 mallocs: 13351636 ``` The benchmark function code with mem measurement is available here https://gist.github.com/hagen1778/b9c3ca7f8ca7d6b21aec9777112c5810 The benchmark contains 3 results: 1. Instant_std+fastjson is the implementation in this change 2. Instant_fastjson-10 is the implementation from https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a5c427bac4102c53d1ea2aefe0b0e6872120ba9c 3. BenchmarkParsePrometheusResponse/Instant_std-10 is implementation before https://github.com/VictoriaMetrics/VictoriaMetrics/commit/a5c427bac4102c53d1ea2aefe0b0e6872120ba9c According to these results, this new implementation is slower than previous, but faster than before switching to fastjson. It also has lower number of allocations and roughly the same memory allocation on heap with GC turned off. --------- Other changes: 1. rm BenchmarkMetrics as it doesn't measure anything 2. simplify BenchmarkParsePrometheusResponse into BenchmarkPromInstantUnmarshal ### Describe Your Changes Please provide a brief description of the changes you made. Be as specific as possible to help others understand the purpose and impact of your modifications. ### Checklist The following checks are **mandatory**: - [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/). Signed-off-by: hagen1778 --- .../datasource/testdata/instant_response.json | 2 +- app/vmalert/datasource/vm_prom_api.go | 25 +++++++++------- .../datasource/vm_prom_api_timing_test.go | 29 ++++--------------- docs/changelog/CHANGELOG.md | 1 + 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/app/vmalert/datasource/testdata/instant_response.json b/app/vmalert/datasource/testdata/instant_response.json index 20af44038c..4a5c973736 100644 --- a/app/vmalert/datasource/testdata/instant_response.json +++ b/app/vmalert/datasource/testdata/instant_response.json @@ -1 +1 @@ -{"status":"success","isPartial":false,"data":{"resultType":"vector","result":[{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"78372"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"47048"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15682"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15642"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/beta/sketches","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/check_run","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/metadata","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/validate","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v2/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/intake","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/query","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/write","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/infra/v2/metrics/events/bulk","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/inventory/deltas","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/opentelemetry/v1/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"opentelemetry","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"promremotewrite","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"9208925"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"vmimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/csv","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"csvimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/native","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"nativeimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/prometheus","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"prometheusimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"65591"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"78343"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"47028"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15676"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15639"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/beta/sketches","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/check_run","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/metadata","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/validate","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v2/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/intake","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/query","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/write","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/infra/v2/metrics/events/bulk","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/inventory/deltas","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/opentelemetry/v1/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"opentelemetry","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"promremotewrite","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"10078748"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"vmimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/csv","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"csvimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/native","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"nativeimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/prometheus","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"prometheusimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"65569"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/admin/tenants","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"78241"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"47034"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15679"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15528"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/delete/{}/prometheus/api/v1/admin/tsdb/delete_series","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"65567"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions/\u003cfunc_name>","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/expand","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/find","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/index.json","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/render","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/\u003ctag_name>","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/tags","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/values","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/delSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/findSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagMultiSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/alerts","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/buildinfo","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/csv","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/native","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/label/{}/values","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"6"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/labels","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"6"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/metadata","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"2"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1436075"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_exemplars","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"332"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_range","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1804"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/rules","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"67"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series/count","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/tsdb","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/expand-with-exprs","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/federate","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/metric-relabel-debug","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/prettify-query","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/target-relabel-debug","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/vmalert","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}prometheus/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/admin/tenants","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"78204"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"46996"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15666"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15542"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/delete/{}/prometheus/api/v1/admin/tsdb/delete_series","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"65518"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions/\u003cfunc_name>","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/expand","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/find","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/index.json","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/render","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/\u003ctag_name>","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/tags","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/values","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/delSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/findSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagMultiSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/alerts","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/buildinfo","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"7"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/csv","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/native","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/label/{}/values","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"8"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/labels","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"8"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/metadata","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1665308"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_exemplars","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"438"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_range","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"2397"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/rules","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"30"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series/count","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/tsdb","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/expand-with-exprs","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/federate","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/metric-relabel-debug","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/prettify-query","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/target-relabel-debug","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/vmalert","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}prometheus/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78342"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47042"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15681"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15619"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65591"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78314"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47029"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15678"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15607"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65574"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78293"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47008"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15670"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15615"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65544"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78248"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"46986"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15660"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15602"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65513"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78260"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"46992"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15664"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15604"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65523"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]}]},"stats":{"seriesFetched": "219","executionTimeMsec":7}} \ No newline at end of file +[{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"78372"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"47048"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15682"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15642"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/beta/sketches","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/check_run","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/metadata","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/validate","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v2/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/intake","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/query","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/write","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/infra/v2/metrics/events/bulk","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/inventory/deltas","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/opentelemetry/v1/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"opentelemetry","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"promremotewrite","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"9208925"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"vmimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/csv","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"csvimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/native","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"nativeimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/prometheus","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","protocol":"prometheusimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.0.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-m544p","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"65591"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"78343"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"47028"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15676"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"15639"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/beta/sketches","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/check_run","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/metadata","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v1/validate","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/api/v2/series","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/datadog/intake","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"datadog","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/query","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/influx/write","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"influx","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/infra/v2/metrics/events/bulk","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/newrelic/inventory/deltas","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"newrelic","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/opentelemetry/v1/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"opentelemetry","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"promremotewrite","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"10078748"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"vmimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/csv","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"csvimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/native","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"nativeimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/insert/{}/prometheus/api/v1/import/prometheus","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","protocol":"prometheusimport","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vminsert","endpoint":"http","instance":"10.71.2.3:8480","job":"vminsert-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vminsert-benchmark-vm-cluster-b955964f8-nbszp","prometheus":"monitoring/monitoring-vmagent","service":"vminsert-benchmark-vm-cluster"},"value":[1715686833.987,"65569"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/admin/tenants","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"78241"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"47034"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15679"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15528"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/delete/{}/prometheus/api/v1/admin/tsdb/delete_series","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"65567"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions/\u003cfunc_name>","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/expand","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/find","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/index.json","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/render","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/\u003ctag_name>","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/tags","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/values","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/delSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/findSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagMultiSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagSeries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/alerts","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/buildinfo","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/csv","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/native","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/label/{}/values","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"6"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/labels","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"6"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/metadata","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"2"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1436075"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_exemplars","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"332"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_range","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1804"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/rules","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"67"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series/count","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/tsdb","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/expand-with-exprs","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/federate","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/metric-relabel-debug","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/prettify-query","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/target-relabel-debug","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/vmalert","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.1.8:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}prometheus/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/admin/tenants","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"78204"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"46996"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15666"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"15542"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/delete/{}/prometheus/api/v1/admin/tsdb/delete_series","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"65518"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/functions/\u003cfunc_name>","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/expand","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/find","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/metrics/index.json","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/render","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/\u003ctag_name>","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/tags","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/autoComplete/values","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/delSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/findSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagMultiSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/graphite/tags/tagSeries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/alerts","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/buildinfo","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"7"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/csv","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/export/native","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/label/{}/values","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"8"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/labels","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"8"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/metadata","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1665308"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_exemplars","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"438"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/query_range","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"2397"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/rules","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"1"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"30"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/series/count","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/top_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/api/v1/status/tsdb","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/expand-with-exprs","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/federate","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/metric-relabel-debug","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/prettify-query","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/target-relabel-debug","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}/prometheus/vmalert","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmselect","endpoint":"http","instance":"10.71.7.6:8481","job":"vmselect-benchmark-vm-cluster","namespace":"benchmark","path":"/select/{}prometheus/api/v1/status/active_queries","pod":"vmselect-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmselect-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78342"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47042"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15681"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15619"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65591"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.0.14:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-0","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78314"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47029"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15678"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15607"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65574"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.1.9:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-3","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78293"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"47008"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15670"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15615"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65544"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.5.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-4","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78248"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"46986"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15660"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15602"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65513"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.6.6:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-2","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"*/favicon.ico","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"78260"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/cmdline","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/default","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"46992"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/mutex","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15664"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/profile","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"15604"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/symbol","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/debug/pprof/trace","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/metrics","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"65523"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/create","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/delete_all","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]},{"metric":{"__name__":"vm_http_requests_total","cluster":"sandbox","container":"vmstorage","endpoint":"http","instance":"10.71.7.7:8482","job":"vmstorage-benchmark-vm-cluster","namespace":"benchmark","path":"/snapshot/list","pod":"vmstorage-benchmark-vm-cluster-1","prometheus":"monitoring/monitoring-vmagent","service":"vmstorage-benchmark-vm-cluster"},"value":[1715686833.987,"0"]}] \ No newline at end of file diff --git a/app/vmalert/datasource/vm_prom_api.go b/app/vmalert/datasource/vm_prom_api.go index 96eedb024e..f2b4bddee7 100644 --- a/app/vmalert/datasource/vm_prom_api.go +++ b/app/vmalert/datasource/vm_prom_api.go @@ -57,20 +57,23 @@ var jsonParserPool fastjson.ParserPool // [{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]}, // {"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}] func (pi *promInstant) Unmarshal(b []byte) error { + var metrics []json.RawMessage + // metrics slice could be large, so parsing it with fastjson could consume a lot of memory. + // We parse the slice with standard lib to keep mem usage low. + // And each metric object will be parsed with fastjson to reduce allocations. + if err := json.Unmarshal(b, &metrics); err != nil { + return fmt.Errorf("cannot unmarshal metrics: %w", err) + } + p := jsonParserPool.Get() defer jsonParserPool.Put(p) - v, err := p.ParseBytes(b) - if err != nil { - return err - } - - rows, err := v.Array() - if err != nil { - return fmt.Errorf("cannot find the top-level array of result objects: %w", err) - } - pi.ms = make([]Metric, len(rows)) - for i, row := range rows { + pi.ms = make([]Metric, len(metrics)) + for i, data := range metrics { + row, err := p.ParseBytes(data) + if err != nil { + return fmt.Errorf("cannot parse metric object: %w", err) + } metric := row.Get("metric") if metric == nil { return fmt.Errorf("can't find `metric` object in %q", row) diff --git a/app/vmalert/datasource/vm_prom_api_timing_test.go b/app/vmalert/datasource/vm_prom_api_timing_test.go index 4bcb84e84f..656f973fce 100644 --- a/app/vmalert/datasource/vm_prom_api_timing_test.go +++ b/app/vmalert/datasource/vm_prom_api_timing_test.go @@ -1,43 +1,24 @@ package datasource import ( - "bytes" - "io" - "net/http" "os" "testing" ) -func BenchmarkMetrics(b *testing.B) { - payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`) - - var pi promInstant - if err := pi.Unmarshal(payload); err != nil { - b.Fatal(err.Error()) - } - b.Run("Instant", func(b *testing.B) { - for i := 0; i < b.N; i++ { - _, _ = pi.metrics() - } - }) -} - -func BenchmarkParsePrometheusResponse(b *testing.B) { - req, _ := http.NewRequest("GET", "", nil) - resp := &http.Response{StatusCode: http.StatusOK} +func BenchmarkPromInstantUnmarshal(b *testing.B) { data, err := os.ReadFile("testdata/instant_response.json") if err != nil { b.Fatalf("error while reading file: %s", err) } - resp.Body = io.NopCloser(bytes.NewReader(data)) - b.Run("Instant", func(b *testing.B) { + // BenchmarkParsePrometheusResponse/Instant_std+fastjson-10 1760 668959 ns/op 280147 B/op 5781 allocs/op + b.Run("Instant std+fastjson", func(b *testing.B) { for i := 0; i < b.N; i++ { - _, err := parsePrometheusResponse(req, resp) + var pi promInstant + err = pi.Unmarshal(data) if err != nil { b.Fatalf("unexpected parse err: %s", err) } - resp.Body = io.NopCloser(bytes.NewReader(data)) } }) } diff --git a/docs/changelog/CHANGELOG.md b/docs/changelog/CHANGELOG.md index 0757a7f796..8d974ffec6 100644 --- a/docs/changelog/CHANGELOG.md +++ b/docs/changelog/CHANGELOG.md @@ -45,6 +45,7 @@ The value of `instance` label for those scrape targets will be changed from `