2019-05-22 23:23:23 +02:00
package main
2019-05-22 23:16:55 +02:00
import (
2021-07-07 16:04:23 +02:00
"embed"
2020-06-30 23:02:02 +02:00
"errors"
2019-05-22 23:16:55 +02:00
"flag"
2019-08-23 08:46:45 +02:00
"fmt"
2019-05-22 23:16:55 +02:00
"net/http"
2022-05-03 19:55:15 +02:00
"net/http/httputil"
"net/url"
2020-05-16 10:59:30 +02:00
"os"
2019-05-22 23:16:55 +02:00
"strings"
"time"
2022-07-06 12:19:45 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/clusternative"
2020-09-10 23:29:26 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphite"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
2020-09-22 00:21:20 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
2019-05-22 23:23:23 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
2020-08-11 21:54:13 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
2020-02-10 12:26:18 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
2019-05-22 23:23:23 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
2019-05-22 23:23:23 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
2023-01-04 07:40:00 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
2022-07-21 18:58:22 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
2022-06-01 01:31:40 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
2020-02-10 12:03:52 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
2021-02-16 22:25:27 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
2019-05-28 16:17:19 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
2022-07-06 12:19:45 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/metrics"
)
var (
2023-01-27 08:08:35 +01:00
httpListenAddr = flag . String ( "httpListenAddr" , ":8481" , "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol" )
useProxyProtocol = flag . Bool ( "httpListenAddr.useProxyProtocol" , false , "Whether to use proxy protocol for connections accepted at -httpListenAddr . " +
2023-03-08 10:26:53 +01:00
"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . " +
"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing" )
2019-05-22 23:23:23 +02:00
cacheDataPath = flag . String ( "cacheDataPath" , "" , "Path to directory for cache files. Cache isn't saved if empty" )
2020-01-17 14:43:47 +01:00
maxConcurrentRequests = flag . Int ( "search.maxConcurrentRequests" , getDefaultMaxConcurrentRequests ( ) , "The maximum number of concurrent search requests. " +
2022-10-09 12:56:55 +02:00
"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. " +
"See also -search.maxQueueDuration and -search.maxMemoryPerQuery" )
2020-09-22 00:21:20 +02:00
maxQueueDuration = flag . Duration ( "search.maxQueueDuration" , 10 * time . Second , "The maximum time the request waits for execution when -search.maxConcurrentRequests " +
"limit is reached; see also -search.maxQueryDuration" )
2022-05-02 20:35:14 +02:00
minScrapeInterval = flag . Duration ( "dedup.minScrapeInterval" , 0 , "Leave only the last sample in every time series per each discrete interval " +
2021-07-02 14:02:24 +02:00
"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication for details" )
2021-06-18 18:04:42 +02:00
resetCacheAuthKey = flag . String ( "search.resetCacheAuthKey" , "" , "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call" )
2023-02-24 03:40:31 +01:00
logSlowQueryDuration = flag . Duration ( "search.logSlowQueryDuration" , 5 * time . Second , "Log queries with execution time exceeding this value. Zero disables slow query logging. " +
"See also -search.logQueryMemoryUsage" )
vmalertProxyURL = flag . String ( "vmalert.proxyURL" , "" , "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules" )
storageNodes = flagutil . NewArrayString ( "storageNode" , "Comma-separated addresses of vmstorage nodes; usage: -storageNode=vmstorage-host1,...,vmstorage-hostN . " +
2022-10-28 12:07:40 +02:00
"Enterprise version of VictoriaMetrics supports automatic discovery of vmstorage addresses via dns+srv records. For example, -storageNode=dns+srv:vmstorage.addrs . " +
"See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#automatic-vmstorage-discovery" )
2022-07-06 12:46:19 +02:00
clusternativeListenAddr = flag . String ( "clusternativeListenAddr" , "" , "TCP address to listen for requests from other vmselect nodes in multi-level cluster setup. " +
2022-11-04 09:49:53 +01:00
"See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multi-level-cluster-setup . Usually :8401 should be set to match default vmstorage port for vmselect. Disabled work if empty" )
2019-05-22 23:16:55 +02:00
)
2021-06-18 18:04:42 +02:00
var slowQueries = metrics . NewCounter ( ` vm_slow_queries_total ` )
2020-01-17 14:43:47 +01:00
func getDefaultMaxConcurrentRequests ( ) int {
2020-12-08 19:49:32 +01:00
n := cgroup . AvailableCPUs ( )
2020-01-17 14:43:47 +01:00
if n <= 4 {
n *= 2
}
if n > 16 {
// A single request can saturate all the CPU cores, so there is no sense
// in allowing higher number of concurrent requests - they will just contend
// for unavailable CPU time.
n = 16
}
return n
}
2022-12-10 12:04:43 +01:00
//go:embed static
var staticFiles embed . FS
var staticServer = http . FileServer ( http . FS ( staticFiles ) )
2019-05-22 23:23:23 +02:00
func main ( ) {
2020-05-16 10:59:30 +02:00
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag . CommandLine . SetOutput ( os . Stdout )
2020-12-03 20:40:30 +01:00
flag . Usage = usage
2020-02-10 12:26:18 +01:00
envflag . Parse ( )
2019-05-22 23:23:23 +02:00
buildinfo . Init ( )
logger . Init ( )
2022-07-22 12:35:58 +02:00
pushmetrics . Init ( )
2019-05-22 23:23:23 +02:00
2019-07-20 09:21:59 +02:00
logger . Infof ( "starting netstorage at storageNodes %s" , * storageNodes )
2019-05-22 23:23:23 +02:00
startTime := time . Now ( )
2021-12-14 19:49:08 +01:00
storage . SetDedupInterval ( * minScrapeInterval )
2019-06-18 09:26:44 +02:00
if len ( * storageNodes ) == 0 {
2019-07-20 09:21:59 +02:00
logger . Fatalf ( "missing -storageNode arg" )
2019-05-22 23:23:23 +02:00
}
2022-11-01 13:54:55 +01:00
if hasEmptyValues ( * storageNodes ) {
logger . Fatalf ( "found empty address of storage node in the -storageNodes flag, please make sure that all -storageNode args are non-empty" )
}
2022-09-08 20:17:58 +02:00
if duplicatedAddr := checkDuplicates ( * storageNodes ) ; duplicatedAddr != "" {
logger . Fatalf ( "found equal addresses of storage nodes in the -storageNodes flag: %q" , duplicatedAddr )
}
2022-10-25 13:41:56 +02:00
netstorage . Init ( * storageNodes )
2020-01-22 17:27:44 +01:00
logger . Infof ( "started netstorage in %.3f seconds" , time . Since ( startTime ) . Seconds ( ) )
2019-05-22 23:23:23 +02:00
if len ( * cacheDataPath ) > 0 {
tmpDataPath := * cacheDataPath + "/tmp"
fs . RemoveDirContents ( tmpDataPath )
netstorage . InitTmpBlocksDir ( tmpDataPath )
promql . InitRollupResultCache ( * cacheDataPath + "/rollupResult" )
} else {
netstorage . InitTmpBlocksDir ( "" )
promql . InitRollupResultCache ( "" )
}
2023-01-07 03:19:05 +01:00
concurrencyLimitCh = make ( chan struct { } , * maxConcurrentRequests )
2022-05-03 19:55:15 +02:00
initVMAlertProxy ( )
2022-07-06 12:19:45 +02:00
var vmselectapiServer * vmselectapi . Server
if * clusternativeListenAddr != "" {
logger . Infof ( "starting vmselectapi server at %q" , * clusternativeListenAddr )
s , err := clusternative . NewVMSelectServer ( * clusternativeListenAddr )
if err != nil {
logger . Fatalf ( "cannot initialize vmselectapi server: %s" , err )
}
vmselectapiServer = s
logger . Infof ( "started vmselectapi server at %q" , * clusternativeListenAddr )
}
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
go func ( ) {
2023-01-27 08:08:35 +01:00
httpserver . Serve ( * httpListenAddr , * useProxyProtocol , requestHandler )
2019-05-22 23:23:23 +02:00
} ( )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
sig := procutil . WaitForSigterm ( )
logger . Infof ( "service received signal %s" , sig )
app/vmstorage: add missing shutdown for http server on graceful shutdown
This could result in the following panic during graceful shutdown when `/metrics` page is requested:
http: panic serving 10.101.66.5:57366: runtime error: invalid memory address or nil pointer dereference
goroutine 2050 [running]:
net/http.(*conn).serve.func1(0xc00ef22000)
net/http/server.go:1772 +0x139
panic(0xa0fc00, 0xe91d80)
runtime/panic.go:973 +0x3e3
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache.(*Cache).UpdateStats(0x0, 0xc0000516c8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache/cache.go:224 +0x37
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*indexDB).UpdateMetrics(0xc00b931d00, 0xc02c41acf8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/index_db.go:258 +0x9f
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*Storage).UpdateMetrics(0xc0000bc7e0, 0xc02c41ac00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/storage.go:413 +0x4c5
main.registerStorageMetrics.func1(0x0)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:186 +0xd9
main.registerStorageMetrics.func3(0xc00008c380)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:196 +0x26
main.registerStorageMetrics.func7(0xc)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:211 +0x26
github.com/VictoriaMetrics/metrics.(*Gauge).marshalTo(0xc000010148, 0xaa407d, 0x20, 0xb50d60, 0xc005319890)
github.com/VictoriaMetrics/metrics@v1.11.2/gauge.go:38 +0x3f
github.com/VictoriaMetrics/metrics.(*Set).WritePrometheus(0xc000084300, 0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/metrics@v1.11.2/set.go:51 +0x1e1
github.com/VictoriaMetrics/metrics.WritePrometheus(0x7fd56809c940, 0xc005319860, 0xa16f01)
github.com/VictoriaMetrics/metrics@v1.11.2/metrics.go:42 +0x41
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.writePrometheusMetrics(0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/metrics.go:16 +0x44
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.handlerWrapper(0xb5a120, 0xc005319860, 0xc005018f00, 0xc00002cc90)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:154 +0x58d
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.gzipHandler.func1(0xb5a120, 0xc005319860, 0xc005018f00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:119 +0x8e
net/http.HandlerFunc.ServeHTTP(0xc00002d110, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2012 +0x44
net/http.serverHandler.ServeHTTP(0xc004414000, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2807 +0xa3
net/http.(*conn).serve(0xc00ef22000, 0xb5bf60, 0xc010532080)
net/http/server.go:1895 +0x86c
created by net/http.(*Server).Serve
net/http/server.go:2933 +0x35c
2020-04-02 20:07:59 +02:00
logger . Infof ( "gracefully shutting down http service at %q" , * httpListenAddr )
2019-05-22 23:23:23 +02:00
startTime = time . Now ( )
if err := httpserver . Stop ( * httpListenAddr ) ; err != nil {
app/vmstorage: add missing shutdown for http server on graceful shutdown
This could result in the following panic during graceful shutdown when `/metrics` page is requested:
http: panic serving 10.101.66.5:57366: runtime error: invalid memory address or nil pointer dereference
goroutine 2050 [running]:
net/http.(*conn).serve.func1(0xc00ef22000)
net/http/server.go:1772 +0x139
panic(0xa0fc00, 0xe91d80)
runtime/panic.go:973 +0x3e3
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache.(*Cache).UpdateStats(0x0, 0xc0000516c8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache/cache.go:224 +0x37
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*indexDB).UpdateMetrics(0xc00b931d00, 0xc02c41acf8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/index_db.go:258 +0x9f
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*Storage).UpdateMetrics(0xc0000bc7e0, 0xc02c41ac00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/storage.go:413 +0x4c5
main.registerStorageMetrics.func1(0x0)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:186 +0xd9
main.registerStorageMetrics.func3(0xc00008c380)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:196 +0x26
main.registerStorageMetrics.func7(0xc)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:211 +0x26
github.com/VictoriaMetrics/metrics.(*Gauge).marshalTo(0xc000010148, 0xaa407d, 0x20, 0xb50d60, 0xc005319890)
github.com/VictoriaMetrics/metrics@v1.11.2/gauge.go:38 +0x3f
github.com/VictoriaMetrics/metrics.(*Set).WritePrometheus(0xc000084300, 0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/metrics@v1.11.2/set.go:51 +0x1e1
github.com/VictoriaMetrics/metrics.WritePrometheus(0x7fd56809c940, 0xc005319860, 0xa16f01)
github.com/VictoriaMetrics/metrics@v1.11.2/metrics.go:42 +0x41
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.writePrometheusMetrics(0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/metrics.go:16 +0x44
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.handlerWrapper(0xb5a120, 0xc005319860, 0xc005018f00, 0xc00002cc90)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:154 +0x58d
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.gzipHandler.func1(0xb5a120, 0xc005319860, 0xc005018f00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:119 +0x8e
net/http.HandlerFunc.ServeHTTP(0xc00002d110, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2012 +0x44
net/http.serverHandler.ServeHTTP(0xc004414000, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2807 +0xa3
net/http.(*conn).serve(0xc00ef22000, 0xb5bf60, 0xc010532080)
net/http/server.go:1895 +0x86c
created by net/http.(*Server).Serve
net/http/server.go:2933 +0x35c
2020-04-02 20:07:59 +02:00
logger . Fatalf ( "cannot stop http service: %s" , err )
2019-05-22 23:23:23 +02:00
}
app/vmstorage: add missing shutdown for http server on graceful shutdown
This could result in the following panic during graceful shutdown when `/metrics` page is requested:
http: panic serving 10.101.66.5:57366: runtime error: invalid memory address or nil pointer dereference
goroutine 2050 [running]:
net/http.(*conn).serve.func1(0xc00ef22000)
net/http/server.go:1772 +0x139
panic(0xa0fc00, 0xe91d80)
runtime/panic.go:973 +0x3e3
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache.(*Cache).UpdateStats(0x0, 0xc0000516c8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache/cache.go:224 +0x37
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*indexDB).UpdateMetrics(0xc00b931d00, 0xc02c41acf8)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/index_db.go:258 +0x9f
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage.(*Storage).UpdateMetrics(0xc0000bc7e0, 0xc02c41ac00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/storage.go:413 +0x4c5
main.registerStorageMetrics.func1(0x0)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:186 +0xd9
main.registerStorageMetrics.func3(0xc00008c380)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:196 +0x26
main.registerStorageMetrics.func7(0xc)
github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/main.go:211 +0x26
github.com/VictoriaMetrics/metrics.(*Gauge).marshalTo(0xc000010148, 0xaa407d, 0x20, 0xb50d60, 0xc005319890)
github.com/VictoriaMetrics/metrics@v1.11.2/gauge.go:38 +0x3f
github.com/VictoriaMetrics/metrics.(*Set).WritePrometheus(0xc000084300, 0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/metrics@v1.11.2/set.go:51 +0x1e1
github.com/VictoriaMetrics/metrics.WritePrometheus(0x7fd56809c940, 0xc005319860, 0xa16f01)
github.com/VictoriaMetrics/metrics@v1.11.2/metrics.go:42 +0x41
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.writePrometheusMetrics(0x7fd56809c940, 0xc005319860)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/metrics.go:16 +0x44
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.handlerWrapper(0xb5a120, 0xc005319860, 0xc005018f00, 0xc00002cc90)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:154 +0x58d
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver.gzipHandler.func1(0xb5a120, 0xc005319860, 0xc005018f00)
github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver/httpserver.go:119 +0x8e
net/http.HandlerFunc.ServeHTTP(0xc00002d110, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2012 +0x44
net/http.serverHandler.ServeHTTP(0xc004414000, 0xb5a660, 0xc0044141c0, 0xc005018f00)
net/http/server.go:2807 +0xa3
net/http.(*conn).serve(0xc00ef22000, 0xb5bf60, 0xc010532080)
net/http/server.go:1895 +0x86c
created by net/http.(*Server).Serve
net/http/server.go:2933 +0x35c
2020-04-02 20:07:59 +02:00
logger . Infof ( "successfully shut down http service in %.3f seconds" , time . Since ( startTime ) . Seconds ( ) )
2019-05-22 23:23:23 +02:00
2022-07-06 12:19:45 +02:00
if vmselectapiServer != nil {
logger . Infof ( "stopping vmselectapi server..." )
vmselectapiServer . MustStop ( )
logger . Infof ( "stopped vmselectapi server" )
}
2019-05-22 23:23:23 +02:00
logger . Infof ( "shutting down neststorage..." )
startTime = time . Now ( )
2022-10-25 13:41:56 +02:00
netstorage . MustStop ( )
2019-05-22 23:23:23 +02:00
if len ( * cacheDataPath ) > 0 {
promql . StopRollupResultCache ( )
}
2020-01-22 17:27:44 +01:00
logger . Infof ( "successfully stopped netstorage in %.3f seconds" , time . Since ( startTime ) . Seconds ( ) )
2019-05-22 23:23:23 +02:00
2019-11-12 15:29:43 +01:00
fs . MustStopDirRemover ( )
2019-05-22 23:23:23 +02:00
logger . Infof ( "the vmselect has been stopped" )
2019-05-22 23:16:55 +02:00
}
2023-01-07 03:19:05 +01:00
var concurrencyLimitCh chan struct { }
2019-05-22 23:23:23 +02:00
2019-08-05 17:27:50 +02:00
var (
concurrencyLimitReached = metrics . NewCounter ( ` vm_concurrent_select_limit_reached_total ` )
concurrencyLimitTimeout = metrics . NewCounter ( ` vm_concurrent_select_limit_timeout_total ` )
_ = metrics . NewGauge ( ` vm_concurrent_select_capacity ` , func ( ) float64 {
2023-01-07 03:19:05 +01:00
return float64 ( cap ( concurrencyLimitCh ) )
2019-08-05 17:27:50 +02:00
} )
_ = metrics . NewGauge ( ` vm_concurrent_select_current ` , func ( ) float64 {
2023-01-07 03:19:05 +01:00
return float64 ( len ( concurrencyLimitCh ) )
2019-08-05 17:27:50 +02:00
} )
)
2019-05-22 23:23:23 +02:00
func requestHandler ( w http . ResponseWriter , r * http . Request ) bool {
2020-12-14 13:02:57 +01:00
if r . URL . Path == "/" {
2023-02-23 03:58:44 +01:00
if r . Method != http . MethodGet {
2021-04-02 21:54:06 +02:00
return false
}
2021-07-07 16:28:06 +02:00
fmt . Fprintf ( w , ` vmselect - a component of VictoriaMetrics cluster < br / >
< a href = "https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html" > docs < / a > < br >
` )
2020-10-06 14:00:38 +02:00
return true
}
2021-07-07 16:28:06 +02:00
2020-02-04 15:13:59 +01:00
startTime := time . Now ( )
2021-07-07 16:28:06 +02:00
defer requestDuration . UpdateDuration ( startTime )
2022-06-01 01:31:40 +02:00
tracerEnabled := searchutils . GetBool ( r , "trace" )
2022-06-08 20:05:17 +02:00
qt := querytracer . New ( tracerEnabled , r . URL . Path )
2021-07-07 16:28:06 +02:00
2019-05-22 23:16:55 +02:00
// Limit the number of concurrent queries.
select {
2023-01-07 03:19:05 +01:00
case concurrencyLimitCh <- struct { } { } :
defer func ( ) { <- concurrencyLimitCh } ( )
2019-08-05 17:27:50 +02:00
default :
// Sleep for a while until giving up. This should resolve short bursts in requests.
concurrencyLimitReached . Inc ( )
2020-09-22 00:21:20 +02:00
d := searchutils . GetMaxQueryDuration ( r )
if d > * maxQueueDuration {
d = * maxQueueDuration
}
t := timerpool . Get ( d )
2019-08-05 17:27:50 +02:00
select {
2023-01-07 03:19:05 +01:00
case concurrencyLimitCh <- struct { } { } :
2019-08-05 17:27:50 +02:00
timerpool . Put ( t )
2023-01-07 03:19:05 +01:00
qt . Printf ( "wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed" , * maxConcurrentRequests )
defer func ( ) { <- concurrencyLimitCh } ( )
2023-05-03 10:42:17 +02:00
case <- r . Context ( ) . Done ( ) :
timerpool . Put ( t )
remoteAddr := httpserver . GetQuotedRemoteAddr ( r )
requestURI := httpserver . GetRequestURI ( r )
logger . Infof ( "client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q" ,
d . Seconds ( ) , remoteAddr , requestURI )
return true
2019-08-05 17:27:50 +02:00
case <- t . C :
timerpool . Put ( t )
concurrencyLimitTimeout . Inc ( )
2019-08-23 08:46:45 +02:00
err := & httpserver . ErrorWithStatusCode {
2023-01-07 03:59:39 +01:00
Err : fmt . Errorf ( "couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests " +
2023-01-07 09:11:44 +01:00
"are executed. Possible solutions: to reduce query load; to add more compute resources to the server; " +
2023-01-11 10:06:05 +01:00
"to increase -search.maxQueueDuration=%s; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests" ,
d . Seconds ( ) , * maxConcurrentRequests , maxQueueDuration ) ,
2019-08-23 08:46:45 +02:00
StatusCode : http . StatusServiceUnavailable ,
}
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2019-08-05 17:27:50 +02:00
return true
}
2019-05-22 23:16:55 +02:00
}
2021-06-18 18:04:42 +02:00
if * logSlowQueryDuration > 0 {
actualStartTime := time . Now ( )
defer func ( ) {
d := time . Since ( actualStartTime )
if d >= * logSlowQueryDuration {
remoteAddr := httpserver . GetQuotedRemoteAddr ( r )
2021-07-07 11:59:03 +02:00
requestURI := httpserver . GetRequestURI ( r )
2021-06-18 18:04:42 +02:00
logger . Warnf ( "slow query according to -search.logSlowQueryDuration=%s: remoteAddr=%s, duration=%.3f seconds; requestURI: %q" ,
* logSlowQueryDuration , remoteAddr , d . Seconds ( ) , requestURI )
slowQueries . Inc ( )
}
} ( )
}
2020-02-21 12:53:18 +01:00
path := strings . Replace ( r . URL . Path , "//" , "/" , - 1 )
2020-07-08 18:09:16 +02:00
if path == "/internal/resetRollupResultCache" {
2023-01-11 00:51:55 +01:00
if ! httpserver . CheckAuthFlag ( w , r , * resetCacheAuthKey , "resetCacheAuthKey" ) {
2020-07-08 18:09:16 +02:00
return true
}
promql . ResetRollupResultCache ( )
return true
2019-05-22 23:23:23 +02:00
}
2020-12-25 15:44:26 +01:00
if path == "/api/v1/status/top_queries" {
2020-12-27 11:53:50 +01:00
globalTopQueriesRequests . Inc ( )
2022-06-08 17:43:05 +02:00
httpserver . EnableCORS ( w , r )
2020-12-27 11:53:50 +01:00
if err := prometheus . QueryStatsHandler ( startTime , nil , w , r ) ; err != nil {
globalTopQueriesErrors . Inc ( )
sendPrometheusError ( w , r , err )
2020-12-25 15:44:26 +01:00
return true
}
return true
}
2022-11-25 19:32:45 +01:00
if path == "/admin/tenants" {
tenantsRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := prometheus . Tenants ( qt , startTime , w , r ) ; err != nil {
tenantsErrors . Inc ( )
httpserver . Errorf ( w , r , "error getting tenants: %s" , err )
return true
}
return true
}
2019-05-22 23:23:23 +02:00
p , err := httpserver . ParsePath ( path )
if err != nil {
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "cannot parse path %q: %s" , path , err )
2019-05-22 23:23:23 +02:00
return true
}
at , err := auth . NewToken ( p . AuthToken )
if err != nil {
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "auth error: %s" , err )
2019-05-22 23:23:23 +02:00
return true
}
switch p . Prefix {
case "select" :
2022-06-01 01:31:40 +02:00
return selectHandler ( qt , startTime , w , r , p , at )
2019-05-22 23:23:23 +02:00
case "delete" :
2020-02-04 15:13:59 +01:00
return deleteHandler ( startTime , w , r , p , at )
2019-05-22 23:23:23 +02:00
default :
// This is not our link
return false
}
}
2021-07-10 11:32:09 +02:00
//go:embed vmui
var vmuiFiles embed . FS
var vmuiFileServer = http . FileServer ( http . FS ( vmuiFiles ) )
2022-06-01 01:31:40 +02:00
func selectHandler ( qt * querytracer . Tracer , startTime time . Time , w http . ResponseWriter , r * http . Request , p * httpserver . Path , at * auth . Token ) bool {
2021-02-16 22:25:27 +01:00
defer func ( ) {
// Count per-tenant cumulative durations and total requests
httpRequests . Get ( at ) . Inc ( )
httpRequestsDuration . Get ( at ) . Add ( int ( time . Since ( startTime ) . Milliseconds ( ) ) )
} ( )
2021-08-29 11:04:23 +02:00
if p . Suffix == "" {
2023-02-23 03:58:44 +01:00
if r . Method != http . MethodGet {
2021-08-29 11:04:23 +02:00
return false
}
2022-03-21 14:34:49 +01:00
w . Header ( ) . Add ( "Content-Type" , "text/html; charset=utf-8" )
2021-08-29 11:04:23 +02:00
fmt . Fprintf ( w , "<h2>VictoriaMetrics cluster - vmselect</h2></br>" )
fmt . Fprintf ( w , "See <a href='https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format'>docs</a></br>" )
fmt . Fprintf ( w , "Useful endpoints:</br>" )
fmt . Fprintf ( w , ` <a href="vmui">Web UI</a><br> ` )
2022-12-15 01:23:19 +01:00
fmt . Fprintf ( w , ` <a href="prometheus/metric-relabel-debug">metric-level relabel debugging</a></br> ` )
fmt . Fprintf ( w , ` <a href="prometheus/target-relabel-debug">target-level relabel debugging</a></br> ` )
fmt . Fprintf ( w , ` <a href="prometheus/expand-with-exprs">WITH expressions' tutorial</a></br> ` )
2021-08-29 11:04:23 +02:00
fmt . Fprintf ( w , ` <a href="prometheus/api/v1/status/tsdb">tsdb status page</a><br> ` )
fmt . Fprintf ( w , ` <a href="prometheus/api/v1/status/top_queries">top queries</a><br> ` )
fmt . Fprintf ( w , ` <a href="prometheus/api/v1/status/active_queries">active queries</a><br> ` )
return true
}
2022-12-10 12:04:43 +01:00
if strings . HasPrefix ( p . Suffix , "static" ) {
prefix := strings . Join ( [ ] string { "" , p . Prefix , p . AuthToken } , "/" )
http . StripPrefix ( prefix , staticServer ) . ServeHTTP ( w , r )
return true
}
if strings . HasPrefix ( p . Suffix , "prometheus/static" ) {
prefix := strings . Join ( [ ] string { "" , p . Prefix , p . AuthToken } , "/" )
r . URL . Path = strings . Replace ( r . URL . Path , "/prometheus/static" , "/static" , 1 )
http . StripPrefix ( prefix , staticServer ) . ServeHTTP ( w , r )
return true
}
2022-07-11 18:48:18 +02:00
if p . Suffix == "vmui" || p . Suffix == "graph" || p . Suffix == "prometheus/vmui" || p . Suffix == "prometheus/graph" {
// VMUI access via incomplete url without `/` in the end. Redirect to complete url.
2021-09-21 12:28:12 +02:00
// Use relative redirect, since, since the hostname and path prefix may be incorrect if VictoriaMetrics
// is hidden behind vmauth or similar proxy.
_ = r . ParseForm ( )
2022-07-11 18:48:18 +02:00
suffix := strings . Replace ( p . Suffix , "prometheus/" , "../prometheus/" , 1 )
newURL := suffix + "/?" + r . Form . Encode ( )
2022-10-01 15:53:33 +02:00
httpserver . Redirect ( w , newURL )
2022-07-11 18:48:18 +02:00
return true
}
if strings . HasPrefix ( p . Suffix , "vmui/" ) || strings . HasPrefix ( p . Suffix , "prometheus/vmui/" ) {
// vmui access.
2023-01-12 08:06:07 +01:00
if p . Suffix == "vmui/custom-dashboards" || p . Suffix == "prometheus/vmui/custom-dashboards" {
2023-01-12 08:25:31 +01:00
if err := handleVMUICustomDashboards ( w ) ; err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
2023-01-12 08:06:07 +01:00
return true
}
2022-07-11 18:48:18 +02:00
prefix := strings . Join ( [ ] string { "" , p . Prefix , p . AuthToken } , "/" )
r . URL . Path = strings . Replace ( r . URL . Path , "/prometheus/vmui/" , "/vmui/" , 1 )
http . StripPrefix ( prefix , vmuiFileServer ) . ServeHTTP ( w , r )
2021-09-21 12:28:12 +02:00
return true
}
if strings . HasPrefix ( p . Suffix , "graph/" ) || strings . HasPrefix ( p . Suffix , "prometheus/graph/" ) {
2021-09-15 15:18:12 +02:00
// This is needed for serving /graph URLs from Prometheus datasource in Grafana.
2023-01-12 08:06:07 +01:00
if p . Suffix == "graph/custom-dashboards" || p . Suffix == "prometheus/graph/custom-dashboards" {
2023-01-12 08:25:31 +01:00
if err := handleVMUICustomDashboards ( w ) ; err != nil {
httpserver . Errorf ( w , r , "%s" , err )
return true
}
2023-01-12 08:06:07 +01:00
return true
}
2021-09-15 15:18:12 +02:00
prefix := strings . Join ( [ ] string { "" , p . Prefix , p . AuthToken } , "/" )
if strings . HasPrefix ( p . Suffix , "prometheus/graph/" ) {
r . URL . Path = strings . Replace ( r . URL . Path , "/prometheus/graph/" , "/vmui/" , 1 )
} else {
r . URL . Path = strings . Replace ( r . URL . Path , "/graph/" , "/vmui/" , 1 )
}
2021-07-09 16:04:28 +02:00
http . StripPrefix ( prefix , vmuiFileServer ) . ServeHTTP ( w , r )
2021-07-08 12:13:01 +02:00
return true
}
2019-05-22 23:23:23 +02:00
if strings . HasPrefix ( p . Suffix , "prometheus/api/v1/label/" ) {
s := p . Suffix [ len ( "prometheus/api/v1/label/" ) : ]
2019-05-22 23:16:55 +02:00
if strings . HasSuffix ( s , "/values" ) {
labelValuesRequests . Inc ( )
labelName := s [ : len ( s ) - len ( "/values" ) ]
httpserver . EnableCORS ( w , r )
2022-06-01 01:31:40 +02:00
if err := prometheus . LabelValuesHandler ( qt , startTime , at , labelName , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
labelValuesErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
}
}
2020-11-16 13:49:46 +01:00
if strings . HasPrefix ( p . Suffix , "graphite/tags/" ) && ! isGraphiteTagsPath ( p . Suffix [ len ( "graphite" ) : ] ) {
2020-11-16 02:31:09 +01:00
tagName := p . Suffix [ len ( "graphite/tags/" ) : ]
graphiteTagValuesRequests . Inc ( )
if err := graphite . TagValuesHandler ( startTime , at , tagName , w , r ) ; err != nil {
graphiteTagValuesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-16 02:31:09 +01:00
return true
}
return true
}
2021-09-16 10:21:07 +02:00
if strings . HasPrefix ( p . Suffix , "graphite/functions" ) {
2023-04-01 08:35:26 +02:00
funcName := p . Suffix [ len ( "graphite/functions" ) : ]
funcName = strings . TrimPrefix ( funcName , "/" )
if funcName == "" {
graphiteFunctionsRequests . Inc ( )
if err := graphite . FunctionsHandler ( startTime , w , r ) ; err != nil {
graphiteFunctionsErrors . Inc ( )
httpserver . Errorf ( w , r , "%s" , err )
return true
}
return true
}
graphiteFunctionDetailsRequests . Inc ( )
if err := graphite . FunctionDetailsHandler ( startTime , funcName , w , r ) ; err != nil {
graphiteFunctionDetailsErrors . Inc ( )
httpserver . Errorf ( w , r , "%s" , err )
return true
}
2021-09-16 10:21:07 +02:00
return true
}
2019-05-22 23:16:55 +02:00
2022-07-25 08:46:26 +02:00
if p . Suffix == "prometheus/vmalert" {
2022-12-07 05:58:16 +01:00
// vmalert access via incomplete url without `/` in the end. Redirect to complete url.
// Use relative redirect, since the hostname and path prefix may be incorrect if VictoriaMetrics
// is hidden behind vmauth or similar proxy.
2022-07-25 08:46:26 +02:00
path := "../" + p . Suffix + "/"
2022-10-01 15:53:33 +02:00
httpserver . Redirect ( w , path )
2022-07-11 18:33:33 +02:00
return true
}
2022-07-25 08:46:26 +02:00
if strings . HasPrefix ( p . Suffix , "prometheus/vmalert/" ) {
2022-07-06 10:47:51 +02:00
vmalertRequests . Inc ( )
if len ( * vmalertProxyURL ) == 0 {
w . WriteHeader ( http . StatusBadRequest )
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2022-07-06 11:44:46 +02:00
fmt . Fprintf ( w , "%s" , ` { "status":"error","msg":"for accessing vmalert flag '-vmalert.proxyURL' must be configured"} ` )
2022-07-06 10:47:51 +02:00
return true
}
proxyVMAlertRequests ( w , r , p . Suffix )
return true
}
2019-05-22 23:23:23 +02:00
switch p . Suffix {
case "prometheus/api/v1/query" :
2019-05-22 23:16:55 +02:00
queryRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2022-06-01 01:31:40 +02:00
if err := prometheus . QueryHandler ( qt , startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
queryErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/api/v1/query_range" :
2019-05-22 23:16:55 +02:00
queryRangeRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2022-06-01 01:31:40 +02:00
if err := prometheus . QueryRangeHandler ( qt , startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
queryRangeErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/api/v1/series" :
2019-05-22 23:16:55 +02:00
seriesRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2022-06-01 01:31:40 +02:00
if err := prometheus . SeriesHandler ( qt , startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
seriesErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/api/v1/series/count" :
2019-05-22 23:16:55 +02:00
seriesCountRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . SeriesCountHandler ( startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
seriesCountErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/api/v1/labels" :
2019-05-22 23:16:55 +02:00
labelsRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2022-06-01 01:31:40 +02:00
if err := prometheus . LabelsHandler ( qt , startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
labelsErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2020-04-22 18:57:36 +02:00
case "prometheus/api/v1/status/tsdb" :
2020-07-08 18:09:16 +02:00
statusTSDBRequests . Inc ( )
2022-06-08 17:43:05 +02:00
httpserver . EnableCORS ( w , r )
2022-06-09 18:46:26 +02:00
if err := prometheus . TSDBStatusHandler ( qt , startTime , at , w , r ) ; err != nil {
2020-07-08 18:09:16 +02:00
statusTSDBErrors . Inc ( )
2020-04-22 18:57:36 +02:00
sendPrometheusError ( w , r , err )
return true
}
return true
2020-07-08 18:09:16 +02:00
case "prometheus/api/v1/status/active_queries" :
statusActiveQueriesRequests . Inc ( )
promql . WriteActiveQueries ( w )
return true
2020-12-27 11:53:50 +01:00
case "prometheus/api/v1/status/top_queries" :
topQueriesRequests . Inc ( )
2022-06-08 17:43:05 +02:00
httpserver . EnableCORS ( w , r )
2020-12-27 11:53:50 +01:00
if err := prometheus . QueryStatsHandler ( startTime , at , w , r ) ; err != nil {
topQueriesErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/api/v1/export" :
2019-05-22 23:16:55 +02:00
exportRequests . Inc ( )
2020-02-04 15:13:59 +01:00
if err := prometheus . ExportHandler ( startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
exportErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2019-05-22 23:16:55 +02:00
return true
}
return true
2020-09-26 03:29:45 +02:00
case "prometheus/api/v1/export/native" :
exportNativeRequests . Inc ( )
if err := prometheus . ExportNativeHandler ( startTime , at , w , r ) ; err != nil {
exportNativeErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-09-26 03:29:45 +02:00
return true
}
return true
2020-10-12 19:01:51 +02:00
case "prometheus/api/v1/export/csv" :
exportCSVRequests . Inc ( )
if err := prometheus . ExportCSVHandler ( startTime , at , w , r ) ; err != nil {
exportCSVErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-10-12 19:01:51 +02:00
return true
}
return true
2019-05-22 23:23:23 +02:00
case "prometheus/federate" :
2019-05-22 23:16:55 +02:00
federateRequests . Inc ( )
2020-02-04 15:13:59 +01:00
if err := prometheus . FederateHandler ( startTime , at , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
federateErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2019-05-22 23:16:55 +02:00
return true
}
return true
2020-09-10 23:29:26 +02:00
case "graphite/metrics/find" , "graphite/metrics/find/" :
graphiteMetricsFindRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsFindHandler ( startTime , at , w , r ) ; err != nil {
graphiteMetricsFindErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-09-10 23:29:26 +02:00
return true
}
return true
case "graphite/metrics/expand" , "graphite/metrics/expand/" :
graphiteMetricsExpandRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsExpandHandler ( startTime , at , w , r ) ; err != nil {
graphiteMetricsExpandErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-09-10 23:29:26 +02:00
return true
}
return true
case "graphite/metrics/index.json" , "graphite/metrics/index.json/" :
graphiteMetricsIndexRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsIndexHandler ( startTime , at , w , r ) ; err != nil {
graphiteMetricsIndexErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-09-10 23:29:26 +02:00
return true
}
return true
2020-11-23 11:33:17 +01:00
case "graphite/tags/tagSeries" :
graphiteTagsTagSeriesRequests . Inc ( )
if err := graphite . TagsTagSeriesHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsTagSeriesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-23 11:33:17 +01:00
return true
}
return true
case "graphite/tags/tagMultiSeries" :
graphiteTagsTagMultiSeriesRequests . Inc ( )
if err := graphite . TagsTagMultiSeriesHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsTagMultiSeriesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-23 11:33:17 +01:00
return true
}
return true
2020-11-16 00:25:38 +01:00
case "graphite/tags" :
graphiteTagsRequests . Inc ( )
if err := graphite . TagsHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-16 00:25:38 +01:00
return true
}
return true
2020-11-16 09:55:55 +01:00
case "graphite/tags/findSeries" :
graphiteTagsFindSeriesRequests . Inc ( )
if err := graphite . TagsFindSeriesHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsFindSeriesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-16 09:55:55 +01:00
return true
}
return true
2020-11-16 16:56:19 +01:00
case "graphite/tags/autoComplete/tags" :
2020-11-16 13:49:46 +01:00
graphiteTagsAutoCompleteTagsRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . TagsAutoCompleteTagsHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsAutoCompleteTagsErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-16 13:49:46 +01:00
return true
}
return true
2020-11-16 16:56:19 +01:00
case "graphite/tags/autoComplete/values" :
2020-11-16 14:22:36 +01:00
graphiteTagsAutoCompleteValuesRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . TagsAutoCompleteValuesHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsAutoCompleteValuesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-16 14:22:36 +01:00
return true
}
return true
2020-11-23 14:26:20 +01:00
case "graphite/tags/delSeries" :
graphiteTagsDelSeriesRequests . Inc ( )
if err := graphite . TagsDelSeriesHandler ( startTime , at , w , r ) ; err != nil {
graphiteTagsDelSeriesErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2020-11-23 14:26:20 +01:00
return true
}
return true
2023-04-01 08:35:26 +02:00
case "graphite/render" :
graphiteRenderRequests . Inc ( )
if err := graphite . RenderHandler ( startTime , at , w , r ) ; err != nil {
graphiteRenderErrors . Inc ( )
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
return true
}
return true
2022-12-10 11:09:21 +01:00
case "prometheus/metric-relabel-debug" , "metric-relabel-debug" :
2023-01-04 07:40:00 +01:00
promrelabelMetricRelabelDebugRequests . Inc ( )
metric := r . FormValue ( "metric" )
relabelConfigs := r . FormValue ( "relabel_configs" )
2023-05-09 02:28:36 +02:00
format := r . FormValue ( "format" )
promrelabel . WriteMetricRelabelDebug ( w , "" , metric , relabelConfigs , format , nil )
2022-12-10 11:09:21 +01:00
return true
2022-12-10 21:44:09 +01:00
case "prometheus/target-relabel-debug" , "target-relabel-debug" :
2023-01-04 07:40:00 +01:00
promrelabelTargetRelabelDebugRequests . Inc ( )
metric := r . FormValue ( "metric" )
relabelConfigs := r . FormValue ( "relabel_configs" )
2023-05-09 02:28:36 +02:00
format := r . FormValue ( "format" )
promrelabel . WriteTargetRelabelDebug ( w , "" , metric , relabelConfigs , format , nil )
2022-12-10 21:44:09 +01:00
return true
2022-12-15 01:01:33 +01:00
case "prometheus/expand-with-exprs" , "expand-with-exprs" :
expandWithExprsRequests . Inc ( )
prometheus . ExpandWithExprs ( w , r )
return true
2021-07-29 08:48:43 +02:00
case "prometheus/api/v1/rules" , "prometheus/rules" :
2021-07-29 05:15:35 +02:00
rulesRequests . Inc ( )
2022-07-06 10:47:51 +02:00
if len ( * vmalertProxyURL ) > 0 {
proxyVMAlertRequests ( w , r , p . Suffix )
return true
}
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#rules
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprint ( w , ` { "status":"success","data": { "groups":[]}} ` )
2021-07-29 05:15:35 +02:00
return true
2021-08-02 16:28:09 +02:00
case "prometheus/api/v1/alerts" , "prometheus/alerts" :
2019-12-03 18:32:57 +01:00
alertsRequests . Inc ( )
2022-07-06 10:47:51 +02:00
if len ( * vmalertProxyURL ) > 0 {
proxyVMAlertRequests ( w , r , p . Suffix )
return true
}
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#alerts
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprint ( w , ` { "status":"success","data": { "alerts":[]}} ` )
2019-12-03 18:32:57 +01:00
return true
2020-02-04 14:53:15 +01:00
case "prometheus/api/v1/metadata" :
2021-04-05 22:25:05 +02:00
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata
2020-02-04 14:53:15 +01:00
metadataRequests . Inc ( )
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2020-02-04 14:53:15 +01:00
fmt . Fprintf ( w , "%s" , ` { "status":"success","data": { }} ` )
return true
2022-04-29 10:36:28 +02:00
case "prometheus/api/v1/status/buildinfo" :
buildInfoRequests . Inc ( )
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprintf ( w , "%s" , ` { "status":"success","data": { }} ` )
return true
2021-04-05 22:25:05 +02:00
case "prometheus/api/v1/query_exemplars" :
// Return dumb placeholder for https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars
queryExemplarsRequests . Inc ( )
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2021-12-23 10:53:50 +01:00
fmt . Fprintf ( w , "%s" , ` { "status":"success","data":[]} ` )
2021-04-05 22:25:05 +02:00
return true
2019-05-22 23:23:23 +02:00
default :
return false
}
}
2020-02-04 15:13:59 +01:00
func deleteHandler ( startTime time . Time , w http . ResponseWriter , r * http . Request , p * httpserver . Path , at * auth . Token ) bool {
2019-05-22 23:23:23 +02:00
switch p . Suffix {
case "prometheus/api/v1/admin/tsdb/delete_series" :
2019-05-22 23:16:55 +02:00
deleteRequests . Inc ( )
2020-02-04 15:13:59 +01:00
if err := prometheus . DeleteHandler ( startTime , at , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
deleteErrors . Inc ( )
2021-07-07 11:59:03 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2019-05-22 23:16:55 +02:00
return true
}
w . WriteHeader ( http . StatusNoContent )
return true
default :
return false
}
}
2020-11-16 13:49:46 +01:00
func isGraphiteTagsPath ( path string ) bool {
switch path {
// See https://graphite.readthedocs.io/en/stable/tags.html for a list of Graphite Tags API paths.
// Do not include `/tags/<tag_name>` here, since this will fool the caller.
case "/tags/tagSeries" , "/tags/tagMultiSeries" , "/tags/findSeries" ,
"/tags/autoComplete/tags" , "/tags/autoComplete/values" , "/tags/delSeries" :
return true
default :
return false
}
}
2019-05-22 23:16:55 +02:00
func sendPrometheusError ( w http . ResponseWriter , r * http . Request , err error ) {
2023-05-19 05:38:25 +02:00
logger . WarnfSkipframes ( 1 , "error in %q: %s" , httpserver . GetRequestURI ( r ) , err )
2019-05-22 23:16:55 +02:00
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-08-23 08:46:45 +02:00
statusCode := http . StatusUnprocessableEntity
2020-06-30 23:02:02 +02:00
var esc * httpserver . ErrorWithStatusCode
if errors . As ( err , & esc ) {
2019-08-23 08:46:45 +02:00
statusCode = esc . StatusCode
}
2019-05-22 23:16:55 +02:00
w . WriteHeader ( statusCode )
2022-08-15 12:38:47 +02:00
2022-08-15 12:50:14 +02:00
var ure * promql . UserReadableError
2022-08-15 12:38:47 +02:00
if errors . As ( err , & ure ) {
2022-08-15 12:50:14 +02:00
prometheus . WriteErrorResponse ( w , statusCode , ure )
2022-08-15 12:38:47 +02:00
return
}
2019-05-22 23:16:55 +02:00
prometheus . WriteErrorResponse ( w , statusCode , err )
}
var (
2021-07-07 12:25:16 +02:00
requestDuration = metrics . NewHistogram ( ` vmselect_request_duration_seconds ` )
2019-05-22 23:23:23 +02:00
labelValuesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/label/ { }/values"} ` )
2020-02-10 21:15:21 +01:00
labelValuesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/label/ { }/values"} ` )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
queryRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/query"} ` )
queryErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/query"} ` )
2019-05-22 23:16:55 +02:00
2020-02-10 21:15:21 +01:00
queryRangeRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/query_range"} ` )
2019-05-22 23:23:23 +02:00
queryRangeErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/query_range"} ` )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
seriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/series"} ` )
seriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/series"} ` )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
seriesCountRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/series/count"} ` )
seriesCountErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/series/count"} ` )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
labelsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/labels"} ` )
labelsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/labels"} ` )
2019-05-22 23:16:55 +02:00
2020-07-08 18:09:16 +02:00
statusTSDBRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/status/tsdb"} ` )
statusTSDBErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/status/tsdb"} ` )
statusActiveQueriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }prometheus/api/v1/status/active_queries"} ` )
2020-04-22 18:57:36 +02:00
2020-12-27 11:53:50 +01:00
topQueriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/status/top_queries"} ` )
topQueriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/status/top_queries"} ` )
globalTopQueriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/status/top_queries"} ` )
globalTopQueriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/status/top_queries"} ` )
2019-05-22 23:23:23 +02:00
deleteRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/delete/ { }/prometheus/api/v1/admin/tsdb/delete_series"} ` )
deleteErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/delete/ { }/prometheus/api/v1/admin/tsdb/delete_series"} ` )
2019-05-22 23:16:55 +02:00
2019-05-22 23:23:23 +02:00
exportRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/export"} ` )
exportErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/export"} ` )
2019-05-22 23:16:55 +02:00
2020-09-26 03:29:45 +02:00
exportNativeRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/export/native"} ` )
exportNativeErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/export/native"} ` )
2020-10-12 19:01:51 +02:00
exportCSVRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/export/csv"} ` )
exportCSVErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/api/v1/export/csv"} ` )
2019-05-22 23:23:23 +02:00
federateRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/federate"} ` )
federateErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/prometheus/federate"} ` )
2019-12-03 18:32:57 +01:00
2020-09-10 23:29:26 +02:00
graphiteMetricsFindRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/metrics/find"} ` )
graphiteMetricsFindErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/metrics/find"} ` )
graphiteMetricsExpandRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/metrics/expand"} ` )
graphiteMetricsExpandErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/metrics/expand"} ` )
graphiteMetricsIndexRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/metrics/index.json"} ` )
graphiteMetricsIndexErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/metrics/index.json"} ` )
2020-11-23 11:33:17 +01:00
graphiteTagsTagSeriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/tagSeries"} ` )
graphiteTagsTagSeriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/tagSeries"} ` )
graphiteTagsTagMultiSeriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/tagMultiSeries"} ` )
graphiteTagsTagMultiSeriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/tagMultiSeries"} ` )
2020-11-16 00:25:38 +01:00
graphiteTagsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags"} ` )
graphiteTagsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags"} ` )
2020-11-16 02:31:09 +01:00
graphiteTagValuesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/<tag_name>"} ` )
graphiteTagValuesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/<tag_name>"} ` )
2020-11-16 09:55:55 +01:00
graphiteTagsFindSeriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/findSeries"} ` )
graphiteTagsFindSeriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/findSeries"} ` )
2020-11-16 13:49:46 +01:00
graphiteTagsAutoCompleteTagsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/autoComplete/tags"} ` )
graphiteTagsAutoCompleteTagsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/autoComplete/tags"} ` )
2020-11-16 14:22:36 +01:00
graphiteTagsAutoCompleteValuesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/autoComplete/values"} ` )
graphiteTagsAutoCompleteValuesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/autoComplete/values"} ` )
2020-11-23 14:26:20 +01:00
graphiteTagsDelSeriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/tags/delSeries"} ` )
graphiteTagsDelSeriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/tags/delSeries"} ` )
2023-04-01 08:35:26 +02:00
graphiteRenderRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/render"} ` )
graphiteRenderErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/render"} ` )
2022-06-22 12:14:47 +02:00
graphiteFunctionsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/functions"} ` )
2023-04-01 08:35:26 +02:00
graphiteFunctionsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/functions"} ` )
graphiteFunctionDetailsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/graphite/functions/<func_name>"} ` )
graphiteFunctionDetailsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/select/ { }/graphite/functions/<func_name>"} ` )
2021-09-15 08:44:05 +02:00
2023-01-04 07:40:00 +01:00
promrelabelMetricRelabelDebugRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/metric-relabel-debug"} ` )
promrelabelTargetRelabelDebugRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/target-relabel-debug"} ` )
2022-12-10 11:09:21 +01:00
2022-12-15 01:01:33 +01:00
expandWithExprsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/expand-with-exprs"} ` )
2022-07-06 10:47:51 +02:00
vmalertRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/vmalert"} ` )
rulesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/rules"} ` )
alertsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/alerts"} ` )
2021-04-05 22:25:05 +02:00
metadataRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/metadata"} ` )
2022-04-29 10:36:28 +02:00
buildInfoRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/buildinfo"} ` )
2021-04-05 22:25:05 +02:00
queryExemplarsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/select/ { }/prometheus/api/v1/query_exemplars"} ` )
2021-02-16 22:25:27 +01:00
2022-11-25 19:32:45 +01:00
tenantsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/admin/tenants"} ` )
tenantsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/admin/tenants"} ` )
2021-03-29 10:59:04 +02:00
httpRequests = tenantmetrics . NewCounterMap ( ` vm_tenant_select_requests_total ` )
httpRequestsDuration = tenantmetrics . NewCounterMap ( ` vm_tenant_select_requests_duration_ms_total ` )
2019-05-22 23:16:55 +02:00
)
2020-12-03 20:40:30 +01:00
func usage ( ) {
const s = `
vmselect processes incoming queries by fetching the requested data from vmstorage nodes configured via - storageNode .
2021-04-20 19:16:17 +02:00
See the docs at https : //docs.victoriametrics.com/Cluster-VictoriaMetrics.html .
2020-12-03 20:40:30 +01:00
`
flagutil . Usage ( s )
}
2022-05-03 19:55:15 +02:00
2022-07-06 10:47:51 +02:00
func proxyVMAlertRequests ( w http . ResponseWriter , r * http . Request , path string ) {
2022-05-03 19:55:15 +02:00
defer func ( ) {
err := recover ( )
if err == nil || err == http . ErrAbortHandler {
// Suppress http.ErrAbortHandler panic.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1353
return
}
// Forward other panics to the caller.
panic ( err )
} ( )
r . URL . Path = strings . TrimPrefix ( path , "prometheus" )
2022-05-04 22:35:57 +02:00
r . Host = vmalertProxyHost
vmalertProxy . ServeHTTP ( w , r )
2022-05-03 19:55:15 +02:00
}
2022-05-04 22:35:57 +02:00
var (
vmalertProxyHost string
vmalertProxy * httputil . ReverseProxy
)
2022-05-03 19:55:15 +02:00
// initVMAlertProxy must be called after flag.Parse(), since it uses command-line flags.
func initVMAlertProxy ( ) {
if len ( * vmalertProxyURL ) == 0 {
return
}
proxyURL , err := url . Parse ( * vmalertProxyURL )
if err != nil {
2022-05-04 22:35:57 +02:00
logger . Fatalf ( "cannot parse -vmalert.proxyURL=%q: %s" , * vmalertProxyURL , err )
2022-05-03 19:55:15 +02:00
}
2022-05-04 22:35:57 +02:00
vmalertProxyHost = proxyURL . Host
vmalertProxy = httputil . NewSingleHostReverseProxy ( proxyURL )
2022-05-03 19:55:15 +02:00
}
2022-09-08 20:17:58 +02:00
func checkDuplicates ( arr [ ] string ) string {
visited := make ( map [ string ] struct { } )
for _ , s := range arr {
if _ , ok := visited [ s ] ; ok {
return s
}
visited [ s ] = struct { } { }
}
return ""
}
2022-11-01 13:54:55 +01:00
func hasEmptyValues ( arr [ ] string ) bool {
for _ , s := range arr {
if s == "" {
return true
}
}
return false
}