2019-05-22 23:16:55 +02:00
package vmselect
import (
2020-06-30 23:02:02 +02:00
"errors"
2019-05-22 23:16:55 +02:00
"flag"
2019-08-23 08:46:45 +02:00
"fmt"
2019-05-22 23:16:55 +02:00
"net/http"
"runtime"
"strings"
"time"
2020-09-10 23:28:19 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/graphite"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
2020-09-22 00:21:20 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
2019-05-28 16:17:19 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/metrics"
)
var (
deleteAuthKey = flag . String ( "deleteAuthKey" , "" , "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series" )
2020-01-17 14:43:47 +01:00
maxConcurrentRequests = flag . Int ( "search.maxConcurrentRequests" , getDefaultMaxConcurrentRequests ( ) , "The maximum number of concurrent search requests. " +
2020-02-04 14:46:13 +01:00
"It shouldn't be high, since a single request can saturate all the CPU cores. See also -search.maxQueueDuration" )
2020-09-22 00:21:20 +02:00
maxQueueDuration = flag . Duration ( "search.maxQueueDuration" , 10 * time . Second , "The maximum time the request waits for execution when -search.maxConcurrentRequests " +
"limit is reached; see also -search.maxQueryDuration" )
2020-07-08 17:55:25 +02:00
resetCacheAuthKey = flag . String ( "search.resetCacheAuthKey" , "" , "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call" )
2019-05-22 23:16:55 +02:00
)
2020-01-17 14:43:47 +01:00
func getDefaultMaxConcurrentRequests ( ) int {
n := runtime . GOMAXPROCS ( - 1 )
if n <= 4 {
n *= 2
}
if n > 16 {
// A single request can saturate all the CPU cores, so there is no sense
// in allowing higher number of concurrent requests - they will just contend
// for unavailable CPU time.
n = 16
}
return n
}
2019-05-22 23:16:55 +02:00
// Init initializes vmselect
func Init ( ) {
promql . InitRollupResultCache ( * vmstorage . DataPath + "/cache/rollupResult" )
2019-08-05 17:27:50 +02:00
2019-05-22 23:16:55 +02:00
concurrencyCh = make ( chan struct { } , * maxConcurrentRequests )
}
// Stop stops vmselect
func Stop ( ) {
promql . StopRollupResultCache ( )
}
2019-08-05 17:27:50 +02:00
var concurrencyCh chan struct { }
var (
concurrencyLimitReached = metrics . NewCounter ( ` vm_concurrent_select_limit_reached_total ` )
concurrencyLimitTimeout = metrics . NewCounter ( ` vm_concurrent_select_limit_timeout_total ` )
_ = metrics . NewGauge ( ` vm_concurrent_select_capacity ` , func ( ) float64 {
return float64 ( cap ( concurrencyCh ) )
} )
_ = metrics . NewGauge ( ` vm_concurrent_select_current ` , func ( ) float64 {
return float64 ( len ( concurrencyCh ) )
} )
)
2019-05-22 23:16:55 +02:00
// RequestHandler handles remote read API requests for Prometheus
func RequestHandler ( w http . ResponseWriter , r * http . Request ) bool {
2020-02-04 15:13:59 +01:00
startTime := time . Now ( )
2019-05-22 23:16:55 +02:00
// Limit the number of concurrent queries.
select {
case concurrencyCh <- struct { } { } :
defer func ( ) { <- concurrencyCh } ( )
2019-08-05 17:27:50 +02:00
default :
// Sleep for a while until giving up. This should resolve short bursts in requests.
concurrencyLimitReached . Inc ( )
2020-09-22 00:21:20 +02:00
d := searchutils . GetMaxQueryDuration ( r )
if d > * maxQueueDuration {
d = * maxQueueDuration
}
t := timerpool . Get ( d )
2019-08-05 17:27:50 +02:00
select {
case concurrencyCh <- struct { } { } :
timerpool . Put ( t )
defer func ( ) { <- concurrencyCh } ( )
case <- t . C :
timerpool . Put ( t )
concurrencyLimitTimeout . Inc ( )
2019-08-23 08:46:45 +02:00
err := & httpserver . ErrorWithStatusCode {
2020-01-17 12:24:37 +01:00
Err : fmt . Errorf ( "cannot handle more than %d concurrent search requests during %s; possible solutions: " +
2020-09-22 00:21:20 +02:00
"increase `-search.maxQueueDuration`; increase `-search.maxQueryDuration`; increase `-search.maxConcurrentRequests`; " +
"increase server capacity" ,
* maxConcurrentRequests , d ) ,
2019-08-23 08:46:45 +02:00
StatusCode : http . StatusServiceUnavailable ,
}
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "%s" , err )
2019-08-05 17:27:50 +02:00
return true
}
2019-05-22 23:16:55 +02:00
}
path := strings . Replace ( r . URL . Path , "//" , "/" , - 1 )
2020-07-08 17:55:25 +02:00
if path == "/internal/resetRollupResultCache" {
if len ( * resetCacheAuthKey ) > 0 && r . FormValue ( "authKey" ) != * resetCacheAuthKey {
sendPrometheusError ( w , r , fmt . Errorf ( "invalid authKey=%q for %q" , r . FormValue ( "authKey" ) , path ) )
return true
}
promql . ResetRollupResultCache ( )
return true
2020-02-21 12:53:18 +01:00
}
2019-05-22 23:16:55 +02:00
if strings . HasPrefix ( path , "/api/v1/label/" ) {
s := r . URL . Path [ len ( "/api/v1/label/" ) : ]
if strings . HasSuffix ( s , "/values" ) {
labelValuesRequests . Inc ( )
labelName := s [ : len ( s ) - len ( "/values" ) ]
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . LabelValuesHandler ( startTime , labelName , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
labelValuesErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
}
}
switch path {
case "/api/v1/query" :
queryRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . QueryHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
queryErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
case "/api/v1/query_range" :
queryRangeRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . QueryRangeHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
queryRangeErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
case "/api/v1/series" :
seriesRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . SeriesHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
seriesErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
case "/api/v1/series/count" :
seriesCountRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . SeriesCountHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
seriesCountErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
case "/api/v1/labels" :
labelsRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . LabelsHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
labelsErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2019-06-10 17:55:20 +02:00
case "/api/v1/labels/count" :
labelsCountRequests . Inc ( )
httpserver . EnableCORS ( w , r )
2020-02-04 15:13:59 +01:00
if err := prometheus . LabelsCountHandler ( startTime , w , r ) ; err != nil {
2019-06-10 17:55:20 +02:00
labelsCountErrors . Inc ( )
sendPrometheusError ( w , r , err )
return true
}
return true
2020-04-22 18:57:36 +02:00
case "/api/v1/status/tsdb" :
2020-07-08 17:55:25 +02:00
statusTSDBRequests . Inc ( )
2020-04-22 18:57:36 +02:00
if err := prometheus . TSDBStatusHandler ( startTime , w , r ) ; err != nil {
2020-07-08 17:55:25 +02:00
statusTSDBErrors . Inc ( )
2020-04-22 18:57:36 +02:00
sendPrometheusError ( w , r , err )
return true
}
return true
2020-07-08 17:55:25 +02:00
case "/api/v1/status/active_queries" :
statusActiveQueriesRequests . Inc ( )
promql . WriteActiveQueries ( w )
return true
2019-05-22 23:16:55 +02:00
case "/api/v1/export" :
exportRequests . Inc ( )
2020-02-04 15:13:59 +01:00
if err := prometheus . ExportHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
exportErrors . Inc ( )
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
2019-05-22 23:16:55 +02:00
return true
}
return true
case "/federate" :
federateRequests . Inc ( )
2020-02-04 15:13:59 +01:00
if err := prometheus . FederateHandler ( startTime , w , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
federateErrors . Inc ( )
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
2019-05-22 23:16:55 +02:00
return true
}
return true
2020-09-10 23:28:19 +02:00
case "/metrics/find" , "/metrics/find/" :
graphiteMetricsFindRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsFindHandler ( startTime , w , r ) ; err != nil {
graphiteMetricsFindErrors . Inc ( )
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
return true
}
return true
case "/metrics/expand" , "/metrics/expand/" :
graphiteMetricsExpandRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsExpandHandler ( startTime , w , r ) ; err != nil {
graphiteMetricsExpandErrors . Inc ( )
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
return true
}
return true
case "/metrics/index.json" , "/metrics/index.json/" :
graphiteMetricsIndexRequests . Inc ( )
httpserver . EnableCORS ( w , r )
if err := graphite . MetricsIndexHandler ( startTime , w , r ) ; err != nil {
graphiteMetricsIndexErrors . Inc ( )
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
return true
}
return true
2019-12-03 18:32:57 +01:00
case "/api/v1/rules" :
// Return dumb placeholder
rulesRequests . Inc ( )
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprintf ( w , "%s" , ` { "status":"success","data": { "groups":[]}} ` )
return true
case "/api/v1/alerts" :
// Return dumb placehloder
alertsRequests . Inc ( )
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprintf ( w , "%s" , ` { "status":"success","data": { "alerts":[]}} ` )
return true
2020-02-04 14:53:15 +01:00
case "/api/v1/metadata" :
// Return dumb placeholder
metadataRequests . Inc ( )
w . Header ( ) . Set ( "Content-Type" , "application/json" )
fmt . Fprintf ( w , "%s" , ` { "status":"success","data": { }} ` )
return true
2019-05-22 23:16:55 +02:00
case "/api/v1/admin/tsdb/delete_series" :
deleteRequests . Inc ( )
authKey := r . FormValue ( "authKey" )
if authKey != * deleteAuthKey {
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "invalid authKey %q. It must match the value from -deleteAuthKey command line flag" , authKey )
2019-05-22 23:16:55 +02:00
return true
}
2020-02-04 15:13:59 +01:00
if err := prometheus . DeleteHandler ( startTime , r ) ; err != nil {
2019-05-22 23:16:55 +02:00
deleteErrors . Inc ( )
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "error in %q: %s" , r . URL . Path , err )
2019-05-22 23:16:55 +02:00
return true
}
w . WriteHeader ( http . StatusNoContent )
return true
default :
return false
}
}
func sendPrometheusError ( w http . ResponseWriter , r * http . Request , err error ) {
2020-04-15 19:50:12 +02:00
logger . Warnf ( "error in %q: %s" , r . RequestURI , err )
2019-05-22 23:16:55 +02:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-08-23 08:46:45 +02:00
statusCode := http . StatusUnprocessableEntity
2020-06-30 23:02:02 +02:00
var esc * httpserver . ErrorWithStatusCode
if errors . As ( err , & esc ) {
2019-08-23 08:46:45 +02:00
statusCode = esc . StatusCode
}
2019-05-22 23:16:55 +02:00
w . WriteHeader ( statusCode )
prometheus . WriteErrorResponse ( w , statusCode , err )
}
var (
labelValuesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/label/ { }/values"} ` )
labelValuesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/label/ { }/values"} ` )
queryRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/query"} ` )
queryErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/query"} ` )
queryRangeRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/query_range"} ` )
queryRangeErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/query_range"} ` )
seriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/series"} ` )
seriesErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/series"} ` )
seriesCountRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/series/count"} ` )
seriesCountErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/series/count"} ` )
labelsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/labels"} ` )
labelsErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/labels"} ` )
2019-06-10 17:55:20 +02:00
labelsCountRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/labels/count"} ` )
labelsCountErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/labels/count"} ` )
2020-07-08 17:55:25 +02:00
statusTSDBRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/status/tsdb"} ` )
statusTSDBErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/status/tsdb"} ` )
statusActiveQueriesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/status/active_queries"} ` )
2020-04-22 18:57:36 +02:00
2019-05-22 23:16:55 +02:00
deleteRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/admin/tsdb/delete_series"} ` )
deleteErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/admin/tsdb/delete_series"} ` )
exportRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/export"} ` )
exportErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/api/v1/export"} ` )
federateRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/federate"} ` )
federateErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/federate"} ` )
2019-12-03 18:32:57 +01:00
2020-09-10 23:28:19 +02:00
graphiteMetricsFindRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/metrics/find"} ` )
graphiteMetricsFindErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/metrics/find"} ` )
graphiteMetricsExpandRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/metrics/expand"} ` )
graphiteMetricsExpandErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/metrics/expand"} ` )
graphiteMetricsIndexRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/metrics/index.json"} ` )
graphiteMetricsIndexErrors = metrics . NewCounter ( ` vm_http_request_errors_total { path="/metrics/index.json"} ` )
2020-02-04 14:53:15 +01:00
rulesRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/rules"} ` )
alertsRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/alerts"} ` )
metadataRequests = metrics . NewCounter ( ` vm_http_requests_total { path="/api/v1/metadata"} ` )
2019-05-22 23:16:55 +02:00
)