mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-23 20:37:12 +01:00
app/vmselect/prometheus: extract common code for obtaining common query args into getCommonParams() function
This commit is contained in:
Aliaksandr Valialkin
parent
2bcb960f17
commit
483b402bb2
@ -22,7 +22,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
@ -58,8 +57,10 @@ const defaultStep = 5 * 60 * 1000
|
||||
func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer federateDuration.UpdateDuration(startTime)
|
||||
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
cp, err := getCommonParams(r, startTime, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lookbackDelta, err := getMaxLookback(r)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -67,23 +68,11 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
|
||||
if lookbackDelta <= 0 {
|
||||
lookbackDelta = defaultStep
|
||||
}
|
||||
start, err := searchutils.GetTime(r, "start", ct-lookbackDelta)
|
||||
if err != nil {
|
||||
return err
|
||||
if cp.IsDefaultTimeRange() {
|
||||
cp.start = cp.end - lookbackDelta
|
||||
}
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if start >= end {
|
||||
start = end - defaultStep
|
||||
}
|
||||
tagFilterss, err := getTagFilterssFromRequest(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxFederateSeries)
|
||||
rss, err := netstorage.ProcessSearchQuery(nil, sq, true, deadline)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxFederateSeries)
|
||||
rss, err := netstorage.ProcessSearchQuery(nil, sq, true, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -116,18 +105,19 @@ var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/fe
|
||||
func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer exportCSVDuration.UpdateDuration(startTime)
|
||||
|
||||
cp, err := getExportParams(r, startTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
format := r.FormValue("format")
|
||||
if len(format) == 0 {
|
||||
return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data")
|
||||
}
|
||||
fieldNames := strings.Split(format, ",")
|
||||
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
|
||||
ep, err := getExportParams(r, startTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries)
|
||||
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@ -143,7 +133,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
doneCh := make(chan error, 1)
|
||||
if !reduceMemUsage {
|
||||
rss, err := netstorage.ProcessSearchQuery(nil, sq, true, ep.deadline)
|
||||
rss, err := netstorage.ProcessSearchQuery(nil, sq, true, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -166,7 +156,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
|
||||
}()
|
||||
} else {
|
||||
go func() {
|
||||
err := netstorage.ExportBlocks(nil, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
err := netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
if err := bw.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -207,24 +197,24 @@ var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/a
|
||||
func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer exportNativeDuration.UpdateDuration(startTime)
|
||||
|
||||
ep, err := getExportParams(r, startTime)
|
||||
cp, err := getExportParams(r, startTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries)
|
||||
w.Header().Set("Content-Type", "VictoriaMetrics/native")
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
|
||||
// Marshal tr
|
||||
trBuf := make([]byte, 0, 16)
|
||||
trBuf = encoding.MarshalInt64(trBuf, ep.start)
|
||||
trBuf = encoding.MarshalInt64(trBuf, ep.end)
|
||||
trBuf = encoding.MarshalInt64(trBuf, cp.start)
|
||||
trBuf = encoding.MarshalInt64(trBuf, cp.end)
|
||||
_, _ = bw.Write(trBuf)
|
||||
|
||||
// Marshal native blocks.
|
||||
err = netstorage.ExportBlocks(nil, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
err = netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
if err := bw.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -269,22 +259,22 @@ var bbPool bytesutil.ByteBufferPool
|
||||
func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer exportDuration.UpdateDuration(startTime)
|
||||
|
||||
ep, err := getExportParams(r, startTime)
|
||||
cp, err := getExportParams(r, startTime)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format := r.FormValue("format")
|
||||
maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line")))
|
||||
reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage")
|
||||
if err := exportHandler(nil, w, ep, format, maxRowsPerLine, reduceMemUsage); err != nil {
|
||||
return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", ep.start, ep.end, err)
|
||||
if err := exportHandler(nil, w, cp, format, maxRowsPerLine, reduceMemUsage); err != nil {
|
||||
return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", cp.start, cp.end, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`)
|
||||
|
||||
func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportParams, format string, maxRowsPerLine int, reduceMemUsage bool) error {
|
||||
func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonParams, format string, maxRowsPerLine int, reduceMemUsage bool) error {
|
||||
writeResponseFunc := WriteExportStdResponse
|
||||
writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) {
|
||||
bb := quicktemplate.AcquireByteBuffer()
|
||||
@ -337,7 +327,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara
|
||||
}
|
||||
}
|
||||
|
||||
sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries)
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
bw := bufferedwriter.Get(w)
|
||||
defer bufferedwriter.Put(bw)
|
||||
@ -345,7 +335,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara
|
||||
resultsCh := make(chan *quicktemplate.ByteBuffer, cgroup.AvailableCPUs())
|
||||
doneCh := make(chan error, 1)
|
||||
if !reduceMemUsage {
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, true, ep.deadline)
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, true, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -371,7 +361,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara
|
||||
} else {
|
||||
qtChild := qt.NewChild("background export format=%s", format)
|
||||
go func() {
|
||||
err := netstorage.ExportBlocks(qtChild, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
err := netstorage.ExportBlocks(qtChild, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error {
|
||||
if err := bw.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -430,17 +420,15 @@ var exportBlockPool = &sync.Pool{
|
||||
func DeleteHandler(startTime time.Time, r *http.Request) error {
|
||||
defer deleteDuration.UpdateDuration(startTime)
|
||||
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
if r.FormValue("start") != "" || r.FormValue("end") != "" {
|
||||
return fmt.Errorf("start and end aren't supported. Remove these args from the query in order to delete all the matching metrics")
|
||||
}
|
||||
tagFilterss, err := getTagFilterssFromRequest(r)
|
||||
cp, err := getCommonParams(r, startTime, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
sq := storage.NewSearchQuery(0, ct, tagFilterss, 0)
|
||||
deletedCount, err := netstorage.DeleteSeries(nil, sq, deadline)
|
||||
if !cp.IsDefaultTimeRange() {
|
||||
return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end)
|
||||
}
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, 0)
|
||||
deletedCount, err := netstorage.DeleteSeries(nil, sq, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot delete time series: %w", err)
|
||||
}
|
||||
@ -458,35 +446,26 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/
|
||||
func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error {
|
||||
defer labelValuesDuration.UpdateDuration(startTime)
|
||||
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
etfs, err := searchutils.GetExtraTagFilters(r)
|
||||
cp, err := getCommonParams(r, startTime, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matches := getMatchesFromRequest(r)
|
||||
var labelValues []string
|
||||
if len(matches) == 0 && len(etfs) == 0 {
|
||||
if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
|
||||
var err error
|
||||
labelValues, err = netstorage.GetLabelValues(qt, labelName, deadline)
|
||||
if len(cp.filterss) == 0 {
|
||||
if cp.IsDefaultTimeRange() {
|
||||
labelValues, err = netstorage.GetLabelValues(qt, labelName, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain label values for %q: %w`, labelName, err)
|
||||
}
|
||||
} else {
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start, err := searchutils.GetTime(r, "start", end-defaultStep)
|
||||
if err != nil {
|
||||
return err
|
||||
if cp.start == 0 {
|
||||
cp.start = cp.end - defaultStep
|
||||
}
|
||||
tr := storage.TimeRange{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
MinTimestamp: cp.start,
|
||||
MaxTimestamp: cp.end,
|
||||
}
|
||||
labelValues, err = netstorage.GetLabelValuesOnTimeRange(qt, labelName, tr, deadline)
|
||||
labelValues, err = netstorage.GetLabelValuesOnTimeRange(qt, labelName, tr, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain label values on time range for %q: %w`, labelName, err)
|
||||
}
|
||||
@ -496,21 +475,12 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
|
||||
// i.e. /api/v1/label/foo/values?match[]=foobar{baz="abc"}&start=...&end=...
|
||||
// is equivalent to `label_values(foobar{baz="abc"}, foo)` call on the selected
|
||||
// time range in Grafana templating.
|
||||
if len(matches) == 0 {
|
||||
matches = []string{fmt.Sprintf("{%s!=''}", labelName)}
|
||||
if cp.start == 0 {
|
||||
cp.start = cp.end - defaultStep
|
||||
}
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
labelValues, err = labelValuesWithMatches(qt, labelName, cp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start, err := searchutils.GetTime(r, "start", end-defaultStep)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
labelValues, err = labelValuesWithMatches(qt, labelName, matches, etfs, start, end, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %w", labelName, matches, start, end, err)
|
||||
return fmt.Errorf("cannot obtain label values for %q on time range [%d...%d]: %w", labelName, cp.start, cp.end, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -524,37 +494,24 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
|
||||
return nil
|
||||
}
|
||||
|
||||
func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, matches []string, etfs [][]storage.TagFilter,
|
||||
start, end int64, deadline searchutils.Deadline) ([]string, error) {
|
||||
tagFilterss, err := getTagFilterssFromMatches(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, cp *commonParams) ([]string, error) {
|
||||
// Add `labelName!=''` tag filter in order to filter out series without the labelName.
|
||||
// There is no need in adding `__name__!=''` filter, since all the time series should
|
||||
// already have non-empty name.
|
||||
if labelName != "__name__" {
|
||||
key := []byte(labelName)
|
||||
for i, tfs := range tagFilterss {
|
||||
tagFilterss[i] = append(tfs, storage.TagFilter{
|
||||
for i, tfs := range cp.filterss {
|
||||
cp.filterss[i] = append(tfs, storage.TagFilter{
|
||||
Key: key,
|
||||
IsNegative: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
if len(tagFilterss) == 0 {
|
||||
logger.Panicf("BUG: tagFilterss must be non-empty")
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
|
||||
m := make(map[string]struct{})
|
||||
if end-start > 24*3600*1000 {
|
||||
if cp.end-cp.start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, deadline)
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
@ -566,7 +523,7 @@ func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, matches []
|
||||
m[string(labelValue)] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline)
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -627,12 +584,11 @@ const secsPerDay = 3600 * 24
|
||||
func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer tsdbStatusDuration.UpdateDuration(startTime)
|
||||
|
||||
deadline := searchutils.GetDeadlineForStatusRequest(r, startTime)
|
||||
etfs, err := searchutils.GetExtraTagFilters(r)
|
||||
cp, err := getCommonParams(r, startTime, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matches := getMatchesFromRequest(r)
|
||||
cp.deadline = searchutils.GetDeadlineForStatusRequest(r, startTime)
|
||||
|
||||
date := fasttime.UnixDate()
|
||||
dateStr := r.FormValue("date")
|
||||
@ -659,13 +615,13 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
|
||||
topN = n
|
||||
}
|
||||
var status *storage.TSDBStatus
|
||||
if len(matches) == 0 && len(etfs) == 0 {
|
||||
status, err = netstorage.GetTSDBStatusForDate(qt, deadline, date, topN, *maxTSDBStatusSeries)
|
||||
if len(cp.filterss) == 0 {
|
||||
status, err = netstorage.GetTSDBStatusForDate(qt, cp.deadline, date, topN, *maxTSDBStatusSeries)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err)
|
||||
}
|
||||
} else {
|
||||
status, err = tsdbStatusWithMatches(qt, matches, etfs, date, topN, *maxTSDBStatusSeries, deadline)
|
||||
status, err = tsdbStatusWithMatches(qt, cp.filterss, date, topN, *maxTSDBStatusSeries, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain tsdb status with matches for date=%d, topN=%d: %w", date, topN, err)
|
||||
}
|
||||
@ -680,18 +636,10 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
|
||||
return nil
|
||||
}
|
||||
|
||||
func tsdbStatusWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storage.TagFilter, date uint64, topN, maxMetrics int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
|
||||
tagFilterss, err := getTagFilterssFromMatches(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
if len(tagFilterss) == 0 {
|
||||
logger.Panicf("BUG: tagFilterss must be non-empty")
|
||||
}
|
||||
func tsdbStatusWithMatches(qt *querytracer.Tracer, filterss [][]storage.TagFilter, date uint64, topN, maxMetrics int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) {
|
||||
start := int64(date*secsPerDay) * 1000
|
||||
end := int64(date*secsPerDay+secsPerDay) * 1000
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss, maxMetrics)
|
||||
sq := storage.NewSearchQuery(start, end, filterss, maxMetrics)
|
||||
status, err := netstorage.GetTSDBStatusWithFilters(qt, deadline, sq, topN)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -707,35 +655,26 @@ var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/
|
||||
func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer labelsDuration.UpdateDuration(startTime)
|
||||
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
etfs, err := searchutils.GetExtraTagFilters(r)
|
||||
cp, err := getCommonParams(r, startTime, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matches := getMatchesFromRequest(r)
|
||||
var labels []string
|
||||
if len(matches) == 0 && len(etfs) == 0 {
|
||||
if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 {
|
||||
var err error
|
||||
labels, err = netstorage.GetLabels(qt, deadline)
|
||||
if len(cp.filterss) == 0 {
|
||||
if cp.IsDefaultTimeRange() {
|
||||
labels, err = netstorage.GetLabels(qt, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain labels: %w", err)
|
||||
}
|
||||
} else {
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start, err := searchutils.GetTime(r, "start", end-defaultStep)
|
||||
if err != nil {
|
||||
return err
|
||||
if cp.start == 0 {
|
||||
cp.start = cp.end - defaultStep
|
||||
}
|
||||
tr := storage.TimeRange{
|
||||
MinTimestamp: start,
|
||||
MaxTimestamp: end,
|
||||
MinTimestamp: cp.start,
|
||||
MaxTimestamp: cp.end,
|
||||
}
|
||||
labels, err = netstorage.GetLabelsOnTimeRange(qt, tr, deadline)
|
||||
labels, err = netstorage.GetLabelsOnTimeRange(qt, tr, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain labels on time range: %w", err)
|
||||
}
|
||||
@ -743,21 +682,12 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
|
||||
} else {
|
||||
// Extended functionality that allows filtering by label filters and time range
|
||||
// i.e. /api/v1/labels?match[]=foobar{baz="abc"}&start=...&end=...
|
||||
if len(matches) == 0 {
|
||||
matches = []string{"{__name__!=''}"}
|
||||
if cp.start == 0 {
|
||||
cp.start = cp.end - defaultStep
|
||||
}
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
labels, err = labelsWithMatches(qt, cp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start, err := searchutils.GetTime(r, "start", end-defaultStep)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
labels, err = labelsWithMatches(qt, matches, etfs, start, end, deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %w", matches, start, end, err)
|
||||
return fmt.Errorf("cannot obtain labels for timeRange=[%d..%d]: %w", cp.start, cp.end, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -771,23 +701,12 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
|
||||
return nil
|
||||
}
|
||||
|
||||
func labelsWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) {
|
||||
tagFilterss, err := getTagFilterssFromMatches(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
if len(tagFilterss) == 0 {
|
||||
logger.Panicf("BUG: tagFilterss must be non-empty")
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit)
|
||||
func labelsWithMatches(qt *querytracer.Tracer, cp *commonParams) ([]string, error) {
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
|
||||
m := make(map[string]struct{})
|
||||
if end-start > 24*3600*1000 {
|
||||
if cp.end-cp.start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, deadline)
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
@ -800,7 +719,7 @@ func labelsWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storag
|
||||
m["__name__"] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline)
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -856,9 +775,7 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path="
|
||||
func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error {
|
||||
defer seriesDuration.UpdateDuration(startTime)
|
||||
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
ct := startTime.UnixNano() / 1e6
|
||||
end, err := searchutils.GetTime(r, "end", ct)
|
||||
cp, err := getCommonParams(r, startTime, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -867,25 +784,16 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
|
||||
// which can take a lot of time for big storages.
|
||||
// It is better setting start as end-defaultStep by default.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91
|
||||
start, err := searchutils.GetTime(r, "start", end-defaultStep)
|
||||
if err != nil {
|
||||
return err
|
||||
if cp.start == 0 {
|
||||
cp.start = cp.end - defaultStep
|
||||
}
|
||||
|
||||
tagFilterss, err := getTagFilterssFromRequest(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if start >= end {
|
||||
end = start + defaultStep
|
||||
}
|
||||
sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit)
|
||||
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
|
||||
qtDone := func() {
|
||||
qt.Donef("start=%d, end=%d", start, end)
|
||||
qt.Donef("start=%d, end=%d", cp.start, cp.end)
|
||||
}
|
||||
if end-start > 24*3600*1000 {
|
||||
if cp.end-cp.start > 24*3600*1000 {
|
||||
// It is cheaper to call SearchMetricNames on time ranges exceeding a day.
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, deadline)
|
||||
mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)
|
||||
}
|
||||
@ -909,7 +817,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
|
||||
seriesDuration.UpdateDuration(startTime)
|
||||
return nil
|
||||
}
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline)
|
||||
rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
|
||||
}
|
||||
@ -1000,13 +908,13 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
|
||||
}
|
||||
filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
|
||||
ep := &exportParams{
|
||||
cp := &commonParams{
|
||||
deadline: deadline,
|
||||
start: start,
|
||||
end: end,
|
||||
filterss: filterss,
|
||||
}
|
||||
if err := exportHandler(qt, w, ep, "promapi", 0, false); err != nil {
|
||||
if err := exportHandler(qt, w, cp, "promapi", 0, false); err != nil {
|
||||
return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err)
|
||||
}
|
||||
queryDuration.UpdateDuration(startTime)
|
||||
@ -1268,30 +1176,6 @@ func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error)
|
||||
return tagFilterss, nil
|
||||
}
|
||||
|
||||
func getTagFilterssFromRequest(r *http.Request) ([][]storage.TagFilter, error) {
|
||||
matches := getMatchesFromRequest(r)
|
||||
if len(matches) == 0 {
|
||||
return nil, fmt.Errorf("missing `match[]` query arg")
|
||||
}
|
||||
tagFilterss, err := getTagFilterssFromMatches(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etfs, err := searchutils.GetExtraTagFilters(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
return tagFilterss, nil
|
||||
}
|
||||
|
||||
func getMatchesFromRequest(r *http.Request) []string {
|
||||
matches := r.Form["match[]"]
|
||||
// This is needed for backwards compatibility
|
||||
matches = append(matches, r.Form["match"]...)
|
||||
return matches
|
||||
}
|
||||
|
||||
func getRoundDigits(r *http.Request) int {
|
||||
s := r.FormValue("round_digits")
|
||||
if len(s) == 0 {
|
||||
@ -1342,19 +1226,50 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque
|
||||
|
||||
var queryStatsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/top_queries"}`)
|
||||
|
||||
// exportParams contains common parameters for all /api/v1/export* handlers
|
||||
// commonParams contains common parameters for all /api/v1/* handlers
|
||||
//
|
||||
// deadline, start, end, match[], extra_label, extra_filters
|
||||
type exportParams struct {
|
||||
// timeout, start, end, match[], extra_label, extra_filters[]
|
||||
type commonParams struct {
|
||||
deadline searchutils.Deadline
|
||||
start int64
|
||||
end int64
|
||||
currentTimestamp int64
|
||||
filterss [][]storage.TagFilter
|
||||
}
|
||||
|
||||
// getExportParams obtains common params from r, which are used for /api/v1/export* handlers
|
||||
func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error) {
|
||||
deadline := searchutils.GetDeadlineForExport(r, startTime)
|
||||
func (cp *commonParams) IsDefaultTimeRange() bool {
|
||||
return cp.start == 0 && cp.currentTimestamp-cp.end < 1000
|
||||
}
|
||||
|
||||
// getCommonParams obtains common params from r, which are used in /api/v1/export* handlers
|
||||
//
|
||||
// - timeout
|
||||
// - start
|
||||
// - end
|
||||
// - match[]
|
||||
// - extra_label
|
||||
// - extra_filters[]
|
||||
//
|
||||
func getExportParams(r *http.Request, startTime time.Time) (*commonParams, error) {
|
||||
cp, err := getCommonParams(r, startTime, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cp.deadline = searchutils.GetDeadlineForExport(r, startTime)
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
// getCommonParams obtains common params from r, which are used in /api/v1/* handlers:
|
||||
//
|
||||
// - timeout
|
||||
// - start
|
||||
// - end
|
||||
// - match[]
|
||||
// - extra_label
|
||||
// - extra_filters[]
|
||||
//
|
||||
func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) {
|
||||
deadline := searchutils.GetDeadlineForQuery(r, startTime)
|
||||
start, err := searchutils.GetTime(r, "start", 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -1367,16 +1282,11 @@ func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error
|
||||
if end < start {
|
||||
end = start
|
||||
}
|
||||
|
||||
matches := r.Form["match[]"]
|
||||
if len(matches) == 0 {
|
||||
// Maintain backwards compatibility
|
||||
match := r.FormValue("match")
|
||||
if len(match) == 0 {
|
||||
matches := append([]string{}, r.Form["match[]"]...)
|
||||
matches = append(matches, r.Form["match"]...)
|
||||
if requireNonEmptyMatch && len(matches) == 0 {
|
||||
return nil, fmt.Errorf("missing `match[]` arg")
|
||||
}
|
||||
matches = []string{match}
|
||||
}
|
||||
tagFilterss, err := getTagFilterssFromMatches(matches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -1386,11 +1296,12 @@ func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error
|
||||
return nil, err
|
||||
}
|
||||
filterss := searchutils.JoinTagFilterss(tagFilterss, etfs)
|
||||
|
||||
return &exportParams{
|
||||
cp := &commonParams{
|
||||
deadline: deadline,
|
||||
start: start,
|
||||
end: end,
|
||||
currentTimestamp: ct,
|
||||
filterss: filterss,
|
||||
}, nil
|
||||
}
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user