From 483b402bb21939af5a836a36d4b759c801c85756 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 9 Jun 2022 20:13:04 +0300 Subject: [PATCH] app/vmselect/prometheus: extract common code for obtaining common query args into getCommonParams() function --- app/vmselect/prometheus/prometheus.go | 373 ++++++++++---------------- 1 file changed, 142 insertions(+), 231 deletions(-) diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index f8e1c61e5..9d853a020 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -22,7 +22,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/metrics" @@ -58,8 +57,10 @@ const defaultStep = 5 * 60 * 1000 func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer federateDuration.UpdateDuration(startTime) - ct := startTime.UnixNano() / 1e6 - deadline := searchutils.GetDeadlineForQuery(r, startTime) + cp, err := getCommonParams(r, startTime, true) + if err != nil { + return err + } lookbackDelta, err := getMaxLookback(r) if err != nil { return err @@ -67,23 +68,11 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request if lookbackDelta <= 0 { lookbackDelta = defaultStep } - start, err := searchutils.GetTime(r, "start", ct-lookbackDelta) - if err != nil { - return err + if cp.IsDefaultTimeRange() { + cp.start = cp.end - lookbackDelta } - end, err := searchutils.GetTime(r, "end", ct) - if err != nil { - return err - } - if start >= end { - start = end - defaultStep - } - tagFilterss, err := getTagFilterssFromRequest(r) - if err != nil { - return err - } - sq := storage.NewSearchQuery(start, end, tagFilterss, *maxFederateSeries) - rss, err := netstorage.ProcessSearchQuery(nil, sq, true, deadline) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxFederateSeries) + rss, err := netstorage.ProcessSearchQuery(nil, sq, true, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -116,18 +105,19 @@ var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/fe func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer exportCSVDuration.UpdateDuration(startTime) + cp, err := getExportParams(r, startTime) + if err != nil { + return err + } + format := r.FormValue("format") if len(format) == 0 { return fmt.Errorf("missing `format` arg; see https://docs.victoriametrics.com/#how-to-export-csv-data") } fieldNames := strings.Split(format, ",") reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") - ep, err := getExportParams(r, startTime) - if err != nil { - return err - } - sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries) w.Header().Set("Content-Type", "text/csv; charset=utf-8") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -143,7 +133,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques } doneCh := make(chan error, 1) if !reduceMemUsage { - rss, err := netstorage.ProcessSearchQuery(nil, sq, true, ep.deadline) + rss, err := netstorage.ProcessSearchQuery(nil, sq, true, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -166,7 +156,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques }() } else { go func() { - err := netstorage.ExportBlocks(nil, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { + err := netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { if err := bw.Error(); err != nil { return err } @@ -207,24 +197,24 @@ var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/a func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer exportNativeDuration.UpdateDuration(startTime) - ep, err := getExportParams(r, startTime) + cp, err := getExportParams(r, startTime) if err != nil { return err } - sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries) w.Header().Set("Content-Type", "VictoriaMetrics/native") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) // Marshal tr trBuf := make([]byte, 0, 16) - trBuf = encoding.MarshalInt64(trBuf, ep.start) - trBuf = encoding.MarshalInt64(trBuf, ep.end) + trBuf = encoding.MarshalInt64(trBuf, cp.start) + trBuf = encoding.MarshalInt64(trBuf, cp.end) _, _ = bw.Write(trBuf) // Marshal native blocks. - err = netstorage.ExportBlocks(nil, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { + err = netstorage.ExportBlocks(nil, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { if err := bw.Error(); err != nil { return err } @@ -269,22 +259,22 @@ var bbPool bytesutil.ByteBufferPool func ExportHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer exportDuration.UpdateDuration(startTime) - ep, err := getExportParams(r, startTime) + cp, err := getExportParams(r, startTime) if err != nil { return err } format := r.FormValue("format") maxRowsPerLine := int(fastfloat.ParseInt64BestEffort(r.FormValue("max_rows_per_line"))) reduceMemUsage := searchutils.GetBool(r, "reduce_mem_usage") - if err := exportHandler(nil, w, ep, format, maxRowsPerLine, reduceMemUsage); err != nil { - return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", ep.start, ep.end, err) + if err := exportHandler(nil, w, cp, format, maxRowsPerLine, reduceMemUsage); err != nil { + return fmt.Errorf("error when exporting data on the time range (start=%d, end=%d): %w", cp.start, cp.end, err) } return nil } var exportDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export"}`) -func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportParams, format string, maxRowsPerLine int, reduceMemUsage bool) error { +func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonParams, format string, maxRowsPerLine int, reduceMemUsage bool) error { writeResponseFunc := WriteExportStdResponse writeLineFunc := func(xb *exportBlock, resultsCh chan<- *quicktemplate.ByteBuffer) { bb := quicktemplate.AcquireByteBuffer() @@ -337,7 +327,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara } } - sq := storage.NewSearchQuery(ep.start, ep.end, ep.filterss, *maxExportSeries) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxExportSeries) w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -345,7 +335,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara resultsCh := make(chan *quicktemplate.ByteBuffer, cgroup.AvailableCPUs()) doneCh := make(chan error, 1) if !reduceMemUsage { - rss, err := netstorage.ProcessSearchQuery(qt, sq, true, ep.deadline) + rss, err := netstorage.ProcessSearchQuery(qt, sq, true, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -371,7 +361,7 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, ep *exportPara } else { qtChild := qt.NewChild("background export format=%s", format) go func() { - err := netstorage.ExportBlocks(qtChild, sq, ep.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { + err := netstorage.ExportBlocks(qtChild, sq, cp.deadline, func(mn *storage.MetricName, b *storage.Block, tr storage.TimeRange) error { if err := bw.Error(); err != nil { return err } @@ -430,17 +420,15 @@ var exportBlockPool = &sync.Pool{ func DeleteHandler(startTime time.Time, r *http.Request) error { defer deleteDuration.UpdateDuration(startTime) - deadline := searchutils.GetDeadlineForQuery(r, startTime) - if r.FormValue("start") != "" || r.FormValue("end") != "" { - return fmt.Errorf("start and end aren't supported. Remove these args from the query in order to delete all the matching metrics") - } - tagFilterss, err := getTagFilterssFromRequest(r) + cp, err := getCommonParams(r, startTime, true) if err != nil { return err } - ct := startTime.UnixNano() / 1e6 - sq := storage.NewSearchQuery(0, ct, tagFilterss, 0) - deletedCount, err := netstorage.DeleteSeries(nil, sq, deadline) + if !cp.IsDefaultTimeRange() { + return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end) + } + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, 0) + deletedCount, err := netstorage.DeleteSeries(nil, sq, cp.deadline) if err != nil { return fmt.Errorf("cannot delete time series: %w", err) } @@ -458,35 +446,26 @@ var deleteDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName string, w http.ResponseWriter, r *http.Request) error { defer labelValuesDuration.UpdateDuration(startTime) - deadline := searchutils.GetDeadlineForQuery(r, startTime) - etfs, err := searchutils.GetExtraTagFilters(r) + cp, err := getCommonParams(r, startTime, false) if err != nil { return err } - matches := getMatchesFromRequest(r) var labelValues []string - if len(matches) == 0 && len(etfs) == 0 { - if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 { - var err error - labelValues, err = netstorage.GetLabelValues(qt, labelName, deadline) + if len(cp.filterss) == 0 { + if cp.IsDefaultTimeRange() { + labelValues, err = netstorage.GetLabelValues(qt, labelName, cp.deadline) if err != nil { return fmt.Errorf(`cannot obtain label values for %q: %w`, labelName, err) } } else { - ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) - if err != nil { - return err - } - start, err := searchutils.GetTime(r, "start", end-defaultStep) - if err != nil { - return err + if cp.start == 0 { + cp.start = cp.end - defaultStep } tr := storage.TimeRange{ - MinTimestamp: start, - MaxTimestamp: end, + MinTimestamp: cp.start, + MaxTimestamp: cp.end, } - labelValues, err = netstorage.GetLabelValuesOnTimeRange(qt, labelName, tr, deadline) + labelValues, err = netstorage.GetLabelValuesOnTimeRange(qt, labelName, tr, cp.deadline) if err != nil { return fmt.Errorf(`cannot obtain label values on time range for %q: %w`, labelName, err) } @@ -496,21 +475,12 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s // i.e. /api/v1/label/foo/values?match[]=foobar{baz="abc"}&start=...&end=... // is equivalent to `label_values(foobar{baz="abc"}, foo)` call on the selected // time range in Grafana templating. - if len(matches) == 0 { - matches = []string{fmt.Sprintf("{%s!=''}", labelName)} + if cp.start == 0 { + cp.start = cp.end - defaultStep } - ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + labelValues, err = labelValuesWithMatches(qt, labelName, cp) if err != nil { - return err - } - start, err := searchutils.GetTime(r, "start", end-defaultStep) - if err != nil { - return err - } - labelValues, err = labelValuesWithMatches(qt, labelName, matches, etfs, start, end, deadline) - if err != nil { - return fmt.Errorf("cannot obtain label values for %q, match[]=%q, start=%d, end=%d: %w", labelName, matches, start, end, err) + return fmt.Errorf("cannot obtain label values for %q on time range [%d...%d]: %w", labelName, cp.start, cp.end, err) } } @@ -524,37 +494,24 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s return nil } -func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, matches []string, etfs [][]storage.TagFilter, - start, end int64, deadline searchutils.Deadline) ([]string, error) { - tagFilterss, err := getTagFilterssFromMatches(matches) - if err != nil { - return nil, err - } - +func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, cp *commonParams) ([]string, error) { // Add `labelName!=''` tag filter in order to filter out series without the labelName. // There is no need in adding `__name__!=''` filter, since all the time series should // already have non-empty name. if labelName != "__name__" { key := []byte(labelName) - for i, tfs := range tagFilterss { - tagFilterss[i] = append(tfs, storage.TagFilter{ + for i, tfs := range cp.filterss { + cp.filterss[i] = append(tfs, storage.TagFilter{ Key: key, IsNegative: true, }) } } - if start >= end { - end = start + defaultStep - } - tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs) - if len(tagFilterss) == 0 { - logger.Panicf("BUG: tagFilterss must be non-empty") - } - sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit) m := make(map[string]struct{}) - if end-start > 24*3600*1000 { + if cp.end-cp.start > 24*3600*1000 { // It is cheaper to call SearchMetricNames on time ranges exceeding a day. - mns, err := netstorage.SearchMetricNames(qt, sq, deadline) + mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline) if err != nil { return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err) } @@ -566,7 +523,7 @@ func labelValuesWithMatches(qt *querytracer.Tracer, labelName string, matches [] m[string(labelValue)] = struct{}{} } } else { - rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline) + rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline) if err != nil { return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -627,12 +584,11 @@ const secsPerDay = 3600 * 24 func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer tsdbStatusDuration.UpdateDuration(startTime) - deadline := searchutils.GetDeadlineForStatusRequest(r, startTime) - etfs, err := searchutils.GetExtraTagFilters(r) + cp, err := getCommonParams(r, startTime, false) if err != nil { return err } - matches := getMatchesFromRequest(r) + cp.deadline = searchutils.GetDeadlineForStatusRequest(r, startTime) date := fasttime.UnixDate() dateStr := r.FormValue("date") @@ -659,13 +615,13 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo topN = n } var status *storage.TSDBStatus - if len(matches) == 0 && len(etfs) == 0 { - status, err = netstorage.GetTSDBStatusForDate(qt, deadline, date, topN, *maxTSDBStatusSeries) + if len(cp.filterss) == 0 { + status, err = netstorage.GetTSDBStatusForDate(qt, cp.deadline, date, topN, *maxTSDBStatusSeries) if err != nil { return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err) } } else { - status, err = tsdbStatusWithMatches(qt, matches, etfs, date, topN, *maxTSDBStatusSeries, deadline) + status, err = tsdbStatusWithMatches(qt, cp.filterss, date, topN, *maxTSDBStatusSeries, cp.deadline) if err != nil { return fmt.Errorf("cannot obtain tsdb status with matches for date=%d, topN=%d: %w", date, topN, err) } @@ -680,18 +636,10 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo return nil } -func tsdbStatusWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storage.TagFilter, date uint64, topN, maxMetrics int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) { - tagFilterss, err := getTagFilterssFromMatches(matches) - if err != nil { - return nil, err - } - tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs) - if len(tagFilterss) == 0 { - logger.Panicf("BUG: tagFilterss must be non-empty") - } +func tsdbStatusWithMatches(qt *querytracer.Tracer, filterss [][]storage.TagFilter, date uint64, topN, maxMetrics int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) { start := int64(date*secsPerDay) * 1000 end := int64(date*secsPerDay+secsPerDay) * 1000 - sq := storage.NewSearchQuery(start, end, tagFilterss, maxMetrics) + sq := storage.NewSearchQuery(start, end, filterss, maxMetrics) status, err := netstorage.GetTSDBStatusWithFilters(qt, deadline, sq, topN) if err != nil { return nil, err @@ -707,35 +655,26 @@ var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer labelsDuration.UpdateDuration(startTime) - deadline := searchutils.GetDeadlineForQuery(r, startTime) - etfs, err := searchutils.GetExtraTagFilters(r) + cp, err := getCommonParams(r, startTime, false) if err != nil { return err } - matches := getMatchesFromRequest(r) var labels []string - if len(matches) == 0 && len(etfs) == 0 { - if len(r.Form["start"]) == 0 && len(r.Form["end"]) == 0 { - var err error - labels, err = netstorage.GetLabels(qt, deadline) + if len(cp.filterss) == 0 { + if cp.IsDefaultTimeRange() { + labels, err = netstorage.GetLabels(qt, cp.deadline) if err != nil { return fmt.Errorf("cannot obtain labels: %w", err) } } else { - ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) - if err != nil { - return err - } - start, err := searchutils.GetTime(r, "start", end-defaultStep) - if err != nil { - return err + if cp.start == 0 { + cp.start = cp.end - defaultStep } tr := storage.TimeRange{ - MinTimestamp: start, - MaxTimestamp: end, + MinTimestamp: cp.start, + MaxTimestamp: cp.end, } - labels, err = netstorage.GetLabelsOnTimeRange(qt, tr, deadline) + labels, err = netstorage.GetLabelsOnTimeRange(qt, tr, cp.deadline) if err != nil { return fmt.Errorf("cannot obtain labels on time range: %w", err) } @@ -743,21 +682,12 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW } else { // Extended functionality that allows filtering by label filters and time range // i.e. /api/v1/labels?match[]=foobar{baz="abc"}&start=...&end=... - if len(matches) == 0 { - matches = []string{"{__name__!=''}"} + if cp.start == 0 { + cp.start = cp.end - defaultStep } - ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + labels, err = labelsWithMatches(qt, cp) if err != nil { - return err - } - start, err := searchutils.GetTime(r, "start", end-defaultStep) - if err != nil { - return err - } - labels, err = labelsWithMatches(qt, matches, etfs, start, end, deadline) - if err != nil { - return fmt.Errorf("cannot obtain labels for match[]=%q, start=%d, end=%d: %w", matches, start, end, err) + return fmt.Errorf("cannot obtain labels for timeRange=[%d..%d]: %w", cp.start, cp.end, err) } } @@ -771,23 +701,12 @@ func LabelsHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW return nil } -func labelsWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storage.TagFilter, start, end int64, deadline searchutils.Deadline) ([]string, error) { - tagFilterss, err := getTagFilterssFromMatches(matches) - if err != nil { - return nil, err - } - if start >= end { - end = start + defaultStep - } - tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs) - if len(tagFilterss) == 0 { - logger.Panicf("BUG: tagFilterss must be non-empty") - } - sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit) +func labelsWithMatches(qt *querytracer.Tracer, cp *commonParams) ([]string, error) { + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit) m := make(map[string]struct{}) - if end-start > 24*3600*1000 { + if cp.end-cp.start > 24*3600*1000 { // It is cheaper to call SearchMetricNames on time ranges exceeding a day. - mns, err := netstorage.SearchMetricNames(qt, sq, deadline) + mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline) if err != nil { return nil, fmt.Errorf("cannot fetch time series for %q: %w", sq, err) } @@ -800,7 +719,7 @@ func labelsWithMatches(qt *querytracer.Tracer, matches []string, etfs [][]storag m["__name__"] = struct{}{} } } else { - rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline) + rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline) if err != nil { return nil, fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -856,9 +775,7 @@ var seriesCountDuration = metrics.NewSummary(`vm_request_duration_seconds{path=" func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWriter, r *http.Request) error { defer seriesDuration.UpdateDuration(startTime) - deadline := searchutils.GetDeadlineForQuery(r, startTime) - ct := startTime.UnixNano() / 1e6 - end, err := searchutils.GetTime(r, "end", ct) + cp, err := getCommonParams(r, startTime, true) if err != nil { return err } @@ -867,25 +784,16 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW // which can take a lot of time for big storages. // It is better setting start as end-defaultStep by default. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/91 - start, err := searchutils.GetTime(r, "start", end-defaultStep) - if err != nil { - return err + if cp.start == 0 { + cp.start = cp.end - defaultStep } - - tagFilterss, err := getTagFilterssFromRequest(r) - if err != nil { - return err - } - if start >= end { - end = start + defaultStep - } - sq := storage.NewSearchQuery(start, end, tagFilterss, *maxSeriesLimit) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit) qtDone := func() { - qt.Donef("start=%d, end=%d", start, end) + qt.Donef("start=%d, end=%d", cp.start, cp.end) } - if end-start > 24*3600*1000 { + if cp.end-cp.start > 24*3600*1000 { // It is cheaper to call SearchMetricNames on time ranges exceeding a day. - mns, err := netstorage.SearchMetricNames(qt, sq, deadline) + mns, err := netstorage.SearchMetricNames(qt, sq, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch time series for %q: %w", sq, err) } @@ -909,7 +817,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW seriesDuration.UpdateDuration(startTime) return nil } - rss, err := netstorage.ProcessSearchQuery(qt, sq, false, deadline) + rss, err := netstorage.ProcessSearchQuery(qt, sq, false, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch data for %q: %w", sq, err) } @@ -1000,13 +908,13 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr } filterss := searchutils.JoinTagFilterss(tagFilterss, etfs) - ep := &exportParams{ + cp := &commonParams{ deadline: deadline, start: start, end: end, filterss: filterss, } - if err := exportHandler(qt, w, ep, "promapi", 0, false); err != nil { + if err := exportHandler(qt, w, cp, "promapi", 0, false); err != nil { return fmt.Errorf("error when exporting data for query=%q on the time range (start=%d, end=%d): %w", childQuery, start, end, err) } queryDuration.UpdateDuration(startTime) @@ -1268,30 +1176,6 @@ func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) return tagFilterss, nil } -func getTagFilterssFromRequest(r *http.Request) ([][]storage.TagFilter, error) { - matches := getMatchesFromRequest(r) - if len(matches) == 0 { - return nil, fmt.Errorf("missing `match[]` query arg") - } - tagFilterss, err := getTagFilterssFromMatches(matches) - if err != nil { - return nil, err - } - etfs, err := searchutils.GetExtraTagFilters(r) - if err != nil { - return nil, err - } - tagFilterss = searchutils.JoinTagFilterss(tagFilterss, etfs) - return tagFilterss, nil -} - -func getMatchesFromRequest(r *http.Request) []string { - matches := r.Form["match[]"] - // This is needed for backwards compatibility - matches = append(matches, r.Form["match"]...) - return matches -} - func getRoundDigits(r *http.Request) int { s := r.FormValue("round_digits") if len(s) == 0 { @@ -1342,19 +1226,50 @@ func QueryStatsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reque var queryStatsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/top_queries"}`) -// exportParams contains common parameters for all /api/v1/export* handlers +// commonParams contains common parameters for all /api/v1/* handlers // -// deadline, start, end, match[], extra_label, extra_filters -type exportParams struct { - deadline searchutils.Deadline - start int64 - end int64 - filterss [][]storage.TagFilter +// timeout, start, end, match[], extra_label, extra_filters[] +type commonParams struct { + deadline searchutils.Deadline + start int64 + end int64 + currentTimestamp int64 + filterss [][]storage.TagFilter } -// getExportParams obtains common params from r, which are used for /api/v1/export* handlers -func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error) { - deadline := searchutils.GetDeadlineForExport(r, startTime) +func (cp *commonParams) IsDefaultTimeRange() bool { + return cp.start == 0 && cp.currentTimestamp-cp.end < 1000 +} + +// getCommonParams obtains common params from r, which are used in /api/v1/export* handlers +// +// - timeout +// - start +// - end +// - match[] +// - extra_label +// - extra_filters[] +// +func getExportParams(r *http.Request, startTime time.Time) (*commonParams, error) { + cp, err := getCommonParams(r, startTime, true) + if err != nil { + return nil, err + } + cp.deadline = searchutils.GetDeadlineForExport(r, startTime) + return cp, nil +} + +// getCommonParams obtains common params from r, which are used in /api/v1/* handlers: +// +// - timeout +// - start +// - end +// - match[] +// - extra_label +// - extra_filters[] +// +func getCommonParams(r *http.Request, startTime time.Time, requireNonEmptyMatch bool) (*commonParams, error) { + deadline := searchutils.GetDeadlineForQuery(r, startTime) start, err := searchutils.GetTime(r, "start", 0) if err != nil { return nil, err @@ -1367,15 +1282,10 @@ func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error if end < start { end = start } - - matches := r.Form["match[]"] - if len(matches) == 0 { - // Maintain backwards compatibility - match := r.FormValue("match") - if len(match) == 0 { - return nil, fmt.Errorf("missing `match[]` arg") - } - matches = []string{match} + matches := append([]string{}, r.Form["match[]"]...) + matches = append(matches, r.Form["match"]...) + if requireNonEmptyMatch && len(matches) == 0 { + return nil, fmt.Errorf("missing `match[]` arg") } tagFilterss, err := getTagFilterssFromMatches(matches) if err != nil { @@ -1386,11 +1296,12 @@ func getExportParams(r *http.Request, startTime time.Time) (*exportParams, error return nil, err } filterss := searchutils.JoinTagFilterss(tagFilterss, etfs) - - return &exportParams{ - deadline: deadline, - start: start, - end: end, - filterss: filterss, - }, nil + cp := &commonParams{ + deadline: deadline, + start: start, + end: end, + currentTimestamp: ct, + filterss: filterss, + } + return cp, nil }