From ec7963208d41823270be49af44873480e1a53e3f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 14 Jun 2022 17:46:16 +0300 Subject: [PATCH] app/vmselect: accept `focusLabel` query arg at /api/v1/status/tsdb This allows filling the seriesCountByFocusLabelValue list in the /api/v1/status/tsdb response with label values for the specified focusLabel, which contain the highest number of time series. TODO: add this to Cardinality explorer at VMUI - https://docs.victoriametrics.com/#cardinality-explorer --- README.md | 4 +- app/vmselect/netstorage/netstorage.go | 24 +---- app/vmselect/prometheus/prometheus.go | 42 ++++---- .../prometheus/tsdb_status_response.qtpl | 1 + .../prometheus/tsdb_status_response.qtpl.go | 96 ++++++++++--------- app/vmstorage/main.go | 14 +-- docs/Single-server-VictoriaMetrics.md | 4 +- lib/storage/index_db.go | 46 +++++---- lib/storage/index_db_test.go | 64 ++++++++++--- lib/storage/storage.go | 6 +- 10 files changed, 161 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index 007536bc6..08343ae1c 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,8 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways: - To identify metric names with the highest number of series. -- To idnetify labels with the highest number of series. +- To identify labels with the highest number of series. +- To identify values with the highest number of series for the selected label (aka `focusLabel`). - To identify label=name pairs with the highest number of series. - To identify labels with the highest number of unique values. @@ -1441,6 +1442,7 @@ VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way simi * `topN=N` where `N` is the number of top entries to return in the response. By default top 10 entries are returned. * `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day. Pass `date=1970-01-01` in order to collect global stats across all the days. +* `focusLabel=LABEL_NAME` returns label values with the highest number of time series for the given `LABEL_NAME` in the `seriesCountByFocusLabelValue` list. * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account. * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details. diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index 3c21025d9..88ae27f67 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -764,25 +764,11 @@ func GetTagValueSuffixes(qt *querytracer.Tracer, tr storage.TimeRange, tagKey, t return suffixes, nil } -// GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats -func GetTSDBStatusForDate(qt *querytracer.Tracer, deadline searchutils.Deadline, date uint64, topN, maxMetrics int) (*storage.TSDBStatus, error) { - qt = qt.NewChild("get tsdb stats for date=%d, topN=%d", date, topN) - defer qt.Done() - if deadline.Exceeded() { - return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) - } - status, err := vmstorage.GetTSDBStatusForDate(qt, date, topN, maxMetrics, deadline.Deadline()) - if err != nil { - return nil, fmt.Errorf("error during tsdb status request: %w", err) - } - return status, nil -} - -// GetTSDBStatusWithFilters returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats +// GetTSDBStatus returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats // // It accepts aribtrary filters on time series in sq. -func GetTSDBStatusWithFilters(qt *querytracer.Tracer, deadline searchutils.Deadline, sq *storage.SearchQuery, topN int) (*storage.TSDBStatus, error) { - qt = qt.NewChild("get tsdb stats: %s, topN=%d", sq, topN) +func GetTSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) { + qt = qt.NewChild("get tsdb stats: %s, focusLabel=%q, topN=%d", sq, focusLabel, topN) defer qt.Done() if deadline.Exceeded() { return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) @@ -796,9 +782,9 @@ func GetTSDBStatusWithFilters(qt *querytracer.Tracer, deadline searchutils.Deadl return nil, err } date := uint64(tr.MinTimestamp) / (3600 * 24 * 1000) - status, err := vmstorage.GetTSDBStatusWithFiltersForDate(qt, tfss, date, topN, sq.MaxMetrics, deadline.Deadline()) + status, err := vmstorage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, sq.MaxMetrics, deadline.Deadline()) if err != nil { - return nil, fmt.Errorf("error during tsdb status with filters request: %w", err) + return nil, fmt.Errorf("error during tsdb status request: %w", err) } return status, nil } diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 5c945fbb2..e4f990b63 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -490,12 +490,17 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo date := fasttime.UnixDate() dateStr := r.FormValue("date") if len(dateStr) > 0 { - t, err := time.Parse("2006-01-02", dateStr) - if err != nil { - return fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err) + if dateStr == "0" { + date = 0 + } else { + t, err := time.Parse("2006-01-02", dateStr) + if err != nil { + return fmt.Errorf("cannot parse `date` arg %q: %w", dateStr, err) + } + date = uint64(t.Unix()) / secsPerDay } - date = uint64(t.Unix()) / secsPerDay } + focusLabel := r.FormValue("focusLabel") topN := 10 topNStr := r.FormValue("topN") if len(topNStr) > 0 { @@ -511,18 +516,14 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo } topN = n } - var status *storage.TSDBStatus - if len(cp.filterss) == 0 { - status, err = netstorage.GetTSDBStatusForDate(qt, cp.deadline, date, topN, *maxTSDBStatusSeries) - if err != nil { - return fmt.Errorf(`cannot obtain tsdb status for date=%d, topN=%d: %w`, date, topN, err) - } - } else { - status, err = tsdbStatusWithMatches(qt, cp.filterss, date, topN, *maxTSDBStatusSeries, cp.deadline) - if err != nil { - return fmt.Errorf("cannot obtain tsdb status with matches for date=%d, topN=%d: %w", date, topN, err) - } + start := int64(date*secsPerDay) * 1000 + end := int64((date+1)*secsPerDay)*1000 - 1 + sq := storage.NewSearchQuery(start, end, cp.filterss, *maxTSDBStatusSeries) + status, err := netstorage.GetTSDBStatus(qt, sq, focusLabel, topN, cp.deadline) + if err != nil { + return fmt.Errorf("cannot obtain tsdb stats: %w", err) } + w.Header().Set("Content-Type", "application/json") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -533,17 +534,6 @@ func TSDBStatusHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo return nil } -func tsdbStatusWithMatches(qt *querytracer.Tracer, filterss [][]storage.TagFilter, date uint64, topN, maxMetrics int, deadline searchutils.Deadline) (*storage.TSDBStatus, error) { - start := int64(date*secsPerDay) * 1000 - end := int64(date*secsPerDay+secsPerDay) * 1000 - sq := storage.NewSearchQuery(start, end, filterss, maxMetrics) - status, err := netstorage.GetTSDBStatusWithFilters(qt, deadline, sq, topN) - if err != nil { - return nil, err - } - return status, nil -} - var tsdbStatusDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/status/tsdb"}`) // LabelsHandler processes /api/v1/labels request. diff --git a/app/vmselect/prometheus/tsdb_status_response.qtpl b/app/vmselect/prometheus/tsdb_status_response.qtpl index 08cdb2c5d..ef327e45d 100644 --- a/app/vmselect/prometheus/tsdb_status_response.qtpl +++ b/app/vmselect/prometheus/tsdb_status_response.qtpl @@ -13,6 +13,7 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb . "totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %}, "seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %}, "seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %}, + "seriesCountByFocusLabelValue":{%= tsdbStatusEntries(status.SeriesCountByFocusLabelValue) %}, "seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %}, "labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %} } diff --git a/app/vmselect/prometheus/tsdb_status_response.qtpl.go b/app/vmselect/prometheus/tsdb_status_response.qtpl.go index 523a74223..b138b8e02 100644 --- a/app/vmselect/prometheus/tsdb_status_response.qtpl.go +++ b/app/vmselect/prometheus/tsdb_status_response.qtpl.go @@ -44,102 +44,106 @@ func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBSta //line app/vmselect/prometheus/tsdb_status_response.qtpl:15 streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName) //line app/vmselect/prometheus/tsdb_status_response.qtpl:15 + qw422016.N().S(`,"seriesCountByFocusLabelValue":`) +//line app/vmselect/prometheus/tsdb_status_response.qtpl:16 + streamtsdbStatusEntries(qw422016, status.SeriesCountByFocusLabelValue) +//line app/vmselect/prometheus/tsdb_status_response.qtpl:16 qw422016.N().S(`,"seriesCountByLabelValuePair":`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:16 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:17 streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:16 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:17 qw422016.N().S(`,"labelValueCountByLabelName":`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:17 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:18 streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:17 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:18 qw422016.N().S(`}`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:19 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:20 qt.Done() -//line app/vmselect/prometheus/tsdb_status_response.qtpl:20 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:21 streamdumpQueryTrace(qw422016, qt) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:20 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:21 qw422016.N().S(`}`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 StreamTSDBStatusResponse(qw422016, status, qt) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) string { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 WriteTSDBStatusResponse(qb422016, status, qt) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 return qs422016 -//line app/vmselect/prometheus/tsdb_status_response.qtpl:22 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:23 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:24 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:25 func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:24 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:25 qw422016.N().S(`[`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:26 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:27 for i, e := range a { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:26 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:27 qw422016.N().S(`{"name":`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:28 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:29 qw422016.N().Q(e.Name) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:28 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:29 qw422016.N().S(`,"value":`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:29 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:30 qw422016.N().D(int(e.Count)) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:29 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:30 qw422016.N().S(`}`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:31 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:32 if i+1 < len(a) { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:31 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:32 qw422016.N().S(`,`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:31 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:32 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:32 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:33 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:32 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:33 qw422016.N().S(`]`) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 streamtsdbStatusEntries(qw422016, a) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 } -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 func tsdbStatusEntries(a []storage.TopHeapEntry) string { -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 writetsdbStatusEntries(qb422016, a) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 return qs422016 -//line app/vmselect/prometheus/tsdb_status_response.qtpl:34 +//line app/vmselect/prometheus/tsdb_status_response.qtpl:35 } diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index d973d048b..b59713629 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -215,18 +215,10 @@ func SearchGraphitePaths(tr storage.TimeRange, query []byte, maxPaths int, deadl return paths, err } -// GetTSDBStatusForDate returns TSDB status for the given date. -func GetTSDBStatusForDate(qt *querytracer.Tracer, date uint64, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) { +// GetTSDBStatus returns TSDB status for given filters on the given date. +func GetTSDBStatus(qt *querytracer.Tracer, tfss []*storage.TagFilters, date uint64, focusLabel string, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) { WG.Add(1) - status, err := Storage.GetTSDBStatusWithFiltersForDate(qt, nil, date, topN, maxMetrics, deadline) - WG.Done() - return status, err -} - -// GetTSDBStatusWithFiltersForDate returns TSDB status for given filters on the given date. -func GetTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss []*storage.TagFilters, date uint64, topN, maxMetrics int, deadline uint64) (*storage.TSDBStatus, error) { - WG.Add(1) - status, err := Storage.GetTSDBStatusWithFiltersForDate(qt, tfss, date, topN, maxMetrics, deadline) + status, err := Storage.GetTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics, deadline) WG.Done() return status, err } diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 9977f1076..85ab20b22 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -272,7 +272,8 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways: - To identify metric names with the highest number of series. -- To idnetify labels with the highest number of series. +- To identify labels with the highest number of series. +- To identify values with the highest number of series for the selected label (aka `focusLabel`). - To identify label=name pairs with the highest number of series. - To identify labels with the highest number of unique values. @@ -1445,6 +1446,7 @@ VictoriaMetrics returns TSDB stats at `/api/v1/status/tsdb` page in the way simi * `topN=N` where `N` is the number of top entries to return in the response. By default top 10 entries are returned. * `date=YYYY-MM-DD` where `YYYY-MM-DD` is the date for collecting the stats. By default the stats is collected for the current day. Pass `date=1970-01-01` in order to collect global stats across all the days. +* `focusLabel=LABEL_NAME` returns label values with the highest number of time series for the given `LABEL_NAME` in the `seriesCountByFocusLabelValue` list. * `match[]=SELECTOR` where `SELECTOR` is an arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors) for series to take into account during stats calculation. By default all the series are taken into account. * `extra_label=LABEL=VALUE`. See [these docs](#prometheus-querying-api-enhancements) for more details. diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index 3e5b32b7d..079d9fdf2 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -1239,11 +1239,11 @@ func (is *indexSearch) getSeriesCount() (uint64, error) { return metricIDsLen, nil } -// GetTSDBStatusWithFiltersForDate returns topN entries for tsdb status for the given tfss and the given date. -func (db *indexDB) GetTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, topN, maxMetrics int, deadline uint64) (*TSDBStatus, error) { +// GetTSDBStatus returns topN entries for tsdb status for the given tfss, date and focusLabel. +func (db *indexDB) GetTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, focusLabel string, topN, maxMetrics int, deadline uint64) (*TSDBStatus, error) { qtChild := qt.NewChild("collect tsdb stats in the current indexdb") is := db.getIndexSearch(deadline) - status, err := is.getTSDBStatusWithFiltersForDate(qtChild, tfss, date, topN, maxMetrics) + status, err := is.getTSDBStatus(qtChild, tfss, date, focusLabel, topN, maxMetrics) qtChild.Done() db.putIndexSearch(is) if err != nil { @@ -1255,7 +1255,7 @@ func (db *indexDB) GetTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss ok := db.doExtDB(func(extDB *indexDB) { qtChild := qt.NewChild("collect tsdb stats in the previous indexdb") is := extDB.getIndexSearch(deadline) - status, err = is.getTSDBStatusWithFiltersForDate(qtChild, tfss, date, topN, maxMetrics) + status, err = is.getTSDBStatus(qtChild, tfss, date, focusLabel, topN, maxMetrics) qtChild.Done() extDB.putIndexSearch(is) }) @@ -1265,8 +1265,8 @@ func (db *indexDB) GetTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss return status, nil } -// getTSDBStatusWithFiltersForDate returns topN entries for tsdb status for the given tfss and the given date. -func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, topN, maxMetrics int) (*TSDBStatus, error) { +// getTSDBStatus returns topN entries for tsdb status for the given tfss, date and focusLabel. +func (is *indexSearch) getTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, focusLabel string, topN, maxMetrics int) (*TSDBStatus, error) { filter, err := is.searchMetricIDsWithFiltersOnDate(qt, tfss, date, maxMetrics) if err != nil { return nil, err @@ -1281,12 +1281,14 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t dmis := is.db.s.getDeletedMetricIDs() thSeriesCountByMetricName := newTopHeap(topN) thSeriesCountByLabelName := newTopHeap(topN) + thSeriesCountByFocusLabelValue := newTopHeap(topN) thSeriesCountByLabelValuePair := newTopHeap(topN) thLabelValueCountByLabelName := newTopHeap(topN) var tmp, prevLabelName, prevLabelValuePair []byte var labelValueCountByLabelName, seriesCountByLabelValuePair uint64 var totalSeries, labelSeries, totalLabelValuePairs uint64 nameEqualBytes := []byte("__name__=") + focusLabelEqualBytes := []byte(focusLabel + "=") loopsPaceLimiter := 0 nsPrefixExpected := byte(nsPrefixDateTagToMetricIDs) @@ -1358,6 +1360,9 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) { thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair) } + if bytes.HasPrefix(prevLabelValuePair, focusLabelEqualBytes) { + thSeriesCountByFocusLabelValue.push(prevLabelValuePair[len(focusLabelEqualBytes):], seriesCountByLabelValuePair) + } seriesCountByLabelValuePair = 0 labelValueCountByLabelName++ prevLabelValuePair = append(prevLabelValuePair[:0], labelValuePair...) @@ -1377,13 +1382,17 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) { thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair) } + if bytes.HasPrefix(prevLabelValuePair, focusLabelEqualBytes) { + thSeriesCountByFocusLabelValue.push(prevLabelValuePair[len(focusLabelEqualBytes):], seriesCountByLabelValuePair) + } status := &TSDBStatus{ - TotalSeries: totalSeries, - TotalLabelValuePairs: totalLabelValuePairs, - SeriesCountByMetricName: thSeriesCountByMetricName.getSortedResult(), - SeriesCountByLabelName: thSeriesCountByLabelName.getSortedResult(), - SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(), - LabelValueCountByLabelName: thLabelValueCountByLabelName.getSortedResult(), + TotalSeries: totalSeries, + TotalLabelValuePairs: totalLabelValuePairs, + SeriesCountByMetricName: thSeriesCountByMetricName.getSortedResult(), + SeriesCountByLabelName: thSeriesCountByLabelName.getSortedResult(), + SeriesCountByFocusLabelValue: thSeriesCountByFocusLabelValue.getSortedResult(), + SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(), + LabelValueCountByLabelName: thLabelValueCountByLabelName.getSortedResult(), } return status, nil } @@ -1392,12 +1401,13 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t // // See https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats type TSDBStatus struct { - TotalSeries uint64 - TotalLabelValuePairs uint64 - SeriesCountByMetricName []TopHeapEntry - SeriesCountByLabelName []TopHeapEntry - SeriesCountByLabelValuePair []TopHeapEntry - LabelValueCountByLabelName []TopHeapEntry + TotalSeries uint64 + TotalLabelValuePairs uint64 + SeriesCountByMetricName []TopHeapEntry + SeriesCountByLabelName []TopHeapEntry + SeriesCountByFocusLabelValue []TopHeapEntry + SeriesCountByLabelValuePair []TopHeapEntry + LabelValueCountByLabelName []TopHeapEntry } func (status *TSDBStatus) hasEntries() bool { diff --git a/lib/storage/index_db_test.go b/lib/storage/index_db_test.go index e177718d5..7a852a3c6 100644 --- a/lib/storage/index_db_test.go +++ b/lib/storage/index_db_test.go @@ -1807,10 +1807,10 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { t.Fatalf("expected %d time series for all days, got %d time series", metricsPerDay*days, len(matchedTSIDs)) } - // Check GetTSDBStatusWithFiltersForDate with nil filters. - status, err := db.GetTSDBStatusWithFiltersForDate(nil, nil, baseDate, 5, 1e6, noDeadline) + // Check GetTSDBStatus with nil filters. + status, err := db.GetTSDBStatus(nil, nil, baseDate, "day", 5, 1e6, noDeadline) if err != nil { - t.Fatalf("error in GetTSDBStatusWithFiltersForDate with nil filters: %s", err) + t.Fatalf("error in GetTSDBStatus with nil filters: %s", err) } if !status.hasEntries() { t.Fatalf("expecting non-empty TSDB status") @@ -1845,6 +1845,15 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { if !reflect.DeepEqual(status.SeriesCountByLabelName, expectedSeriesCountByLabelName) { t.Fatalf("unexpected SeriesCountByLabelName;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelName, expectedSeriesCountByLabelName) } + expectedSeriesCountByFocusLabelValue := []TopHeapEntry{ + { + Name: "0", + Count: 1000, + }, + } + if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) { + t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) + } expectedLabelValueCountByLabelName := []TopHeapEntry{ { Name: "uniqueid", @@ -1900,14 +1909,14 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs) } - // Check GetTSDBStatusWithFiltersForDate with non-nil filter, which matches all the series + // Check GetTSDBStatus with non-nil filter, which matches all the series tfs = NewTagFilters() if err := tfs.Add([]byte("day"), []byte("0"), false, false); err != nil { t.Fatalf("cannot add filter: %s", err) } - status, err = db.GetTSDBStatusWithFiltersForDate(nil, []*TagFilters{tfs}, baseDate, 5, 1e6, noDeadline) + status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline) if err != nil { - t.Fatalf("error in GetTSDBStatusWithFiltersForDate: %s", err) + t.Fatalf("error in GetTSDBStatus: %s", err) } if !status.hasEntries() { t.Fatalf("expecting non-empty TSDB status") @@ -1930,10 +1939,10 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs) } - // Check GetTSDBStatusWithFiltersOnDate, which matches all the series on a global time range - status, err = db.GetTSDBStatusWithFiltersForDate(nil, nil, 0, 5, 1e6, noDeadline) + // Check GetTSDBStatus, which matches all the series on a global time range + status, err = db.GetTSDBStatus(nil, nil, 0, "day", 5, 1e6, noDeadline) if err != nil { - t.Fatalf("error in GetTSDBStatusWithFiltersForDate: %s", err) + t.Fatalf("error in GetTSDBStatus: %s", err) } if !status.hasEntries() { t.Fatalf("expecting non-empty TSDB status") @@ -1955,15 +1964,40 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { if status.TotalLabelValuePairs != expectedLabelValuePairs { t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs) } + expectedSeriesCountByFocusLabelValue = []TopHeapEntry{ + { + Name: "0", + Count: 1000, + }, + { + Name: "1", + Count: 1000, + }, + { + Name: "2", + Count: 1000, + }, + { + Name: "3", + Count: 1000, + }, + { + Name: "4", + Count: 1000, + }, + } + if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) { + t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) + } - // Check GetTSDBStatusWithFiltersForDate with non-nil filter, which matches only 3 series + // Check GetTSDBStatus with non-nil filter, which matches only 3 series tfs = NewTagFilters() if err := tfs.Add([]byte("uniqueid"), []byte("0|1|3"), false, true); err != nil { t.Fatalf("cannot add filter: %s", err) } - status, err = db.GetTSDBStatusWithFiltersForDate(nil, []*TagFilters{tfs}, baseDate, 5, 1e6, noDeadline) + status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline) if err != nil { - t.Fatalf("error in GetTSDBStatusWithFiltersForDate: %s", err) + t.Fatalf("error in GetTSDBStatus: %s", err) } if !status.hasEntries() { t.Fatalf("expecting non-empty TSDB status") @@ -1986,10 +2020,10 @@ func TestSearchTSIDWithTimeRange(t *testing.T) { t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs) } - // Check GetTSDBStatusWithFiltersForDate with non-nil filter on global time range, which matches only 15 series - status, err = db.GetTSDBStatusWithFiltersForDate(nil, []*TagFilters{tfs}, 0, 5, 1e6, noDeadline) + // Check GetTSDBStatus with non-nil filter on global time range, which matches only 15 series + status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, 0, "", 5, 1e6, noDeadline) if err != nil { - t.Fatalf("error in GetTSDBStatusWithFiltersForDate: %s", err) + t.Fatalf("error in GetTSDBStatus: %s", err) } if !status.hasEntries() { t.Fatalf("expecting non-empty TSDB status") diff --git a/lib/storage/storage.go b/lib/storage/storage.go index 11d99b56a..22ea03790 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1467,9 +1467,9 @@ func (s *Storage) GetSeriesCount(deadline uint64) (uint64, error) { return s.idb().GetSeriesCount(deadline) } -// GetTSDBStatusWithFiltersForDate returns TSDB status data for /api/v1/status/tsdb with match[] filters. -func (s *Storage) GetTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, topN, maxMetrics int, deadline uint64) (*TSDBStatus, error) { - return s.idb().GetTSDBStatusWithFiltersForDate(qt, tfss, date, topN, maxMetrics, deadline) +// GetTSDBStatus returns TSDB status data for /api/v1/status/tsdb +func (s *Storage) GetTSDBStatus(qt *querytracer.Tracer, tfss []*TagFilters, date uint64, focusLabel string, topN, maxMetrics int, deadline uint64) (*TSDBStatus, error) { + return s.idb().GetTSDBStatus(qt, tfss, date, focusLabel, topN, maxMetrics, deadline) } // MetricRow is a metric to insert into storage.