lib/storage: show top labels with the highest number of series in cardinality explorer

README.md

@@ -268,6 +268,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
 
 - To identify metric names with the highest number of series.
+- To idnetify labels with the highest number of series.
 - To identify label=name pairs with the highest number of series.
 - To identify labels with the highest number of unique values.
 
@@ -275,8 +276,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
 By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
 matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
 
-Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
-
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -12,6 +12,7 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 		"totalSeries": {%dul= status.TotalSeries %},
 		"totalLabelValuePairs": {%dul= status.TotalLabelValuePairs %},
 		"seriesCountByMetricName":{%= tsdbStatusEntries(status.SeriesCountByMetricName) %},
+		"seriesCountByLabelName":{%= tsdbStatusEntries(status.SeriesCountByLabelName) %},
 		"seriesCountByLabelValuePair":{%= tsdbStatusEntries(status.SeriesCountByLabelValuePair) %},
 		"labelValueCountByLabelName":{%= tsdbStatusEntries(status.LabelValueCountByLabelName) %}
 	}

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -40,102 +40,106 @@ func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBSta
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:14
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByMetricName)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+	qw422016.N().S(`,"seriesCountByLabelName":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	qw422016.N().S(`,"seriesCountByLabelValuePair":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
 	qw422016.N().S(`,"labelValueCountByLabelName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
 	streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
 	qt.Done()
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	StreamTSDBStatusResponse(qw422016, status, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	WriteTSDBStatusResponse(qb422016, status, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
 	for i, e := range a {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
 		qw422016.N().S(`{"name":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
 		qw422016.N().Q(e.Name)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
 		qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
 		qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
 		if i+1 < len(a) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
 		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
 	}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
 	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	streamtsdbStatusEntries(qw422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 func tsdbStatusEntries(a []storage.TopHeapEntry) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	writetsdbStatusEntries(qb422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 }

app/vmselect/vmui/asset-manifest.json

@@ -1,12 +1,12 @@
 {
   "files": {
     "main.css": "./static/css/main.7e6d0c89.css",
-    "main.js": "./static/js/main.a6bca65f.js",
+    "main.js": "./static/js/main.f7185a13.js",
     "static/js/27.939f971b.chunk.js": "./static/js/27.939f971b.chunk.js",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.7e6d0c89.css",
-    "static/js/main.a6bca65f.js"
+    "static/js/main.f7185a13.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.a6bca65f.js"></script><link href="./static/css/main.7e6d0c89.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.f7185a13.js"></script><link href="./static/css/main.7e6d0c89.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmui/packages/vmui/src/components/CardinalityPanel/CardinalityConfigurator/CardinalityConfigurator.tsx

@@ -54,12 +54,12 @@ const CardinalityConfigurator: FC<CardinalityConfiguratorProps> = ({
         <QueryEditor
           query={query} index={0} autocomplete={autocomplete} queryOptions={queryOptions}
           error={error} setHistoryIndex={onSetHistory} runQuery={onRunQuery} setQuery={onSetQuery}
-          label={"Arbitrary time series selector"}
+          label={"Time series selector"}
         />
         <Box display="flex" alignItems="center">
           <Box ml={2}>
             <TextField
-              label="Number of top entries"
+              label="Number of entries per table"
               type="number"
               size="small"
               variant="outlined"
@@ -82,7 +82,7 @@ const CardinalityConfigurator: FC<CardinalityConfiguratorProps> = ({
       </Box>
     </Box>
     <Box>
-      Analyzed <b>{totalSeries}</b> series and <b>{totalLabelValuePairs}</b> label=value pairs
+      Analyzed <b>{totalSeries}</b> series with <b>{totalLabelValuePairs}</b> label=value pairs
       at <b>{date}</b> {match && <span>for series selector <b>{match}</b></span>}. Show top {topN} entries per table.
     </Box>
   </Box>;

app/vmui/packages/vmui/src/components/CardinalityPanel/CardinalityPanel.tsx

@@ -3,12 +3,10 @@ import {SyntheticEvent} from "react";
 import {Alert} from "@mui/material";
 import {useFetchQuery} from "../../hooks/useCardinalityFetch";
 import {
-  LABEL_VALUE_PAIR_CONTENT_TITLE,
+  METRIC_NAMES_HEADERS,
-  LABEL_VALUE_PAIRS_TABLE_HEADERS,
+  LABEL_NAMES_HEADERS,
-  LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS,
+  LABEL_VALUE_PAIRS_HEADERS,
-  LABELS_CONTENT_TITLE, METRICS_TABLE_HEADERS,
+  LABELS_WITH_UNIQUE_VALUES_HEADERS,
-  SERIES_CONTENT_TITLE,
-  SPINNER_TITLE,
   spinnerContainerStyles
 } from "./consts";
 import {defaultProperties, queryUpdater} from "./helpers";
@@ -76,7 +74,7 @@ const CardinalityPanel: FC = () => {
         height={"800px"}
         containerStyles={spinnerContainerStyles("100%")}
         title={<Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>
-          {SPINNER_TITLE}
+          Please wait while cardinality stats is calculated. This may take some time if the db contains big number of time series
         </Alert>}
       />}
       <CardinalityConfigurator error={configError} query={query} onRunQuery={onRunQuery} onSetQuery={onSetQuery}
@@ -84,6 +82,7 @@ const CardinalityPanel: FC = () => {
         totalSeries={tsdbStatus.totalSeries} totalLabelValuePairs={tsdbStatus.totalLabelValuePairs}/>
       {error && <Alert color="error" severity="error" sx={{whiteSpace: "pre-wrap", mt: 2}}>{error}</Alert>}
       <MetricsContent
+        sectionTitle={"Metric names with the highest number of series"}
         activeTab={stateTabs.seriesCountByMetricName}
         rows={tsdbStatus.seriesCountByMetricName as unknown as Data[]}
         onChange={handleTabChange}
@@ -92,10 +91,22 @@ const CardinalityPanel: FC = () => {
         chartContainer={defaultProps.containerRefs.seriesCountByMetricName}
         totalSeries={tsdbStatus.totalSeries}
         tabId={"seriesCountByMetricName"}
-        sectionTitle={SERIES_CONTENT_TITLE}
+        tableHeaderCells={METRIC_NAMES_HEADERS}
-        tableHeaderCells={METRICS_TABLE_HEADERS}
       />
       <MetricsContent
+        sectionTitle={"Labels with the highest number of series"}
+        activeTab={stateTabs.seriesCountByLabelName}
+        rows={tsdbStatus.seriesCountByLabelName as unknown as Data[]}
+        onChange={handleTabChange}
+        onActionClick={handleFilterClick("seriesCountByLabelName")}
+        tabs={defaultProps.tabs.seriesCountByLabelName}
+        chartContainer={defaultProps.containerRefs.seriesCountByLabelName}
+        totalSeries={tsdbStatus.totalSeries}
+        tabId={"seriesCountByLabelName"}
+        tableHeaderCells={LABEL_NAMES_HEADERS}
+      />
+      <MetricsContent
+        sectionTitle={"Label=value pairs with the highest number of series"}
         activeTab={stateTabs.seriesCountByLabelValuePair}
         rows={tsdbStatus.seriesCountByLabelValuePair as unknown as Data[]}
         onChange={handleTabChange}
@@ -104,10 +115,10 @@ const CardinalityPanel: FC = () => {
         chartContainer={defaultProps.containerRefs.seriesCountByLabelValuePair}
         totalSeries={tsdbStatus.totalSeries}
         tabId={"seriesCountByLabelValuePair"}
-        sectionTitle={LABEL_VALUE_PAIR_CONTENT_TITLE}
+        tableHeaderCells={LABEL_VALUE_PAIRS_HEADERS}
-        tableHeaderCells={LABEL_VALUE_PAIRS_TABLE_HEADERS}
       />
       <MetricsContent
+        sectionTitle={"Labels with the highest number of unique values"}
         activeTab={stateTabs.labelValueCountByLabelName}
         rows={tsdbStatus.labelValueCountByLabelName as unknown as Data[]}
         onChange={handleTabChange}
@@ -116,8 +127,7 @@ const CardinalityPanel: FC = () => {
         chartContainer={defaultProps.containerRefs.labelValueCountByLabelName}
         totalSeries={-1}
         tabId={"labelValueCountByLabelName"}
-        sectionTitle={LABELS_CONTENT_TITLE}
+        tableHeaderCells={LABELS_WITH_UNIQUE_VALUES_HEADERS}
-        tableHeaderCells={LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS}
       />
     </>
   );

app/vmui/packages/vmui/src/components/CardinalityPanel/consts.ts

@@ -1,10 +1,64 @@
 import {HeadCell} from "../Table/types";
 
-export const METRICS_TABLE_HEADERS = [
+export const METRIC_NAMES_HEADERS = [
   {
     disablePadding: false,
     id: "name",
-    label: "Metrics name",
+    label: "Metric name",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "value",
+    label: "Number of series",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "percentage",
+    label: "Percent of series",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "action",
+    label: "Action",
+    numeric: false,
+  }
+] as HeadCell[];
+
+export const LABEL_NAMES_HEADERS = [
+  {
+    disablePadding: false,
+    id: "name",
+    label: "Label name",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "value",
+    label: "Number of series",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "percentage",
+    label: "Percent of series",
+    numeric: false,
+  },
+  {
+    disablePadding: false,
+    id: "action",
+    label: "Action",
+    numeric: false,
+  }
+] as HeadCell[];
+
+export const LABEL_VALUE_PAIRS_HEADERS = [
+  {
+    disablePadding: false,
+    id: "name",
+    label: "Label=value pair",
     numeric: false,
   },
   {
@@ -27,34 +81,7 @@ export const METRICS_TABLE_HEADERS = [
   }
 ]as HeadCell[];
 
-export const LABEL_VALUE_PAIRS_TABLE_HEADERS = [
+export const LABELS_WITH_UNIQUE_VALUES_HEADERS = [
-  {
-    disablePadding: false,
-    id: "name",
-    label: "Lable=value pair",
-    numeric: false,
-  },
-  {
-    disablePadding: false,
-    id: "value",
-    label: "Number of series",
-    numeric: false,
-  },
-  {
-    disablePadding: false,
-    id: "percentage",
-    label: "Percent of total label value pairs",
-    numeric: false,
-  },
-  {
-    disablePadding: false,
-    id: "action",
-    label: "Action",
-    numeric: false,
-  }
-]as HeadCell[];
-
-export const LABEL_WITH_UNIQUE_VALUES_TABLE_HEADERS = [
   {
     disablePadding: false,
     id: "name",
@@ -86,9 +113,3 @@ export const spinnerContainerStyles = (height: string) =>  {
     zIndex: 1000,
   };
 };
-
-export const SPINNER_TITLE = "Please wait while cardinality stats is calculated. " +
-  "This may take some time if the db contains big number of time series";
-export const SERIES_CONTENT_TITLE = "Metric names with the highest number of series";
-export const LABEL_VALUE_PAIR_CONTENT_TITLE = "Label=value pairs with the highest number of series";
-export const LABELS_CONTENT_TITLE = "Labels with the highest number of unique values";

app/vmui/packages/vmui/src/components/CardinalityPanel/helpers.ts

@@ -2,16 +2,17 @@ import {Containers, DefaultState, QueryUpdater, Tabs, TSDBStatus} from "./types"
 import {useRef} from "preact/compat";
 
 export const queryUpdater: QueryUpdater = {
-  labelValueCountByLabelName: (query: string): string => `{${query}!=""}`,
+  seriesCountByMetricName: (query: string): string => {
+    return getSeriesSelector("__name__", query);
+  },
+  seriesCountByLabelName: (query: string): string => `{${query}!=""}`,
   seriesCountByLabelValuePair: (query: string): string => {
     const a = query.split("=");
     const label = a[0];
     const value = a.slice(1).join("=");
     return getSeriesSelector(label, value);
   },
-  seriesCountByMetricName: (query: string): string => {
+  labelValueCountByLabelName: (query: string): string => `{${query}!=""}`,
-    return getSeriesSelector("__name__", query);
-  },
 };
 
 const getSeriesSelector = (label: string, value: string): string => {

app/vmui/packages/vmui/src/components/CardinalityPanel/types.ts

@@ -1,11 +1,12 @@
 import {MutableRef} from "preact/hooks";
 
 export interface TSDBStatus {
-  labelValueCountByLabelName: TopHeapEntry[];
-  seriesCountByLabelValuePair: TopHeapEntry[];
-  seriesCountByMetricName: TopHeapEntry[];
   totalSeries: number;
   totalLabelValuePairs: number;
+  seriesCountByMetricName: TopHeapEntry[];
+  seriesCountByLabelName: TopHeapEntry[];
+  seriesCountByLabelValuePair: TopHeapEntry[];
+  labelValueCountByLabelName: TopHeapEntry[];
 }
 
 export interface TopHeapEntry {
@@ -18,19 +19,22 @@ export type QueryUpdater = {
 }
 
 export interface Tabs {
-  labelValueCountByLabelName: string[];
-  seriesCountByLabelValuePair: string[];
   seriesCountByMetricName: string[];
+  seriesCountByLabelName: string[];
+  seriesCountByLabelValuePair: string[];
+  labelValueCountByLabelName: string[];
 }
 
 export interface Containers<T> {
-  labelValueCountByLabelName: MutableRef<T>;
-  seriesCountByLabelValuePair: MutableRef<T>;
   seriesCountByMetricName: MutableRef<T>;
+  seriesCountByLabelName: MutableRef<T>;
+  seriesCountByLabelValuePair: MutableRef<T>;
+  labelValueCountByLabelName: MutableRef<T>;
 }
 
 export interface DefaultState {
-  labelValueCountByLabelName: number;
-  seriesCountByLabelValuePair: number;
   seriesCountByMetricName: number;
+  seriesCountByLabelName: number;
+  seriesCountByLabelValuePair: number;
+  labelValueCountByLabelName: number;
 }

app/vmui/packages/vmui/src/hooks/useCardinalityFetch.ts

@@ -12,6 +12,7 @@ const defaultTSDBStatus = {
   totalSeries: 0,
   totalLabelValuePairs: 0,
   seriesCountByMetricName: [],
+  seriesCountByLabelName: [],
   seriesCountByLabelValuePair: [],
   labelValueCountByLabelName: [],
 };

docs/README.md

@@ -268,6 +268,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
 
 - To identify metric names with the highest number of series.
+- To idnetify labels with the highest number of series.
 - To identify label=name pairs with the highest number of series.
 - To identify labels with the highest number of unique values.
 
@@ -275,8 +276,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
 By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
 matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
 
-Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
-
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

docs/Single-server-VictoriaMetrics.md

@@ -272,6 +272,7 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 VictoriaMetrics provides an ability to explore time series cardinality at `cardinality` tab in [vmui](#vmui) in the following ways:
 
 - To identify metric names with the highest number of series.
+- To idnetify labels with the highest number of series.
 - To identify label=name pairs with the highest number of series.
 - To identify labels with the highest number of unique values.
 
@@ -279,8 +280,6 @@ By default cardinality explorer analyzes time series for the current date. It pr
 By default all the time series for the selected date are analyzed. It is possible to narrow down the analysis to series
 matching the specified [series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors).
 
-Cardinality explorer takes into account [deleted time series](#how-to-delete-time-series), because they stay in the inverted index for up to [-retentionPeriod](#retention). This means that the deleted time series take RAM, CPU, disk IO and disk space for the inverted index in the same way as other time series.
-
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
 See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).

lib/storage/index_db.go

@@ -834,10 +834,7 @@ func (is *indexSearch) searchLabelNamesWithFiltersOnDate(qt *querytracer.Tracer,
 		if err := mp.Init(item, nsPrefixExpected); err != nil {
 			return err
 		}
-		if mp.IsDeletedTag(dmis) {
+		if mp.GetMatchingSeriesCount(filter, dmis) == 0 {
-			continue
-		}
-		if mp.GetMatchingSeriesCount(filter) == 0 {
 			continue
 		}
 		labelName := mp.Tag.Key
@@ -1000,10 +997,7 @@ func (is *indexSearch) searchLabelValuesWithFiltersOnDate(qt *querytracer.Tracer
 		if err := mp.Init(item, nsPrefixExpected); err != nil {
 			return err
 		}
-		if mp.IsDeletedTag(dmis) {
+		if mp.GetMatchingSeriesCount(filter, dmis) == 0 {
-			continue
-		}
-		if mp.GetMatchingSeriesCount(filter) == 0 {
 			continue
 		}
 		labelValue := mp.Tag.Value
@@ -1150,7 +1144,7 @@ func (is *indexSearch) searchTagValueSuffixesForPrefix(tvss map[string]struct{},
 		if err := mp.Init(item, nsPrefix); err != nil {
 			return err
 		}
-		if mp.IsDeletedTag(dmis) {
+		if mp.GetMatchingSeriesCount(nil, dmis) == 0 {
 			continue
 		}
 		tagValue := mp.Tag.Value
@@ -1284,12 +1278,14 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t
 	ts := &is.ts
 	kb := &is.kb
 	mp := &is.mp
-	thLabelValueCountByLabelName := newTopHeap(topN)
+	dmis := is.db.s.getDeletedMetricIDs()
-	thSeriesCountByLabelValuePair := newTopHeap(topN)
 	thSeriesCountByMetricName := newTopHeap(topN)
-	var tmp, labelName, labelNameValue []byte
+	thSeriesCountByLabelName := newTopHeap(topN)
+	thSeriesCountByLabelValuePair := newTopHeap(topN)
+	thLabelValueCountByLabelName := newTopHeap(topN)
+	var tmp, prevLabelName, prevLabelValuePair []byte
 	var labelValueCountByLabelName, seriesCountByLabelValuePair uint64
-	var totalSeries, totalLabelValuePairs uint64
+	var totalSeries, labelSeries, totalLabelValuePairs uint64
 	nameEqualBytes := []byte("__name__=")
 
 	loopsPaceLimiter := 0
@@ -1314,69 +1310,80 @@ func (is *indexSearch) getTSDBStatusWithFiltersForDate(qt *querytracer.Tracer, t
 		if err := mp.Init(item, nsPrefixExpected); err != nil {
 			return nil, err
 		}
-		matchingSeriesCount := mp.GetMatchingSeriesCount(filter)
+		matchingSeriesCount := mp.GetMatchingSeriesCount(filter, dmis)
 		if matchingSeriesCount == 0 {
 			// Skip rows without matching metricIDs.
 			continue
 		}
 		tmp = append(tmp[:0], mp.Tag.Key...)
-		tagKey := tmp
+		labelName := tmp
-		if isArtificialTagKey(tagKey) {
+		if isArtificialTagKey(labelName) {
 			// Skip artificially created tag keys.
 			kb.B = append(kb.B[:0], prefix...)
-			if len(tagKey) > 0 && tagKey[0] == compositeTagKeyPrefix {
+			if len(labelName) > 0 && labelName[0] == compositeTagKeyPrefix {
 				kb.B = append(kb.B, compositeTagKeyPrefix)
 			} else {
-				kb.B = marshalTagValue(kb.B, tagKey)
+				kb.B = marshalTagValue(kb.B, labelName)
 			}
 			kb.B[len(kb.B)-1]++
 			ts.Seek(kb.B)
 			continue
 		}
-		if len(tagKey) == 0 {
+		if len(labelName) == 0 {
-			tagKey = append(tagKey, "__name__"...)
+			labelName = append(labelName, "__name__"...)
-			tmp = tagKey
+			tmp = labelName
+		}
+		if string(labelName) == "__name__" {
+			totalSeries += uint64(matchingSeriesCount)
 		}
 		tmp = append(tmp, '=')
 		tmp = append(tmp, mp.Tag.Value...)
-		tagKeyValue := tmp
+		labelValuePair := tmp
-		if string(tagKey) == "__name__" {
+		if len(prevLabelName) == 0 {
-			totalSeries += uint64(matchingSeriesCount)
+			prevLabelName = append(prevLabelName[:0], labelName...)
 		}
-		if !bytes.Equal(tagKey, labelName) {
+		if string(labelName) != string(prevLabelName) {
-			thLabelValueCountByLabelName.pushIfNonEmpty(labelName, labelValueCountByLabelName)
+			thLabelValueCountByLabelName.push(prevLabelName, labelValueCountByLabelName)
+			thSeriesCountByLabelName.push(prevLabelName, labelSeries)
+			labelSeries = 0
 			labelValueCountByLabelName = 0
-			labelName = append(labelName[:0], tagKey...)
+			prevLabelName = append(prevLabelName[:0], labelName...)
 		}
-		if !bytes.Equal(tagKeyValue, labelNameValue) {
+		if len(prevLabelValuePair) == 0 {
-			thSeriesCountByLabelValuePair.pushIfNonEmpty(labelNameValue, seriesCountByLabelValuePair)
+			prevLabelValuePair = append(prevLabelValuePair[:0], labelValuePair...)
-			if bytes.HasPrefix(labelNameValue, nameEqualBytes) {
+			labelValueCountByLabelName++
-				thSeriesCountByMetricName.pushIfNonEmpty(labelNameValue[len(nameEqualBytes):], seriesCountByLabelValuePair)
+		}
+		if string(labelValuePair) != string(prevLabelValuePair) {
+			thSeriesCountByLabelValuePair.push(prevLabelValuePair, seriesCountByLabelValuePair)
+			if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) {
+				thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair)
 			}
 			seriesCountByLabelValuePair = 0
 			labelValueCountByLabelName++
-			labelNameValue = append(labelNameValue[:0], tagKeyValue...)
+			prevLabelValuePair = append(prevLabelValuePair[:0], labelValuePair...)
 		}
-		// Take into account deleted timeseries too.
 		// It is OK if series can be counted multiple times in rare cases -
 		// the returned number is an estimation.
+		labelSeries += uint64(matchingSeriesCount)
 		seriesCountByLabelValuePair += uint64(matchingSeriesCount)
 		totalLabelValuePairs += uint64(matchingSeriesCount)
 	}
 	if err := ts.Error(); err != nil {
 		return nil, fmt.Errorf("error when counting time series by metric names: %w", err)
 	}
-	thLabelValueCountByLabelName.pushIfNonEmpty(labelName, labelValueCountByLabelName)
+	thLabelValueCountByLabelName.push(prevLabelName, labelValueCountByLabelName)
-	thSeriesCountByLabelValuePair.pushIfNonEmpty(labelNameValue, seriesCountByLabelValuePair)
+	thSeriesCountByLabelName.push(prevLabelName, labelSeries)
-	if bytes.HasPrefix(labelNameValue, nameEqualBytes) {
+	thSeriesCountByLabelValuePair.push(prevLabelValuePair, seriesCountByLabelValuePair)
-		thSeriesCountByMetricName.pushIfNonEmpty(labelNameValue[len(nameEqualBytes):], seriesCountByLabelValuePair)
+	if bytes.HasPrefix(prevLabelValuePair, nameEqualBytes) {
+		thSeriesCountByMetricName.push(prevLabelValuePair[len(nameEqualBytes):], seriesCountByLabelValuePair)
 	}
 	status := &TSDBStatus{
-		SeriesCountByMetricName:     thSeriesCountByMetricName.getSortedResult(),
-		LabelValueCountByLabelName:  thLabelValueCountByLabelName.getSortedResult(),
-		SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(),
 		TotalSeries:                 totalSeries,
 		TotalLabelValuePairs:        totalLabelValuePairs,
+		SeriesCountByMetricName:     thSeriesCountByMetricName.getSortedResult(),
+		SeriesCountByLabelName:      thSeriesCountByLabelName.getSortedResult(),
+		SeriesCountByLabelValuePair: thSeriesCountByLabelValuePair.getSortedResult(),
+		LabelValueCountByLabelName:  thLabelValueCountByLabelName.getSortedResult(),
 	}
 	return status, nil
 }
@@ -1388,8 +1395,9 @@ type TSDBStatus struct {
 	TotalSeries                 uint64
 	TotalLabelValuePairs        uint64
 	SeriesCountByMetricName     []TopHeapEntry
-	LabelValueCountByLabelName  []TopHeapEntry
+	SeriesCountByLabelName      []TopHeapEntry
 	SeriesCountByLabelValuePair []TopHeapEntry
+	LabelValueCountByLabelName  []TopHeapEntry
 }
 
 func (status *TSDBStatus) hasEntries() bool {
@@ -1415,7 +1423,7 @@ type TopHeapEntry struct {
 	Count uint64
 }
 
-func (th *topHeap) pushIfNonEmpty(name []byte, count uint64) {
+func (th *topHeap) push(name []byte, count uint64) {
 	if count == 0 {
 		return
 	}
@@ -3108,39 +3116,27 @@ func (mp *tagToMetricIDsRowParser) ParseMetricIDs() {
 	mp.metricIDsParsed = true
 }
 
-// GetMatchingSeriesCount returns the number of series in mp, which match metricIDs from the given filter.
+// GetMatchingSeriesCount returns the number of series in mp, which match metricIDs from the given filter
+// and do not match metricIDs from negativeFilter.
 //
 // if filter is empty, then all series in mp are taken into account.
-func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter *uint64set.Set) int {
+func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter, negativeFilter *uint64set.Set) int {
-	if filter == nil {
+	if filter == nil && negativeFilter.Len() == 0 {
 		return mp.MetricIDsLen()
 	}
 	mp.ParseMetricIDs()
 	n := 0
 	for _, metricID := range mp.MetricIDs {
-		if filter.Has(metricID) {
+		if filter != nil && !filter.Has(metricID) {
+			continue
+		}
+		if !negativeFilter.Has(metricID) {
 			n++
 		}
 	}
 	return n
 }
 
-// IsDeletedTag verifies whether the tag from mp is deleted according to dmis.
-//
-// dmis must contain deleted MetricIDs.
-func (mp *tagToMetricIDsRowParser) IsDeletedTag(dmis *uint64set.Set) bool {
-	if dmis.Len() == 0 {
-		return false
-	}
-	mp.ParseMetricIDs()
-	for _, metricID := range mp.MetricIDs {
-		if !dmis.Has(metricID) {
-			return false
-		}
-	}
-	return true
-}
-
 func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
 	data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
 	data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)

lib/storage/index_db_test.go

@@ -1824,6 +1824,27 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
 	if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
 		t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
 	}
+	expectedSeriesCountByLabelName := []TopHeapEntry{
+		{
+			Name:  "__name__",
+			Count: 1000,
+		},
+		{
+			Name:  "constant",
+			Count: 1000,
+		},
+		{
+			Name:  "day",
+			Count: 1000,
+		},
+		{
+			Name:  "uniqueid",
+			Count: 1000,
+		},
+	}
+	if !reflect.DeepEqual(status.SeriesCountByLabelName, expectedSeriesCountByLabelName) {
+		t.Fatalf("unexpected SeriesCountByLabelName;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelName, expectedSeriesCountByLabelName)
+	}
 	expectedLabelValueCountByLabelName := []TopHeapEntry{
 		{
 			Name:  "uniqueid",