From b542e5068078f4e01892f44af65345c4102f1194 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 19 Jun 2020 01:10:18 +0300
Subject: [PATCH] app/vminsert: export metrics for determining ingested rows
 with dropped or truncated labels

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/565
---
 README.md                             |  4 ++++
 app/vminsert/main.go                  | 11 +++++++++++
 docs/Single-server-VictoriaMetrics.md |  4 ++++
 lib/storage/metric_name.go            | 15 +++++++++++++++
 4 files changed, 34 insertions(+)

diff --git a/README.md b/README.md
index cca34ca2f..9af260c7d 100644
--- a/README.md
+++ b/README.md
@@ -987,6 +987,10 @@ The most interesting metrics are:
   VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
   while `topN` equals to 10.
 
+* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.
+  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+
 
 ### Backfilling
 
diff --git a/app/vminsert/main.go b/app/vminsert/main.go
index 29c340e18..34cf9804c 100644
--- a/app/vminsert/main.go
+++ b/app/vminsert/main.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -160,4 +161,14 @@ var (
 	promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
+
+	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+	})
 )
diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md
index cca34ca2f..9af260c7d 100644
--- a/docs/Single-server-VictoriaMetrics.md
+++ b/docs/Single-server-VictoriaMetrics.md
@@ -987,6 +987,10 @@ The most interesting metrics are:
   VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
   while `topN` equals to 10.
 
+* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.
+  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+
 
 ### Backfilling
 
diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go
index 5c4c105d2..ab9b4e248 100644
--- a/lib/storage/metric_name.go
+++ b/lib/storage/metric_name.go
@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -417,13 +418,16 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 	dstSize := dstLen
 	for i := range labels {
 		if i >= maxLabelsPerTimeseries {
+			atomic.AddUint64(&MetricsWithDroppedLabels, 1)
 			break
 		}
 		label := &labels[i]
 		if len(label.Name) > maxLabelNameLen {
+			atomic.AddUint64(&TooLongLabelNames, 1)
 			label.Name = label.Name[:maxLabelNameLen]
 		}
 		if len(label.Value) > maxLabelValueLen {
+			atomic.AddUint64(&TooLongLabelValues, 1)
 			label.Value = label.Value[:maxLabelValueLen]
 		}
 		if len(label.Value) == 0 {
@@ -455,6 +459,17 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 	return dst
 }
 
+var (
+	// MetricsWithDroppedLabels is the number of metrics with at least a single dropped label
+	MetricsWithDroppedLabels uint64
+
+	// TooLongLabelNames is the number of too long label names
+	TooLongLabelNames uint64
+
+	// TooLongLabelValues is the number of too long label values
+	TooLongLabelValues uint64
+)
+
 // marshalRaw marshals mn to dst and returns the result.
 //
 // The results may be unmarshaled with MetricName.unmarshalRaw.