app/vminsert: export metrics for determining ingested rows with dropped or truncated labels

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/565

README.md

@@ -987,6 +987,10 @@ The most interesting metrics are:
   VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
   while `topN` equals to 10.
 
+* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.
+  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+
 
 ### Backfilling
 

app/vminsert/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite"
@@ -160,4 +161,14 @@ var (
 	promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`)
 
 	promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`)
+
+	_ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelNames))
+	})
+	_ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 {
+		return float64(atomic.LoadUint64(&storage.TooLongLabelValues))
+	})
 )

docs/Single-server-VictoriaMetrics.md

@@ -987,6 +987,10 @@ The most interesting metrics are:
   VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date,
   while `topN` equals to 10.
 
+* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.
+  This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total`
+  metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload.
+
 
 ### Backfilling
 

lib/storage/metric_name.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -417,13 +418,16 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 	dstSize := dstLen
 	for i := range labels {
 		if i >= maxLabelsPerTimeseries {
+			atomic.AddUint64(&MetricsWithDroppedLabels, 1)
 			break
 		}
 		label := &labels[i]
 		if len(label.Name) > maxLabelNameLen {
+			atomic.AddUint64(&TooLongLabelNames, 1)
 			label.Name = label.Name[:maxLabelNameLen]
 		}
 		if len(label.Value) > maxLabelValueLen {
+			atomic.AddUint64(&TooLongLabelValues, 1)
 			label.Value = label.Value[:maxLabelValueLen]
 		}
 		if len(label.Value) == 0 {
@@ -455,6 +459,17 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
 	return dst
 }
 
+var (
+	// MetricsWithDroppedLabels is the number of metrics with at least a single dropped label
+	MetricsWithDroppedLabels uint64
+
+	// TooLongLabelNames is the number of too long label names
+	TooLongLabelNames uint64
+
+	// TooLongLabelValues is the number of too long label values
+	TooLongLabelValues uint64
+)
+
 // marshalRaw marshals mn to dst and returns the result.
 //
 // The results may be unmarshaled with MetricName.unmarshalRaw.