diff --git a/README.md b/README.md index cca34ca2f..9af260c7d 100644 --- a/README.md +++ b/README.md @@ -987,6 +987,10 @@ The most interesting metrics are: VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date, while `topN` equals to 10. +* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag. + This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total` + metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload. + ### Backfilling diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 29c340e18..34cf9804c 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync/atomic" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/csvimport" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite" @@ -160,4 +161,14 @@ var ( promscrapeTargetsRequests = metrics.NewCounter(`vm_http_requests_total{path="/targets"}`) promscrapeConfigReloadRequests = metrics.NewCounter(`vm_http_requests_total{path="/-/reload"}`) + + _ = metrics.NewGauge(`vm_metrics_with_dropped_labels_total`, func() float64 { + return float64(atomic.LoadUint64(&storage.MetricsWithDroppedLabels)) + }) + _ = metrics.NewGauge(`vm_too_long_label_names_total`, func() float64 { + return float64(atomic.LoadUint64(&storage.TooLongLabelNames)) + }) + _ = metrics.NewGauge(`vm_too_long_label_values_total`, func() float64 { + return float64(atomic.LoadUint64(&storage.TooLongLabelValues)) + }) ) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index cca34ca2f..9af260c7d 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -987,6 +987,10 @@ The most interesting metrics are: VictoriaMetrics accepts optional `date=YYYY-MM-DD` and `topN=42` args on this page. By default `date` equals to the current date, while `topN` equals to 10. +* VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag. + This prevents from ingesting metrics with too many labels. It is recommended [monitoring](#monitoring) `vm_metrics_with_dropped_labels_total` + metric in order to determine whether `-maxLabelsPerTimeseries` must be adjusted for your workload. + ### Backfilling diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index 5c4c105d2..ab9b4e248 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -6,6 +6,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding" @@ -417,13 +418,16 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte { dstSize := dstLen for i := range labels { if i >= maxLabelsPerTimeseries { + atomic.AddUint64(&MetricsWithDroppedLabels, 1) break } label := &labels[i] if len(label.Name) > maxLabelNameLen { + atomic.AddUint64(&TooLongLabelNames, 1) label.Name = label.Name[:maxLabelNameLen] } if len(label.Value) > maxLabelValueLen { + atomic.AddUint64(&TooLongLabelValues, 1) label.Value = label.Value[:maxLabelValueLen] } if len(label.Value) == 0 { @@ -455,6 +459,17 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte { return dst } +var ( + // MetricsWithDroppedLabels is the number of metrics with at least a single dropped label + MetricsWithDroppedLabels uint64 + + // TooLongLabelNames is the number of too long label names + TooLongLabelNames uint64 + + // TooLongLabelValues is the number of too long label values + TooLongLabelValues uint64 +) + // marshalRaw marshals mn to dst and returns the result. // // The results may be unmarshaled with MetricName.unmarshalRaw.