From d1d108fe77f091659d4f3719830c8928b6f9709c Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 14 Mar 2023 13:59:40 +0400 Subject: [PATCH] lib/storage: log original labels set when label value is truncated (#3952) lib/storage: log original labels set when label value is truncated --- docs/CHANGELOG.md | 2 ++ lib/storage/metric_name.go | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c9bfcecc34..faa268393d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* FEATURE: log metrics with truncated labels if the length of label value in the ingested metric exceeds `-maxLabelValueLen`. This should simplify debugging for this case. + ## [v1.89.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.89.1) Released at 2023-03-12 diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index 57857c27a3..0d5dec09ff 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -547,7 +547,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom label.Name = label.Name[:maxLabelNameLen] } if len(label.Value) > maxLabelValueLen { - atomic.AddUint64(&TooLongLabelValues, 1) + trackTruncatedLabels(labels, label) label.Value = label.Value[:maxLabelValueLen] } if len(label.Value) == 0 { @@ -597,7 +597,7 @@ func trackDroppedLabels(labels, droppedLabels []prompb.Label) { select { case <-droppedLabelsLogTicker.C: // Do not call logger.WithThrottler() here, since this will result in increased CPU usage - // because labelsToString() will be called with each trackDroppedLAbels call. + // because labelsToString() will be called with each trackDroppedLabels call. logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ "or increase -maxLabelsPerTimeseries=%d command-line flag value", len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) @@ -605,7 +605,21 @@ func trackDroppedLabels(labels, droppedLabels []prompb.Label) { } } +func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) { + atomic.AddUint64(&TooLongLabelValues, 1) + select { + case <-truncatedLabelsLogTicker.C: + // Do not call logger.WithThrottler() here, since this will result in increased CPU usage + // because labelsToString() will be called with each trackTruncatedLabels call. + logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+ + " truncated label: %s; original labels: %s; either reduce the label value length or increase -maxLabelValueLen=%d;", + maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), maxLabelValueLen) + default: + } +} + var droppedLabelsLogTicker = time.NewTicker(5 * time.Second) +var truncatedLabelsLogTicker = time.NewTicker(5 * time.Second) func labelsToString(labels []prompb.Label) string { labelsCopy := append([]prompb.Label{}, labels...)