Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/storage: log original labels set when label value is truncated (#3952)

Browse Source 
lib/storage: log original labels set when label value is truncated
This commit is contained in:
Zakhar Bessarab 2023-03-14 13:59:40 +04:00 committed by Aliaksandr Valialkin
parent 235477628e
commit d1d108fe77
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/CHANGELOG.md
View File

@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip ## tip
* FEATURE: log metrics with truncated labels if the length of label value in the ingested metric exceeds `-maxLabelValueLen`. This should simplify debugging for this case.
## [v1.89.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.89.1) ## [v1.89.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.89.1)
Released at 2023-03-12 Released at 2023-03-12

18
lib/storage/metric_name.go
View File

@ -547,7 +547,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
label.Name = label.Name[:maxLabelNameLen] label.Name = label.Name[:maxLabelNameLen]
} }
if len(label.Value) > maxLabelValueLen { if len(label.Value) > maxLabelValueLen {
atomic.AddUint64(&TooLongLabelValues, 1) trackTruncatedLabels(labels, label)
label.Value = label.Value[:maxLabelValueLen] label.Value = label.Value[:maxLabelValueLen]
} }
if len(label.Value) == 0 { if len(label.Value) == 0 {
@ -597,7 +597,7 @@ func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
select { select {
case <-droppedLabelsLogTicker.C: case <-droppedLabelsLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage // Do not call logger.WithThrottler() here, since this will result in increased CPU usage
// because labelsToString() will be called with each trackDroppedLAbels call. // because labelsToString() will be called with each trackDroppedLabels call.
logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+
"or increase -maxLabelsPerTimeseries=%d command-line flag value", "or increase -maxLabelsPerTimeseries=%d command-line flag value",
len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries)
@ -605,7 +605,21 @@ func trackDroppedLabels(labels, droppedLabels []prompb.Label) {
} }
} }
func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) {
atomic.AddUint64(&TooLongLabelValues, 1)
select {
case <-truncatedLabelsLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
// because labelsToString() will be called with each trackTruncatedLabels call.
logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+
" truncated label: %s; original labels: %s; either reduce the label value length or increase -maxLabelValueLen=%d;",
maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), maxLabelValueLen)
default:
}
}
var droppedLabelsLogTicker = time.NewTicker(5 * time.Second) var droppedLabelsLogTicker = time.NewTicker(5 * time.Second)
var truncatedLabelsLogTicker = time.NewTicker(5 * time.Second)
func labelsToString(labels []prompb.Label) string { func labelsToString(labels []prompb.Label) string {
labelsCopy := append([]prompb.Label{}, labels...) labelsCopy := append([]prompb.Label{}, labels...)