From 60ef978ffc5ddf2f46e376d748fb2c39ed63729d Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Tue, 23 Jan 2024 04:27:59 +0400 Subject: [PATCH] lib/storage: print tenant ID in log when discarding or truncating labels (#5658) Previously, it was not possible to determine which tenant sends metrics with excessive amount of labels of label values. Signed-off-by: Zakhar Bessarab Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 1 + lib/storage/metric_name.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 59babec408..fad3229326 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -53,6 +53,7 @@ The sandbox cluster installation is running under the constant load generated by * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses. * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): add `job` label to `DiskRunsOutOfSpace` alerting rule, so it is easier to understand to which installation the triggered instance belongs. +* FEATURE: `vmstorage`: add tenant identifier for log messages regarding dropping excessive labels due to limits defined by `-maxLabelsPerTimeseries` or `-maxLabelValueLen` command-line flags. Previously, it was hard to understand to which tenant the dropped labels belong. * FEATURE: add [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/grafana-datasource) to docker compose environment. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5363). * BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649). diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index e1f5b36633..7a1c15904f 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -578,7 +578,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom dstSize := dstLen + 8 for i := range labels { if i >= maxLabelsPerTimeseries { - trackDroppedLabels(labels, labels[i:]) + trackDroppedLabels(labels, labels[i:], accountID, projectID) break } label := &labels[i] @@ -587,7 +587,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom label.Name = label.Name[:maxLabelNameLen] } if len(label.Value) > maxLabelValueLen { - trackTruncatedLabels(labels, label) + trackTruncatedLabels(labels, label, accountID, projectID) label.Value = label.Value[:maxLabelValueLen] } if len(label.Value) == 0 { @@ -632,28 +632,28 @@ var ( TooLongLabelValues uint64 ) -func trackDroppedLabels(labels, droppedLabels []prompb.Label) { +func trackDroppedLabels(labels, droppedLabels []prompb.Label, accountID, projectID uint32) { atomic.AddUint64(&MetricsWithDroppedLabels, 1) select { case <-droppedLabelsLogTicker.C: // Do not call logger.WithThrottler() here, since this will result in increased CPU usage // because labelsToString() will be called with each trackDroppedLabels call. - logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ + logger.Warnf("dropping %d labels for %s; dropped labels: %s; tenant: %d:%d; either reduce the number of labels for this metric "+ "or increase -maxLabelsPerTimeseries=%d command-line flag value", - len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) + len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), accountID, projectID, maxLabelsPerTimeseries) default: } } -func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) { +func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label, accountID, projectID uint32) { atomic.AddUint64(&TooLongLabelValues, 1) select { case <-truncatedLabelsLogTicker.C: // Do not call logger.WithThrottler() here, since this will result in increased CPU usage // because labelsToString() will be called with each trackTruncatedLabels call. logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+ - " truncated label: %s; original labels: %s; either reduce the label value length or increase -maxLabelValueLen=%d;", - maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), maxLabelValueLen) + " truncated label: %s; original labels: %s; tenant: %d:%d; either reduce the label value length or increase -maxLabelValueLen=%d;", + maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), accountID, projectID, maxLabelValueLen) default: } }