Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/storage: print tenant ID in log when discarding or truncating labels (#5658)

Browse Source 
Previously, it was not possible to determine which tenant sends metrics with excessive amount of labels of label values.

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Zakhar Bessarab 2024-01-23 04:27:59 +04:00 committed by GitHub
parent 22a8664f51
commit 60ef978ffc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/CHANGELOG.md
View File

@ -53,6 +53,7 @@ The sandbox cluster installation is running under the constant load generated by
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-vm-native-src-insecure-skip-verify` and `-vm-native-dst-insecure-skip-verify` command-line flags for native protocol. It can be used for skipping TLS certificate verification when connecting to the source or destination addresses.
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): add `job` label to `DiskRunsOutOfSpace` alerting rule, so it is easier to understand to which installation the triggered instance belongs. * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): add `job` label to `DiskRunsOutOfSpace` alerting rule, so it is easier to understand to which installation the triggered instance belongs.
* FEATURE: `vmstorage`: add tenant identifier for log messages regarding dropping excessive labels due to limits defined by `-maxLabelsPerTimeseries` or `-maxLabelValueLen` command-line flags. Previously, it was hard to understand to which tenant the dropped labels belong.
* FEATURE: add [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/grafana-datasource) to docker compose environment. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5363). * FEATURE: add [VictoriaMetrics datasource](https://github.com/VictoriaMetrics/grafana-datasource) to docker compose environment. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5363).
* BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649). * BUGFIX: properly return errors from [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). Previously these errors were silently suppressed. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5649).

16
lib/storage/metric_name.go
View File

@ -578,7 +578,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
dstSize := dstLen + 8 dstSize := dstLen + 8
for i := range labels { for i := range labels {
if i >= maxLabelsPerTimeseries { if i >= maxLabelsPerTimeseries {
trackDroppedLabels(labels, labels[i:]) trackDroppedLabels(labels, labels[i:], accountID, projectID)
break break
} }
label := &labels[i] label := &labels[i]
@ -587,7 +587,7 @@ func MarshalMetricNameRaw(dst []byte, accountID, projectID uint32, labels []prom
label.Name = label.Name[:maxLabelNameLen] label.Name = label.Name[:maxLabelNameLen]
} }
if len(label.Value) > maxLabelValueLen { if len(label.Value) > maxLabelValueLen {
trackTruncatedLabels(labels, label) trackTruncatedLabels(labels, label, accountID, projectID)
label.Value = label.Value[:maxLabelValueLen] label.Value = label.Value[:maxLabelValueLen]
} }
if len(label.Value) == 0 { if len(label.Value) == 0 {
@ -632,28 +632,28 @@ var (
TooLongLabelValues uint64 TooLongLabelValues uint64
) )
func trackDroppedLabels(labels, droppedLabels []prompb.Label) { func trackDroppedLabels(labels, droppedLabels []prompb.Label, accountID, projectID uint32) {
atomic.AddUint64(&MetricsWithDroppedLabels, 1) atomic.AddUint64(&MetricsWithDroppedLabels, 1)
select { select {
case <-droppedLabelsLogTicker.C: case <-droppedLabelsLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage // Do not call logger.WithThrottler() here, since this will result in increased CPU usage
// because labelsToString() will be called with each trackDroppedLabels call. // because labelsToString() will be called with each trackDroppedLabels call.
logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ logger.Warnf("dropping %d labels for %s; dropped labels: %s; tenant: %d:%d; either reduce the number of labels for this metric "+
"or increase -maxLabelsPerTimeseries=%d command-line flag value", "or increase -maxLabelsPerTimeseries=%d command-line flag value",
len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), accountID, projectID, maxLabelsPerTimeseries)
default: default:
} }
} }
func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label) { func trackTruncatedLabels(labels []prompb.Label, truncated *prompb.Label, accountID, projectID uint32) {
atomic.AddUint64(&TooLongLabelValues, 1) atomic.AddUint64(&TooLongLabelValues, 1)
select { select {
case <-truncatedLabelsLogTicker.C: case <-truncatedLabelsLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage // Do not call logger.WithThrottler() here, since this will result in increased CPU usage
// because labelsToString() will be called with each trackTruncatedLabels call. // because labelsToString() will be called with each trackTruncatedLabels call.
logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+ logger.Warnf("truncated label value as it exceeds configured maximal label value length: max %d, actual %d;"+
" truncated label: %s; original labels: %s; either reduce the label value length or increase -maxLabelValueLen=%d;", " truncated label: %s; original labels: %s; tenant: %d:%d; either reduce the label value length or increase -maxLabelValueLen=%d;",
maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), maxLabelValueLen) maxLabelValueLen, len(truncated.Value), truncated.Name, labelsToString(labels), accountID, projectID, maxLabelValueLen)
default: default:
} }
} }