diff --git a/app/vmagent/remotewrite/remotewrite.go b/app/vmagent/remotewrite/remotewrite.go index f8d1805db8..b5e57377e8 100644 --- a/app/vmagent/remotewrite/remotewrite.go +++ b/app/vmagent/remotewrite/remotewrite.go @@ -390,6 +390,8 @@ var labelsHashBufPool bytesutil.ByteBufferPool func logSkippedSeries(labels []prompbmarshal.Label, flagName string, flagValue int) { select { case <-logSkippedSeriesTicker.C: + // Do not use logger.WithThrottler() here, since this will increase CPU usage + // because every call to logSkippedSeries will result to a call to labelsToString. logger.Warnf("skip series %s because %s=%d reached", labelsToString(labels), flagName, flagValue) default: } diff --git a/lib/storage/metric_name.go b/lib/storage/metric_name.go index e997006d5f..f16ddd9296 100644 --- a/lib/storage/metric_name.go +++ b/lib/storage/metric_name.go @@ -511,6 +511,8 @@ func trackDroppedLabels(labels, droppedLabels []prompb.Label) { atomic.AddUint64(&MetricsWithDroppedLabels, 1) select { case <-droppedLabelsLogTicker.C: + // Do not call logger.WithThrottler() here, since this will result in increased CPU usage + // because labelsToString() will be called with each trackDroppedLAbels call. logger.Warnf("dropping %d labels for %s; dropped labels: %s; either reduce the number of labels for this metric "+ "or increase -maxLabelsPerTimeseries=%d command-line flag value", len(droppedLabels), labelsToString(labels), labelsToString(droppedLabels), maxLabelsPerTimeseries) diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 93fc92ac85..52c1048474 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -841,7 +841,8 @@ func (pt *partition) ForceMergeAllParts() error { maxOutBytes := fs.MustGetFreeSpace(pt.bigPartsPath) if newPartSize > maxOutBytes { freeSpaceNeededBytes := newPartSize - maxOutBytes - logger.Warnf("cannot initiate force merge for the partition %s; additional space needed: %d bytes", pt.name, freeSpaceNeededBytes) + logger.WithThrottler("forceMerge", time.Minute).Warnf("cannot initiate force merge for the partition %s; additional space needed: %d bytes", + pt.name, freeSpaceNeededBytes) return nil } diff --git a/lib/storage/storage.go b/lib/storage/storage.go index e10df1a80b..5123edb7bb 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1808,7 +1808,7 @@ func (s *Storage) add(rows []rawRow, dstMrs []*MetricRow, mrs []MetricRow, preci atomic.AddUint64(&s.slowRowInserts, slowInsertsCount) } if firstWarn != nil { - logger.Warnf("warn occurred during rows addition: %s", firstWarn) + logger.WithThrottler("storageAddRows", 5*time.Second).Warnf("warn occurred during rows addition: %s", firstWarn) } dstMrs = dstMrs[:j] rows = rows[:j] @@ -1843,6 +1843,8 @@ func (s *Storage) isSeriesCardinalityExceeded(metricID uint64, metricNameRaw []b func logSkippedSeries(metricNameRaw []byte, flagName string, flagValue int) { select { case <-logSkippedSeriesTicker.C: + // Do not use logger.WithThrottler() here, since this will result in increased CPU load + // because of getUserReadableMetricName() calls per each logSkippedSeries call. logger.Warnf("skip series %s because %s=%d reached", getUserReadableMetricName(metricNameRaw), flagName, flagValue) default: }