From f42853275ff86f5d3cc99da1ace739286eb671ff Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Tue, 9 Aug 2022 11:17:00 +0200 Subject: [PATCH] lib/storage: prevent excessive loops when storage is in RO (#2962) * lib/storage: prevent excessive loops when storage is in RO Returning nil error when storage is in RO mode results into excessive loops and function calls which could result into CPU exhaustion. Returning an err instead will trigger delays in the for loop and save some resources. Signed-off-by: hagen1778 * document the change Co-authored-by: Aliaksandr Valialkin --- docs/CHANGELOG.md | 1 + lib/storage/partition.go | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b94a8a8560..86c1108a4f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). ## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0) diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 969211c83e..7946eeeebc 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -569,7 +569,7 @@ func (pt *partition) addRowsPart(rows []rawRow) { atomic.AddUint64(&pt.smallAssistedMerges, 1) return } - if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) { + if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) { return } logger.Panicf("FATAL: cannot merge small parts: %s", err) @@ -956,7 +956,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error { // The merger has been stopped. return nil } - if !errors.Is(err, errNothingToMerge) { + if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) { return err } if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { @@ -1012,11 +1012,13 @@ func (pt *partition) canBackgroundMerge() bool { return atomic.LoadUint32(pt.isReadOnly) == 0 } +var errReadOnlyMode = fmt.Errorf("storage is in readonly mode") + func (pt *partition) mergeBigParts(isFinal bool) error { if !pt.canBackgroundMerge() { // Do not perform merge in read-only mode, since this may result in disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount) @@ -1032,7 +1034,7 @@ func (pt *partition) mergeSmallParts(isFinal bool) error { if !pt.canBackgroundMerge() { // Do not perform merge in read-only mode, since this may result in disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } // Try merging small parts to a big part at first. maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)