diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b94a8a8560..86c1108a4f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). ## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0) diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 969211c83e..7946eeeebc 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -569,7 +569,7 @@ func (pt *partition) addRowsPart(rows []rawRow) { atomic.AddUint64(&pt.smallAssistedMerges, 1) return } - if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) { + if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) { return } logger.Panicf("FATAL: cannot merge small parts: %s", err) @@ -956,7 +956,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error { // The merger has been stopped. return nil } - if !errors.Is(err, errNothingToMerge) { + if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) { return err } if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { @@ -1012,11 +1012,13 @@ func (pt *partition) canBackgroundMerge() bool { return atomic.LoadUint32(pt.isReadOnly) == 0 } +var errReadOnlyMode = fmt.Errorf("storage is in readonly mode") + func (pt *partition) mergeBigParts(isFinal bool) error { if !pt.canBackgroundMerge() { // Do not perform merge in read-only mode, since this may result in disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount) @@ -1032,7 +1034,7 @@ func (pt *partition) mergeSmallParts(isFinal bool) error { if !pt.canBackgroundMerge() { // Do not perform merge in read-only mode, since this may result in disk space shortage. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 - return nil + return errReadOnlyMode } // Try merging small parts to a big part at first. maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)