Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-07 08:32:18 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/storage: prevent excessive loops when storage is in RO (#2962)

Browse Source 
* lib/storage: prevent excessive loops when storage is in RO

Returning nil error when storage is in RO mode results
into excessive loops and function calls which could
result into CPU exhaustion. Returning an err instead
will trigger delays in the for loop and save some resources.

Signed-off-by: hagen1778 <roman@victoriametrics.com>

* document the change

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Roman Khavronenko 2022-08-09 11:17:00 +02:00 committed by Aliaksandr Valialkin
parent f90f654cf2
commit f42853275f
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/CHANGELOG.md
View File

@ -15,6 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip ## tip
* BUGFIX: prevent from excess CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode).
## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0) ## [v1.80.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.80.0)

10
lib/storage/partition.go
View File

@ -569,7 +569,7 @@ func (pt *partition) addRowsPart(rows []rawRow) {
atomic.AddUint64(&pt.smallAssistedMerges, 1) atomic.AddUint64(&pt.smallAssistedMerges, 1)
return return
} }
if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) { if errors.Is(err, errNothingToMerge) || errors.Is(err, errForciblyStopped) || errors.Is(err, errReadOnlyMode) {
return return
} }
logger.Panicf("FATAL: cannot merge small parts: %s", err) logger.Panicf("FATAL: cannot merge small parts: %s", err)
@ -956,7 +956,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
// The merger has been stopped. // The merger has been stopped.
return nil return nil
} }
if !errors.Is(err, errNothingToMerge) { if !errors.Is(err, errNothingToMerge) && !errors.Is(err, errReadOnlyMode) {
return err return err
} }
if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
@ -1012,11 +1012,13 @@ func (pt *partition) canBackgroundMerge() bool {
return atomic.LoadUint32(pt.isReadOnly) == 0 return atomic.LoadUint32(pt.isReadOnly) == 0
} }
var errReadOnlyMode = fmt.Errorf("storage is in readonly mode")
func (pt *partition) mergeBigParts(isFinal bool) error { func (pt *partition) mergeBigParts(isFinal bool) error {
if !pt.canBackgroundMerge() { if !pt.canBackgroundMerge() {
// Do not perform merge in read-only mode, since this may result in disk space shortage. // Do not perform merge in read-only mode, since this may result in disk space shortage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
return nil return errReadOnlyMode
} }
maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount) maxOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)
@ -1032,7 +1034,7 @@ func (pt *partition) mergeSmallParts(isFinal bool) error {
if !pt.canBackgroundMerge() { if !pt.canBackgroundMerge() {
// Do not perform merge in read-only mode, since this may result in disk space shortage. // Do not perform merge in read-only mode, since this may result in disk space shortage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2603
return nil return errReadOnlyMode
} }
// Try merging small parts to a big part at first. // Try merging small parts to a big part at first.
maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount) maxBigPartOutBytes := getMaxOutBytes(pt.bigPartsPath, bigMergeWorkersCount)