Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 08:23:34 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

app/vmstorage: disable final merge by default, since it may result in high disk IO and CPU usage without measurable benefits such as increased query performance and reduced disk space usage

Browse Source
This commit is contained in:
Aliaksandr Valialkin 2021-01-08 00:09:00 +02:00
parent bd7cb4f5be
commit d5a2b120e9
3 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/vmstorage/main.go
View File

@ -31,9 +31,9 @@ var (
forceMergeAuthKey = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages") forceMergeAuthKey = flag.String("forceMergeAuthKey", "", "authKey, which must be passed in query string to /internal/force_merge pages")
forceFlushAuthKey = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages") forceFlushAuthKey = flag.String("forceFlushAuthKey", "", "authKey, which must be passed in query string to /internal/force_flush pages")
finalMergeDelay = flag.Duration("finalMergeDelay", 30*time.Second, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+ finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
"Query speed and disk space usage is usually reduced after the final merge is complete. Too low delay for final merge may result in increased "+ "Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
"disk IO usage and CPU usage") "Zero value disables final merge")
bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0") bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0") smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+ minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Remove superflouos samples from time series if they are located closer to each other than this duration. "+

2
docs/CHANGELOG.md
View File

@ -2,6 +2,8 @@
# tip # tip
* FEATURE: disable final merge for data for the previous month at the beginning of new month, since it may result in high disk IO and CPU usage. Final merge can be enabled by setting `-finalMergeDelay` command-line flag to positive duration.
# [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0) # [v1.51.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.51.0)

11
lib/storage/partition.go
View File

@ -976,7 +976,7 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
if !errors.Is(err, errNothingToMerge) { if !errors.Is(err, errNothingToMerge) {
return err return err
} }
if fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds { if finalMergeDelaySeconds > 0 && fasttime.UnixTimestamp()-lastMergeTime > finalMergeDelaySeconds {
// We have free time for merging into bigger parts. // We have free time for merging into bigger parts.
// This should improve select performance. // This should improve select performance.
lastMergeTime = fasttime.UnixTimestamp() lastMergeTime = fasttime.UnixTimestamp()
@ -998,17 +998,18 @@ func (pt *partition) partsMerger(mergerFunc func(isFinal bool) error) error {
} }
} }
var finalMergeDelaySeconds = uint64(30) // Disable final merge by default, since it may lead to high disk IO and CPU usage
// at the beginning of every month when merging data for the previous month.
var finalMergeDelaySeconds = uint64(0)
// SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data. // SetFinalMergeDelay sets the delay before doing final merge for partitions without newly ingested data.
// //
// This function may be called only before Storage initialization. // This function may be called only before Storage initialization.
func SetFinalMergeDelay(delay time.Duration) { func SetFinalMergeDelay(delay time.Duration) {
delaySeconds := int(delay.Seconds() + 0.5) if delay <= 0 {
if delaySeconds <= 0 {
return return
} }
finalMergeDelaySeconds = uint64(delaySeconds) finalMergeDelaySeconds = uint64(delay.Seconds() + 1)
} }
func maxRowsByPath(path string) uint64 { func maxRowsByPath(path string) uint64 {