From f6d11a49aa20374ab7d97430c06ebbf7fdcebf84 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 25 May 2022 15:57:01 +0300 Subject: [PATCH] lib/storage: add ability to change the indexdb rotation time offset with -retentionTimezoneOffset command-line flag This is a follow-up for 0fbf59199ab807182232ad5ee8aff1c45e5fc0ed See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574 --- app/vmstorage/main.go | 8 ++++++-- docs/CHANGELOG.md | 1 + lib/storage/storage.go | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index e738ea318..1e724c549 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -36,8 +36,11 @@ var ( finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+ "Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+ "Zero value disables final merge") - bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0") - smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0") + bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0") + smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0") + retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+ + "If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+ + "If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)") logNewSeries = flag.Bool("logNewSeries", false, "Whether to log new series. This option is for debug purposes only. It can lead to performance issues "+ "when big number of new series are ingested into VictoriaMetrics") @@ -90,6 +93,7 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) { storage.SetFinalMergeDelay(*finalMergeDelay) storage.SetBigMergeWorkersCount(*bigMergeConcurrency) storage.SetSmallMergeWorkersCount(*smallMergeConcurrency) + storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset) storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N) storage.SetTSIDCacheSize(cacheSizeStorageTSID.N) mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.N) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2d09271ab..f3adb5a86 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): expose `/api/v1/status/config` endpoint in the same way as Prometheus does. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/api/#config). +* FEATURE: add ability to change the `indexdb` rotation timezone offset via `-retentionTimezoneOffset` command-line flag. Previously it was performed at 4am UTC time. This could lead to performance degradation in the middle of the day when VictoriaMetrics runs in time zones located too far from UTC. Thanks to @cnych for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574). * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): properly apply `alert_relabel_configs` relabeling rules to `-notifier.config` according to [these docs](https://docs.victoriametrics.com/vmalert.html#notifier-configuration-file). Thanks to @spectvtor for [the bugfix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2633). diff --git a/lib/storage/storage.go b/lib/storage/storage.go index d62425e5c..c45bbbdf6 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -1035,19 +1035,26 @@ func (s *Storage) mustSaveCache(c *workingsetcache.Cache, info, name string) { // saveCacheLock prevents from data races when multiple concurrent goroutines save the same cache. var saveCacheLock sync.Mutex +// SetRetentionTimezoneOffset sets the offset, which is used for calculating the time for indexdb rotation. +// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574 +func SetRetentionTimezoneOffset(offset time.Duration) { + retentionTimezoneOffsetMsecs = offset.Milliseconds() +} + +var retentionTimezoneOffsetMsecs int64 + func nextRetentionDuration(retentionMsecs int64) time.Duration { // Round retentionMsecs to days. This guarantees that per-day inverted index works as expected. retentionMsecs = ((retentionMsecs + msecPerDay - 1) / msecPerDay) * msecPerDay - now := time.Now() - t := now.UnixNano() / 1e6 + t := time.Now().UnixNano() / 1e6 deadline := ((t + retentionMsecs - 1) / retentionMsecs) * retentionMsecs // Schedule the deadline to +4 hours from the next retention period start. // This should prevent from possible double deletion of indexdb // due to time drift - see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/248 . deadline += 4 * 3600 * 1000 - // The effect of time zone on retention period is moved out - _, offset := now.Zone() - deadline -= int64(offset) * 1000 + // The effect of time zone on retention period is moved out. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2574 + deadline -= retentionTimezoneOffsetMsecs return time.Duration(deadline-t) * time.Millisecond }