diff --git a/README.md b/README.md index 0a6a6d0ef..a2738ccbc 100644 --- a/README.md +++ b/README.md @@ -2176,7 +2176,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li ``` -bigMergeConcurrency int - The maximum number of CPU cores to use for big merges. Default value is used if set to 0 + Deprecated: this flag does nothing. Please use -smallMergeConcurrency for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey string @@ -2526,7 +2526,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -selfScrapeJob string Value for 'job' label, which is added to self-scraped metrics (default "victoria-metrics") -smallMergeConcurrency int - The maximum number of CPU cores to use for small merges. Default value is used if set to 0 + The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries -snapshotAuthKey string authKey, which must be passed in query string to /snapshot* pages -snapshotCreateTimeout duration diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index 948ce2c86..2fa08e3d6 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -38,8 +38,10 @@ var ( finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+ "Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+ "Zero value disables final merge") - bigMergeConcurrency = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0") - smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0") + _ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing. Please use -smallMergeConcurrency "+ + "for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage") + smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . "+ + "It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries") retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+ "If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+ "If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)") @@ -92,7 +94,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) { resetResponseCacheIfNeeded = resetCacheIfNeeded storage.SetLogNewSeries(*logNewSeries) storage.SetFinalMergeDelay(*finalMergeDelay) - storage.SetBigMergeWorkersCount(*bigMergeConcurrency) storage.SetMergeWorkersCount(*smallMergeConcurrency) storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset) storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 89327173b..938585855 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): store backup creation and completion time in `backup_complete.ignore` file of backup contents. This is useful to determine point in time when backup was created and completed. * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details. +* FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case. ## [v1.90.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.90.0) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index cca5a7327..bf722d8fd 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -1235,7 +1235,7 @@ Below is the output for `/path/to/vmstorage -help`: ``` -bigMergeConcurrency int - The maximum number of CPU cores to use for big merges. Default value is used if set to 0 + Deprecated: this flag does nothing. Please use -smallMergeConcurrency for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -cluster.tls @@ -1370,7 +1370,7 @@ Below is the output for `/path/to/vmstorage -help`: -search.maxUniqueTimeseries int The maximum number of unique time series, which can be scanned during every query. This allows protecting against heavy queries, which select unexpectedly high number of series. Zero means 'no limit'. See also -search.max* command-line flags at vmselect -smallMergeConcurrency int - The maximum number of CPU cores to use for small merges. Default value is used if set to 0 + The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries -snapshotAuthKey string authKey, which must be passed in query string to /snapshot* pages -snapshotCreateTimeout duration diff --git a/docs/README.md b/docs/README.md index 0b5517b6e..e83b65020 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2177,7 +2177,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li ``` -bigMergeConcurrency int - The maximum number of CPU cores to use for big merges. Default value is used if set to 0 + Deprecated: this flag does nothing. Please use -smallMergeConcurrency for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey string @@ -2527,7 +2527,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -selfScrapeJob string Value for 'job' label, which is added to self-scraped metrics (default "victoria-metrics") -smallMergeConcurrency int - The maximum number of CPU cores to use for small merges. Default value is used if set to 0 + The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries -snapshotAuthKey string authKey, which must be passed in query string to /snapshot* pages -snapshotCreateTimeout duration diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index a49688744..6c062c948 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2180,7 +2180,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li ``` -bigMergeConcurrency int - The maximum number of CPU cores to use for big merges. Default value is used if set to 0 + Deprecated: this flag does nothing. Please use -smallMergeConcurrency for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage -cacheExpireDuration duration Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s) -configAuthKey string @@ -2530,7 +2530,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -selfScrapeJob string Value for 'job' label, which is added to self-scraped metrics (default "victoria-metrics") -smallMergeConcurrency int - The maximum number of CPU cores to use for small merges. Default value is used if set to 0 + The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries -snapshotAuthKey string authKey, which must be passed in query string to /snapshot* pages -snapshotCreateTimeout duration diff --git a/lib/storage/partition.go b/lib/storage/partition.go index 709f8fc7f..af61a5956 100644 --- a/lib/storage/partition.go +++ b/lib/storage/partition.go @@ -1002,8 +1002,6 @@ func hasActiveMerges(pws []*partWrapper) bool { var mergeWorkersLimitCh = make(chan struct{}, adjustMergeWorkersLimit(getDefaultMergeConcurrency(16))) -var bigMergeWorkersLimitCh = make(chan struct{}, getDefaultMergeConcurrency(4)) - func getDefaultMergeConcurrency(max int) int { v := (cgroup.AvailableCPUs() + 1) / 2 if v > max { @@ -1012,17 +1010,6 @@ func getDefaultMergeConcurrency(max int) int { return v } -// SetBigMergeWorkersCount sets the maximum number of concurrent mergers for big blocks. -// -// The function must be called before opening or creating any storage. -func SetBigMergeWorkersCount(n int) { - if n <= 0 { - // Do nothing - return - } - bigMergeWorkersLimitCh = make(chan struct{}, n) -} - // SetMergeWorkersCount sets the maximum number of concurrent mergers for parts. // // The function must be called before opening or creating any storage. @@ -1143,7 +1130,8 @@ func (pt *partition) getMaxSmallPartSize() uint64 { } func (pt *partition) getMaxBigPartSize() uint64 { - return getMaxOutBytes(pt.bigPartsPath, cap(bigMergeWorkersLimitCh)) + workersCount := getDefaultMergeConcurrency(4) + return getMaxOutBytes(pt.bigPartsPath, workersCount) } func getMaxOutBytes(path string, workersCount int) uint64 { @@ -1277,13 +1265,6 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}, isFi mergeIdx := pt.nextMergeIdx() dstPartPath := pt.getDstPartPath(dstPartType, mergeIdx) - if dstPartType == partBig { - bigMergeWorkersLimitCh <- struct{}{} - defer func() { - <-bigMergeWorkersLimitCh - }() - } - if !isDedupEnabled() && isFinal && len(pws) == 1 && pws[0].mp != nil { // Fast path: flush a single in-memory part to disk. mp := pws[0].mp