From 75dd7b30ba4705d34597c50e77bdd3a7db968565 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 4 Oct 2023 16:18:40 +0200 Subject: [PATCH] lib/filestream: add `-filestream.disableFadvise` syscall for unconditional disabling of `fadvise` syscall This may be needed in rare cases when performing backups on systems with big number of CPU cores and big value passed to -concurrency command-line flag. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5120 --- README.md | 2 ++ app/vmbackup/README.md | 2 ++ docs/CHANGELOG.md | 1 + docs/Cluster-VictoriaMetrics.md | 2 ++ docs/README.md | 2 ++ docs/Single-server-VictoriaMetrics.md | 2 ++ docs/vmbackup.md | 2 ++ lib/filestream/filestream.go | 10 ++++++++++ 8 files changed, 23 insertions(+) diff --git a/README.md b/README.md index 3752a2adf..9aefaeb7d 100644 --- a/README.md +++ b/README.md @@ -2373,6 +2373,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -flagsAuthKey string diff --git a/app/vmbackup/README.md b/app/vmbackup/README.md index 340c99d6b..8b93abdf9 100644 --- a/app/vmbackup/README.md +++ b/app/vmbackup/README.md @@ -292,6 +292,8 @@ Run `vmbackup -help` in order to see all the available options: Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -flagsAuthKey string Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -fs.disableMmap diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bcb4c6218..f56af1602 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,7 @@ The sandbox cluster installation is running under the constant load generated by ## tip +* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): add `-filestream.disableFadvise` command-line flag, which can be used for disabling `fadvise` syscall during backup upload to the remote storage. By default `vmbackup` uses `fadvise` syscall in order to prevent from eviction of recently accessed data from the [OS page cache](https://en.wikipedia.org/wiki/Page_cache) when backing up large files. Sometimes the `fadvise` syscall may take significant amounts of CPU when the backup is performed with large value of `-concurrency` command-line flag on systems with big number of CPU cores. In this case it is better to manually disable `fadvise` syscall by passing `-filestream.disableFadvise` command-line flag to `vmbackup`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5120) for details. * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): account for `vmauth` component for alerts `ServiceDown` and `TooManyRestarts`. ## [v1.94.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.94.0) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index eb2a3d5d9..2d768dbce 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -1304,6 +1304,8 @@ Below is the output for `/path/to/vmstorage -help`: Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -flagsAuthKey string diff --git a/docs/README.md b/docs/README.md index 5afa8bbbc..2c68d2e86 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2376,6 +2376,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -flagsAuthKey string diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 495fc34f8..a5f36202b 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2384,6 +2384,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -finalMergeDelay duration The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge -flagsAuthKey string diff --git a/docs/vmbackup.md b/docs/vmbackup.md index bff38459a..93ec5e22e 100644 --- a/docs/vmbackup.md +++ b/docs/vmbackup.md @@ -303,6 +303,8 @@ Run `vmbackup -help` in order to see all the available options: Prefix for environment variables if -envflag.enable is set -eula Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html + -filestream.disableFadvise + Whether to disable fadvise() syscall when reading large data files. The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. In some rare cases it is better to disable the syscall if it uses too much CPU -flagsAuthKey string Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings -fs.disableMmap diff --git a/lib/filestream/filestream.go b/lib/filestream/filestream.go index f156fb0d8..339f72855 100644 --- a/lib/filestream/filestream.go +++ b/lib/filestream/filestream.go @@ -2,6 +2,7 @@ package filestream import ( "bufio" + "flag" "fmt" "io" "os" @@ -13,6 +14,10 @@ import ( "github.com/VictoriaMetrics/metrics" ) +var disableFadvise = flag.Bool("filestream.disableFadvise", false, "Whether to disable fadvise() syscall when reading large data files. "+ + "The fadvise() syscall prevents from eviction of recently accessed data from OS page cache during background merges and backups. "+ + "In some rare cases it is better to disable the syscall if it uses too much CPU") + const dontNeedBlockSize = 16 * 1024 * 1024 // ReadCloser is a standard interface for filestream Reader. @@ -89,6 +94,11 @@ func MustOpen(path string, nocache bool) *Reader { f: f, br: getBufioReader(f), } + if *disableFadvise { + // Unconditionally disable fadvise() syscall + // See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5120 for details on why this is needed + nocache = false + } if nocache { r.st.fd = f.Fd() }