From 29cebd82fbe7470b323c4f51cbea63fffa28629b Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Mon, 30 Oct 2023 10:52:57 +0100 Subject: [PATCH] lib/storage: log warning about RO mode only on state change (#5191) Before, vmstorage would log the same message each second producing excessive amount of logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5159 Signed-off-by: hagen1778 --- docs/CHANGELOG.md | 1 + lib/storage/storage.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bdd5081ef..5cd1aa04a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -71,6 +71,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly drop samples if `-streamAggr.dropInput` command-line flag is set and `-remoteWrite.streamAggr.config` contains an empty file. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5207). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): do not print redundant error logs when failed to scrape consul or nomad target. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5239). * BUGFIX: [vmstorage](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): prevent deleted series to be searchable via `/api/v1/series` API if they were re-ingested with staleness markers. This situation could happen if user deletes the series from the target and from VM, and then vmagent sends stale markers for absent series. Thanks to @ilyatrefilov for the [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069) and [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5174). +* BUGFIX: [vmstorage](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): log warning about switching to ReadOnly mode only on state change. Before, vmstorage would log this warning every 1s. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5159) for details. ## [v1.94.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.94.0) diff --git a/lib/storage/storage.go b/lib/storage/storage.go index a270808f7..5c4ebfd71 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -648,9 +648,12 @@ func (s *Storage) startFreeDiskSpaceWatcher() { f := func() { freeSpaceBytes := fs.MustGetFreeSpace(s.path) if freeSpaceBytes < freeDiskSpaceLimitBytes { + if atomic.LoadUint32(&s.isReadOnly) == 0 { + // log notification only on state change + logger.Warnf("switching the storage at %s to read-only mode, since it has less than -storage.minFreeDiskSpaceBytes=%d of free space: %d bytes left", + s.path, freeDiskSpaceLimitBytes, freeSpaceBytes) + } // Switch the storage to readonly mode if there is no enough free space left at s.path - logger.Warnf("switching the storage at %s to read-only mode, since it has less than -storage.minFreeDiskSpaceBytes=%d of free space: %d bytes left", - s.path, freeDiskSpaceLimitBytes, freeSpaceBytes) atomic.StoreUint32(&s.isReadOnly, 1) return }