From f9d86a913cbbd86261a6075821a7a6652afe031e Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 18 Oct 2024 01:11:23 +0200 Subject: [PATCH] lib/logstorage: do not persist streamIDCache, since it may go out of sync with partition directories, which can be changed manually between VictoriaLogs restarts Partition directories can be manually deleted and copied from another sources such as backups or other VitoriaLogs instances. In this case the persisted cache becomes out of sync with partitions. This can result in missing index entries during data ingestion or in incorrect results during querying. So it is better to do not persist caches. This shouldn't hurt VictoriaLogs performance just after the restart too much, since its caches usually contain small amounts of data, which can be quickly re-populated from the persisted data. (cherry picked from commit 8aa144fa74338cdc982b173040d23aee3b582298) --- lib/logstorage/filenames.go | 3 --- lib/logstorage/storage.go | 15 ++++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/logstorage/filenames.go b/lib/logstorage/filenames.go index 909286ba23..ff3458e971 100644 --- a/lib/logstorage/filenames.go +++ b/lib/logstorage/filenames.go @@ -17,10 +17,7 @@ const ( metadataFilename = "metadata.json" partsFilename = "parts.json" - streamIDCacheFilename = "stream_id.bin" - indexdbDirname = "indexdb" datadbDirname = "datadb" - cacheDirname = "cache" partitionsDirname = "partitions" ) diff --git a/lib/logstorage/storage.go b/lib/logstorage/storage.go index a448bacc4d..093658f4fb 100644 --- a/lib/logstorage/storage.go +++ b/lib/logstorage/storage.go @@ -244,9 +244,8 @@ func MustOpenStorage(path string, cfg *StorageConfig) *Storage { // Load caches mem := memory.Allowed() - streamIDCachePath := filepath.Join(path, cacheDirname, streamIDCacheFilename) - streamIDCache := workingsetcache.Load(streamIDCachePath, mem/16) + streamIDCache := workingsetcache.New(mem / 16) filterStreamCache := workingsetcache.New(mem / 10) s := &Storage{ @@ -457,11 +456,13 @@ func (s *Storage) MustClose() { s.partitions = nil s.ptwHot = nil - // Save caches - streamIDCachePath := filepath.Join(s.path, cacheDirname, streamIDCacheFilename) - if err := s.streamIDCache.Save(streamIDCachePath); err != nil { - logger.Panicf("FATAL: cannot save streamID cache to %q: %s", streamIDCachePath, err) - } + // Stop caches + + // Do not persist caches, since they may become out of sync with partitions + // if partitions are deleted, restored from backups or copied from other sources + // between VictoriaLogs restarts. This may result in various issues + // during data ingestion and querying. + s.streamIDCache.Stop() s.streamIDCache = nil