Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-18 06:30:27 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/logstorage: do not persist streamIDCache, since it may go out of sync with partition directories, which can be changed manually between VictoriaLogs restarts

Browse Source 
Partition directories can be manually deleted and copied from another sources such as backups or other VitoriaLogs instances.
In this case the persisted cache becomes out of sync with partitions. This can result in missing index entries
during data ingestion or in incorrect results during querying. So it is better to do not persist caches.
This shouldn't hurt VictoriaLogs performance just after the restart too much, since its caches usually contain
small amounts of data, which can be quickly re-populated from the persisted data.

(cherry picked from commit 8aa144fa74)
This commit is contained in:
Aliaksandr Valialkin 2024-10-18 01:11:23 +02:00 committed by hagen1778
parent b9fae4378a
commit f9d86a913c
No known key found for this signature in database
GPG Key ID: E92986095E0DD614
2 changed files with 8 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/logstorage/filenames.go
View File

@ -17,10 +17,7 @@ const (
metadataFilename = "metadata.json" metadataFilename = "metadata.json"
partsFilename = "parts.json" partsFilename = "parts.json"
streamIDCacheFilename = "stream_id.bin"
indexdbDirname = "indexdb" indexdbDirname = "indexdb"
datadbDirname = "datadb" datadbDirname = "datadb"
cacheDirname = "cache"
partitionsDirname = "partitions" partitionsDirname = "partitions"
) )

15
lib/logstorage/storage.go
View File

@ -244,9 +244,8 @@ func MustOpenStorage(path string, cfg *StorageConfig) *Storage {
// Load caches // Load caches
mem := memory.Allowed() mem := memory.Allowed()
streamIDCachePath := filepath.Join(path, cacheDirname, streamIDCacheFilename)
streamIDCache := workingsetcache.Load(streamIDCachePath, mem/16)
streamIDCache := workingsetcache.New(mem / 16)
filterStreamCache := workingsetcache.New(mem / 10) filterStreamCache := workingsetcache.New(mem / 10)
s := &Storage{ s := &Storage{
@ -457,11 +456,13 @@ func (s *Storage) MustClose() {
s.partitions = nil s.partitions = nil
s.ptwHot = nil s.ptwHot = nil
// Save caches // Stop caches
streamIDCachePath := filepath.Join(s.path, cacheDirname, streamIDCacheFilename)
if err := s.streamIDCache.Save(streamIDCachePath); err != nil { // Do not persist caches, since they may become out of sync with partitions
logger.Panicf("FATAL: cannot save streamID cache to %q: %s", streamIDCachePath, err) // if partitions are deleted, restored from backups or copied from other sources
} // between VictoriaLogs restarts. This may result in various issues
// during data ingestion and querying.
s.streamIDCache.Stop() s.streamIDCache.Stop()
s.streamIDCache = nil s.streamIDCache = nil