mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-15 00:13:30 +01:00
120f3bc467
- Compare the actual free disk space to the value provided via -storage.minFreeDiskSpaceBytes directly inside the Storage.IsReadOnly(). This should work fast in most cases. This simplifies the logic at lib/storage. - Do not take into account -storage.minFreeDiskSpaceBytes during background merges, since it results in uncontrolled growth of small parts when the free disk space approaches -storage.minFreeDiskSpaceBytes. The background merge logic uses another mechanism for determining whether there is enough disk space for the merge - it reserves the needed disk space before the merge and releases it after the merge. This prevents from out of disk space errors during background merge. - Properly handle corner cases for flushing in-memory data to disk when the storage enters read-only mode. This is better than losing the in-memory data. - Return back Storage.MustAddRows() instead of Storage.AddRows(), since the only case when AddRows() can return error is when the storage is in read-only mode. This case must be handled by the caller by calling Storage.IsReadOnly() before adding rows to the storage. This simplifies the code a bit, since the caller of Storage.MustAddRows() shouldn't handle errors returned by Storage.AddRows(). - Properly store parsed logs to Storage if parts of the request contain invalid log lines. Previously the parsed logs could be lost in this case. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4737 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4945
103 lines
2.6 KiB
Go
103 lines
2.6 KiB
Go
package logstorage
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
|
)
|
|
|
|
func TestStorageLifecycle(_ *testing.T) {
|
|
const path = "TestStorageLifecycle"
|
|
|
|
for i := 0; i < 3; i++ {
|
|
cfg := &StorageConfig{}
|
|
s := MustOpenStorage(path, cfg)
|
|
s.MustClose()
|
|
}
|
|
fs.MustRemoveAll(path)
|
|
}
|
|
|
|
func TestStorageMustAddRows(t *testing.T) {
|
|
const path = "TestStorageMustAddRows"
|
|
|
|
var sStats StorageStats
|
|
|
|
cfg := &StorageConfig{}
|
|
s := MustOpenStorage(path, cfg)
|
|
|
|
// Try adding the same entry multiple times.
|
|
totalRowsCount := uint64(0)
|
|
for i := 0; i < 100; i++ {
|
|
lr := newTestLogRows(1, 1, 0)
|
|
lr.timestamps[0] = time.Now().UTC().UnixNano()
|
|
totalRowsCount += uint64(len(lr.timestamps))
|
|
s.MustAddRows(lr)
|
|
sStats.Reset()
|
|
s.UpdateStats(&sStats)
|
|
if n := sStats.RowsCount(); n != totalRowsCount {
|
|
t.Fatalf("unexpected number of entries in storage; got %d; want %d", n, totalRowsCount)
|
|
}
|
|
}
|
|
|
|
s.MustClose()
|
|
|
|
// Re-open the storage and try writing data to it
|
|
s = MustOpenStorage(path, cfg)
|
|
|
|
sStats.Reset()
|
|
s.UpdateStats(&sStats)
|
|
if n := sStats.RowsCount(); n != totalRowsCount {
|
|
t.Fatalf("unexpected number of entries in storage; got %d; want %d", n, totalRowsCount)
|
|
}
|
|
|
|
lr := newTestLogRows(3, 10, 0)
|
|
for i := range lr.timestamps {
|
|
lr.timestamps[i] = time.Now().UTC().UnixNano()
|
|
}
|
|
totalRowsCount += uint64(len(lr.timestamps))
|
|
s.MustAddRows(lr)
|
|
sStats.Reset()
|
|
s.UpdateStats(&sStats)
|
|
if n := sStats.RowsCount(); n != totalRowsCount {
|
|
t.Fatalf("unexpected number of entries in storage; got %d; want %d", n, totalRowsCount)
|
|
}
|
|
|
|
s.MustClose()
|
|
|
|
// Re-open the storage with big retention and try writing data
|
|
// to different days in the past and in the future
|
|
cfg = &StorageConfig{
|
|
Retention: 365 * 24 * time.Hour,
|
|
FutureRetention: 365 * 24 * time.Hour,
|
|
}
|
|
s = MustOpenStorage(path, cfg)
|
|
|
|
lr = newTestLogRows(3, 10, 0)
|
|
now := time.Now().UTC().UnixNano() - int64(len(lr.timestamps)/2)*nsecPerDay
|
|
for i := range lr.timestamps {
|
|
lr.timestamps[i] = now
|
|
now += nsecPerDay
|
|
}
|
|
totalRowsCount += uint64(len(lr.timestamps))
|
|
s.MustAddRows(lr)
|
|
sStats.Reset()
|
|
s.UpdateStats(&sStats)
|
|
if n := sStats.RowsCount(); n != totalRowsCount {
|
|
t.Fatalf("unexpected number of entries in storage; got %d; want %d", n, totalRowsCount)
|
|
}
|
|
|
|
s.MustClose()
|
|
|
|
// Make sure the stats is valid after re-opening the storage
|
|
s = MustOpenStorage(path, cfg)
|
|
sStats.Reset()
|
|
s.UpdateStats(&sStats)
|
|
if n := sStats.RowsCount(); n != totalRowsCount {
|
|
t.Fatalf("unexpected number of entries in storage; got %d; want %d", n, totalRowsCount)
|
|
}
|
|
s.MustClose()
|
|
|
|
fs.MustRemoveAll(path)
|
|
}
|