lib/storage: imrpove cache effectiveness for time series ids matching the given filters

Previously the maximum cache lifetime has been limited by 10 seconds. Now it is extended up to a day.
This should reduce CPU usage in the following cases:

* when querying recently added data with small churn rate for time series
* when querying historical data
This commit is contained in:
Aliaksandr Valialkin 2020-10-01 14:35:49 +03:00
parent 40df42e1e5
commit 3ad7566a87

View File

@ -462,12 +462,13 @@ func marshalTagFiltersKey(dst []byte, tfss []*TagFilters, tr TimeRange, versione
if versioned { if versioned {
prefix = atomic.LoadUint64(&tagFiltersKeyGen) prefix = atomic.LoadUint64(&tagFiltersKeyGen)
} }
const cacheGranularityMs = 1000 * 10 // Round start and end times to per-day granularity according to per-day inverted index.
startTime := (uint64(tr.MinTimestamp) / cacheGranularityMs) * cacheGranularityMs startDate := uint64(tr.MinTimestamp) / msecPerDay
endTime := (uint64(tr.MaxTimestamp) / cacheGranularityMs) * cacheGranularityMs endDate := uint64(tr.MaxTimestamp) / msecPerDay
dst = append(dst, tagFiltersKeyVersion)
dst = encoding.MarshalUint64(dst, prefix) dst = encoding.MarshalUint64(dst, prefix)
dst = encoding.MarshalUint64(dst, startTime) dst = encoding.MarshalUint64(dst, startDate)
dst = encoding.MarshalUint64(dst, endTime) dst = encoding.MarshalUint64(dst, endDate)
if len(tfss) == 0 { if len(tfss) == 0 {
return dst return dst
} }
@ -482,6 +483,18 @@ func marshalTagFiltersKey(dst []byte, tfss []*TagFilters, tr TimeRange, versione
return dst return dst
} }
// The version for tagCache key.
// Update it every time key generation scheme changes at marshalTagFiltersKey.
const tagFiltersKeyVersion = 1
func invalidateTagCache() {
// This function must be fast, since it is called each
// time new timeseries is added.
atomic.AddUint64(&tagFiltersKeyGen, 1)
}
var tagFiltersKeyGen uint64
func marshalTSIDs(dst []byte, tsids []TSID) []byte { func marshalTSIDs(dst []byte, tsids []TSID) []byte {
dst = encoding.MarshalUint64(dst, uint64(len(tsids))) dst = encoding.MarshalUint64(dst, uint64(len(tsids)))
for i := range tsids { for i := range tsids {
@ -514,14 +527,6 @@ func unmarshalTSIDs(dst []TSID, src []byte) ([]TSID, error) {
return dst, nil return dst, nil
} }
func invalidateTagCache() {
// This function must be fast, since it is called each
// time new timeseries is added.
atomic.AddUint64(&tagFiltersKeyGen, 1)
}
var tagFiltersKeyGen uint64
// getTSIDByNameNoCreate fills the dst with TSID for the given metricName. // getTSIDByNameNoCreate fills the dst with TSID for the given metricName.
// //
// It returns io.EOF if the given mn isn't found locally. // It returns io.EOF if the given mn isn't found locally.