Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 08:23:34 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/storage: tag filters sorting...

Browse Source
This commit is contained in:
Aliaksandr Valialkin 2021-02-17 17:55:29 +02:00
parent f5e841c1e9
commit bbc287ea6a
1 changed files with 5 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
lib/storage/index_db.go
View File

@ -25,7 +25,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache" "github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
"github.com/VictoriaMetrics/fastcache" "github.com/VictoriaMetrics/fastcache"
xxhash "github.com/cespare/xxhash/v2" xxhash "github.com/cespare/xxhash/v2"
"github.com/valyala/fastrand"
) )
const ( const (
@ -2811,36 +2810,18 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
lastQueryTimestamp uint64 lastQueryTimestamp uint64
} }
tfws := make([]tagFilterWithWeight, len(tfs.tfs)) tfws := make([]tagFilterWithWeight, len(tfs.tfs))
ct := fasttime.UnixTimestamp()
var tfwsEst []*tagFilterWithWeight
for i := range tfs.tfs { for i := range tfs.tfs {
tf := &tfs.tfs[i] tf := &tfs.tfs[i]
durationSeconds, lastQueryTimestamp := is.getDurationAndTimestampForDateFilter(date, tf) durationSeconds, lastQueryTimestamp := is.getDurationAndTimestampForDateFilter(date, tf)
if durationSeconds == 0 {
// Assume that unknown tag filters can take quite big amounts of time.
durationSeconds = 1.0
}
tfws[i] = tagFilterWithWeight{ tfws[i] = tagFilterWithWeight{
tf: tf, tf: tf,
durationSeconds: durationSeconds, durationSeconds: durationSeconds,
lastQueryTimestamp: lastQueryTimestamp, lastQueryTimestamp: lastQueryTimestamp,
} }
if ct > lastQueryTimestamp+5*60+uint64(fastrand.Uint32n(5*60)) {
// Re-estimate the time required for tf execution.
tfwsEst = append(tfwsEst, &tfws[i])
}
}
maxDateMetrics := maxMetrics * 50
if len(tfwsEst) > 0 {
sort.Slice(tfwsEst, func(i, j int) bool {
return tfwsEst[i].tf.Less(tfwsEst[j].tf)
})
// Allocate up to one second for the estimation of tf execution in order to reduce
// possible negative impact on query duration.
// It is OK if the deadline is exceeded during the estimation - the corresponding filters
// will be re-estimated after the lastQueryTimestamp expiration.
isEst := is.db.getIndexSearch(ct + 1)
for _, tfw := range tfwsEst {
_, _ = isEst.getMetricIDsForDateTagFilter(tfw.tf, 0, date, tfs.commonPrefix, maxDateMetrics)
tfw.durationSeconds, tfw.lastQueryTimestamp = is.getDurationAndTimestampForDateFilter(date, tfw.tf)
}
is.db.putIndexSearch(isEst)
} }
sort.Slice(tfws, func(i, j int) bool { sort.Slice(tfws, func(i, j int) bool {
a, b := &tfws[i], &tfws[j] a, b := &tfws[i], &tfws[j]
@ -2854,6 +2835,7 @@ func (is *indexSearch) getMetricIDsForDateAndFilters(date uint64, tfs *TagFilter
var tfsPostponed []*tagFilter var tfsPostponed []*tagFilter
var metricIDs *uint64set.Set var metricIDs *uint64set.Set
tfwsRemaining := tfws[:0] tfwsRemaining := tfws[:0]
maxDateMetrics := maxMetrics * 50
for i := range tfws { for i := range tfws {
tfw := tfws[i] tfw := tfws[i]
tf := tfw.tf tf := tfw.tf