Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-23 20:37:12 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

app/vmselect/netstorage: reduce the number of memory allocations in ProcessSearchQuery() by storing all the metric names in a single byte slice

Browse Source 
This reduces the number of memory allocations at the cost of possible memory usage increase,
since now different metric name strings may hold references to the previous byte slice.
This is good tradeoff, since ProcessSearchQuery is called in vmselect, and vmselect isn't usually limited by memory.

This change has been extracted from https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5527
This commit is contained in:
Aliaksandr Valialkin 2024-01-22 19:13:39 +02:00
parent ffaf48b99e
commit 508c608062
No known key found for this signature in database
GPG Key ID: 52C003EE2BCDB9EB
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/vmselect/netstorage/netstorage.go
View File

@ -1178,6 +1178,12 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
samples := 0
tbf := getTmpBlocksFile()
var buf []byte
// metricNamesBuf is used for holding all the loaded unique metric names.
// It should reduce pressure on Go garbage collector by reducing
// the number of memory allocations when constructing metricName string from byte slice.
var metricNamesBuf []byte
for sr.NextMetricBlock() {
blocksRead++
if deadline.Exceeded() {
@ -1199,8 +1205,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
putStorageSearch(sr)
return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
}
// Do not intern mb.MetricName, since it leads to increased memory usage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
metricName := sr.MetricBlockRef.MetricName
brs := m[string(metricName)]
if brs == nil {
@ -1212,7 +1216,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
addr: addr,
})
if len(brs.brs) == 1 {
metricNameStr := string(metricName)
metricNamesBufLen := len(metricNamesBuf)
metricNamesBuf = append(metricNamesBuf, metricName...)
metricNameStr := bytesutil.ToUnsafeString(metricNamesBuf[metricNamesBufLen:])
orderedMetricNames = append(orderedMetricNames, metricNameStr)
m[metricNameStr] = brs
}