Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 08:23:34 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

app/vmselect/netstorage: reduce the number of memory allocations in ProcessSearchQuery() by storing all the metric names in a single byte slice

Browse Source 
This reduces the number of memory allocations at the cost of possible memory usage increase,
since now different metric name strings may hold references to the previous byte slice.
This is good tradeoff, since ProcessSearchQuery is called in vmselect, and vmselect isn't usually limited by memory.

This change has been extracted from https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5527
This commit is contained in:
Aliaksandr Valialkin 2024-01-22 19:13:39 +02:00
parent 60ef978ffc
commit 2ab9a75cca
No known key found for this signature in database
GPG Key ID: 52C003EE2BCDB9EB
1 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/vmselect/netstorage/netstorage.go
View File

@ -1351,6 +1351,11 @@ type tmpBlocksFileWrapper struct {
tbfs []*tmpBlocksFile
ms []map[string]*blockAddrs
orderedMetricNamess [][]string
// metricNamesBufs are per-worker bufs for holding all the loaded unique metric names.
// It should reduce pressure on Go garbage collector by reducing
// the number of memory allocations when constructing metricName string from byte slice.
metricNamesBufs [][]byte
}
type blockAddrs struct {
@ -1378,6 +1383,7 @@ func newTmpBlocksFileWrapper(sns []*storageNode) *tmpBlocksFileWrapper {
tbfs: tbfs,
ms: ms,
orderedMetricNamess: make([][]string, n),
metricNamesBufs: make([][]byte, n),
}
}
@ -1389,8 +1395,6 @@ func (tbfw *tmpBlocksFileWrapper) RegisterAndWriteBlock(mb *storage.MetricBlock,
if err != nil {
return err
}
// Do not intern mb.MetricName, since it leads to increased memory usage.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
metricName := mb.MetricName
m := tbfw.ms[workerID]
addrs := m[string(metricName)]
@ -1399,13 +1403,17 @@ func (tbfw *tmpBlocksFileWrapper) RegisterAndWriteBlock(mb *storage.MetricBlock,
}
addrs.addrs = append(addrs.addrs, addr)
if len(addrs.addrs) == 1 {
// An optimization for big number of time series with long names: store only a single copy of metricNameStr
// in both tbfw.orderedMetricNamess and tbfw.ms.
metricNamesBuf := tbfw.metricNamesBufs[workerID]
metricNamesBufLen := len(metricNamesBuf)
metricNamesBuf = append(metricNamesBuf, metricName...)
metricNameStr := bytesutil.ToUnsafeString(metricNamesBuf[metricNamesBufLen:])
orderedMetricNames := tbfw.orderedMetricNamess[workerID]
metricNameStr := string(metricName)
orderedMetricNames = append(orderedMetricNames, metricNameStr)
m[metricNameStr] = addrs
tbfw.orderedMetricNamess[workerID] = orderedMetricNames
tbfw.metricNamesBufs[workerID] = metricNamesBuf
}
return nil
}