2019-05-22 23:16:55 +02:00
package netstorage
import (
"container/heap"
2020-08-10 12:17:12 +02:00
"errors"
2019-05-22 23:16:55 +02:00
"flag"
"fmt"
"runtime"
"sort"
"sync"
2020-09-26 03:29:45 +02:00
"sync/atomic"
2019-05-22 23:16:55 +02:00
2020-09-11 12:18:57 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
2020-06-24 18:36:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
2020-07-22 13:53:54 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/metrics"
)
var (
2020-09-10 23:28:19 +02:00
maxTagKeysPerSearch = flag . Int ( "search.maxTagKeys" , 100e3 , "The maximum number of tag keys returned from /api/v1/labels" )
maxTagValuesPerSearch = flag . Int ( "search.maxTagValues" , 100e3 , "The maximum number of tag values returned from /api/v1/label/<label_name>/values" )
maxTagValueSuffixesPerSearch = flag . Int ( "search.maxTagValueSuffixesPerSearch" , 100e3 , "The maximum number of tag value suffixes returned from /metrics/find" )
maxMetricsPerSearch = flag . Int ( "search.maxUniqueTimeseries" , 300e3 , "The maximum number of unique time series each search can scan" )
2019-05-22 23:16:55 +02:00
)
// Result is a single timeseries result.
//
// ProcessSearchQuery returns Result slice.
type Result struct {
// The name of the metric.
MetricName storage . MetricName
// Values are sorted by Timestamps.
Values [ ] float64
Timestamps [ ] int64
// Marshaled MetricName. Used only for results sorting
// in app/vmselect/promql
MetricNameMarshaled [ ] byte
}
func ( r * Result ) reset ( ) {
r . MetricName . Reset ( )
r . Values = r . Values [ : 0 ]
r . Timestamps = r . Timestamps [ : 0 ]
r . MetricNameMarshaled = r . MetricNameMarshaled [ : 0 ]
}
// Results holds results returned from ProcessSearchQuery.
type Results struct {
2019-08-04 21:15:33 +02:00
tr storage . TimeRange
fetchData bool
2020-09-11 12:18:57 +02:00
deadline searchutils . Deadline
2019-05-22 23:16:55 +02:00
packedTimeseries [ ] packedTimeseries
2020-04-27 07:13:41 +02:00
sr * storage . Search
2020-11-04 15:46:10 +01:00
tbf * tmpBlocksFile
2019-05-22 23:16:55 +02:00
}
// Len returns the number of results in rss.
func ( rss * Results ) Len ( ) int {
return len ( rss . packedTimeseries )
}
// Cancel cancels rss work.
func ( rss * Results ) Cancel ( ) {
2020-04-27 07:13:41 +02:00
rss . mustClose ( )
}
func ( rss * Results ) mustClose ( ) {
putStorageSearch ( rss . sr )
rss . sr = nil
2020-11-04 15:46:10 +01:00
putTmpBlocksFile ( rss . tbf )
rss . tbf = nil
2019-05-22 23:16:55 +02:00
}
2020-08-06 16:22:10 +02:00
var timeseriesWorkCh = make ( chan * timeseriesWork , gomaxprocs * 16 )
2020-06-23 19:29:19 +02:00
type timeseriesWork struct {
2020-09-27 22:17:14 +02:00
mustStop uint64
rss * Results
pts * packedTimeseries
f func ( rs * Result , workerID uint ) error
doneCh chan error
2020-06-23 19:29:19 +02:00
rowsProcessed int
}
func init ( ) {
for i := 0 ; i < gomaxprocs ; i ++ {
go timeseriesWorker ( uint ( i ) )
}
}
func timeseriesWorker ( workerID uint ) {
var rs Result
2020-06-24 18:36:55 +02:00
var rsLastResetTime uint64
2020-06-23 19:29:19 +02:00
for tsw := range timeseriesWorkCh {
rss := tsw . rss
2020-07-21 17:34:59 +02:00
if rss . deadline . Exceeded ( ) {
2020-06-23 19:29:19 +02:00
tsw . doneCh <- fmt . Errorf ( "timeout exceeded during query execution: %s" , rss . deadline . String ( ) )
continue
}
2020-09-27 22:17:14 +02:00
if atomic . LoadUint64 ( & tsw . mustStop ) != 0 {
tsw . doneCh <- nil
continue
}
2020-11-04 15:46:10 +01:00
if err := tsw . pts . Unpack ( & rs , rss . tbf , rss . tr , rss . fetchData ) ; err != nil {
2020-06-30 21:58:18 +02:00
tsw . doneCh <- fmt . Errorf ( "error during time series unpacking: %w" , err )
2020-06-23 19:29:19 +02:00
continue
}
if len ( rs . Timestamps ) > 0 || ! rss . fetchData {
2020-09-27 22:17:14 +02:00
if err := tsw . f ( & rs , workerID ) ; err != nil {
tsw . doneCh <- err
continue
}
2020-06-23 19:29:19 +02:00
}
tsw . rowsProcessed = len ( rs . Values )
tsw . doneCh <- nil
2020-06-24 18:36:55 +02:00
currentTime := fasttime . UnixTimestamp ( )
if cap ( rs . Values ) > 1024 * 1024 && 4 * len ( rs . Values ) < cap ( rs . Values ) && currentTime - rsLastResetTime > 10 {
2020-06-23 19:29:19 +02:00
// Reset rs in order to preseve memory usage after processing big time series with millions of rows.
rs = Result { }
2020-06-24 18:36:55 +02:00
rsLastResetTime = currentTime
2020-06-23 19:29:19 +02:00
}
}
}
2020-07-23 18:21:49 +02:00
// RunParallel runs f in parallel for all the results from rss.
2019-05-22 23:16:55 +02:00
//
// f shouldn't hold references to rs after returning.
2019-07-12 14:51:02 +02:00
// workerID is the id of the worker goroutine that calls f.
2020-09-27 22:17:14 +02:00
// Data processing is immediately stopped if f returns non-nil error.
2019-05-22 23:16:55 +02:00
//
// rss becomes unusable after the call to RunParallel.
2020-09-27 22:17:14 +02:00
func ( rss * Results ) RunParallel ( f func ( rs * Result , workerID uint ) error ) error {
2020-04-27 07:13:41 +02:00
defer rss . mustClose ( )
2019-05-22 23:16:55 +02:00
// Feed workers with work.
2020-06-23 19:29:19 +02:00
tsws := make ( [ ] * timeseriesWork , len ( rss . packedTimeseries ) )
2019-05-22 23:16:55 +02:00
for i := range rss . packedTimeseries {
2020-06-23 19:29:19 +02:00
tsw := & timeseriesWork {
rss : rss ,
pts : & rss . packedTimeseries [ i ] ,
f : f ,
doneCh : make ( chan error , 1 ) ,
}
timeseriesWorkCh <- tsw
tsws [ i ] = tsw
2019-05-22 23:16:55 +02:00
}
2019-11-23 12:22:55 +01:00
seriesProcessedTotal := len ( rss . packedTimeseries )
2019-05-22 23:16:55 +02:00
rss . packedTimeseries = rss . packedTimeseries [ : 0 ]
2020-06-23 19:29:19 +02:00
// Wait until work is complete.
var firstErr error
rowsProcessedTotal := 0
for _ , tsw := range tsws {
if err := <- tsw . doneCh ; err != nil && firstErr == nil {
// Return just the first error, since other errors
// are likely duplicate the first error.
firstErr = err
2020-09-27 22:17:14 +02:00
// Notify all the the tsws that they shouldn't be executed.
for _ , tsw := range tsws {
atomic . StoreUint64 ( & tsw . mustStop , 1 )
}
2019-05-22 23:16:55 +02:00
}
2020-06-23 19:29:19 +02:00
rowsProcessedTotal += tsw . rowsProcessed
2019-05-22 23:16:55 +02:00
}
2020-06-23 19:29:19 +02:00
2019-11-23 12:22:55 +01:00
perQueryRowsProcessed . Update ( float64 ( rowsProcessedTotal ) )
perQuerySeriesProcessed . Update ( float64 ( seriesProcessedTotal ) )
2020-06-23 19:29:19 +02:00
return firstErr
2019-05-22 23:16:55 +02:00
}
2019-11-23 12:22:55 +01:00
var perQueryRowsProcessed = metrics . NewHistogram ( ` vm_per_query_rows_processed_count ` )
var perQuerySeriesProcessed = metrics . NewHistogram ( ` vm_per_query_series_processed_count ` )
2019-05-22 23:16:55 +02:00
var gomaxprocs = runtime . GOMAXPROCS ( - 1 )
type packedTimeseries struct {
metricName string
2020-11-04 15:46:10 +01:00
brs [ ] blockRef
2019-05-22 23:16:55 +02:00
}
2020-08-06 16:22:10 +02:00
var unpackWorkCh = make ( chan * unpackWork , gomaxprocs * 128 )
2019-05-22 23:16:55 +02:00
2020-08-06 16:42:15 +02:00
type unpackWorkItem struct {
2020-11-04 15:46:10 +01:00
br blockRef
2020-08-06 16:42:15 +02:00
tr storage . TimeRange
}
2020-06-23 19:29:19 +02:00
type unpackWork struct {
2020-11-04 15:46:10 +01:00
tbf * tmpBlocksFile
2020-09-24 19:16:19 +02:00
ws [ ] unpackWorkItem
sbs [ ] * sortBlock
doneCh chan error
2020-06-23 19:29:19 +02:00
}
2019-05-22 23:16:55 +02:00
2020-07-22 13:53:54 +02:00
func ( upw * unpackWork ) reset ( ) {
2020-11-04 15:46:10 +01:00
upw . tbf = nil
2020-08-06 16:42:15 +02:00
ws := upw . ws
for i := range ws {
w := & ws [ i ]
2020-11-04 15:46:10 +01:00
w . br = blockRef { }
2020-08-06 16:42:15 +02:00
w . tr = storage . TimeRange { }
}
upw . ws = upw . ws [ : 0 ]
sbs := upw . sbs
for i := range sbs {
sbs [ i ] = nil
}
upw . sbs = upw . sbs [ : 0 ]
2020-07-22 13:53:54 +02:00
if n := len ( upw . doneCh ) ; n > 0 {
logger . Panicf ( "BUG: upw.doneCh must be empty; it contains %d items now" , n )
}
}
2020-09-15 20:06:04 +02:00
func ( upw * unpackWork ) unpack ( tmpBlock * storage . Block ) {
2020-08-06 16:42:15 +02:00
for _ , w := range upw . ws {
sb := getSortBlock ( )
2020-11-04 15:46:10 +01:00
if err := sb . unpackFrom ( tmpBlock , upw . tbf , w . br , w . tr ) ; err != nil {
2020-08-06 16:42:15 +02:00
putSortBlock ( sb )
upw . doneCh <- fmt . Errorf ( "cannot unpack block: %w" , err )
return
}
upw . sbs = append ( upw . sbs , sb )
}
upw . doneCh <- nil
}
2020-07-22 13:53:54 +02:00
func getUnpackWork ( ) * unpackWork {
v := unpackWorkPool . Get ( )
if v != nil {
return v . ( * unpackWork )
}
return & unpackWork {
doneCh : make ( chan error , 1 ) ,
}
}
func putUnpackWork ( upw * unpackWork ) {
upw . reset ( )
unpackWorkPool . Put ( upw )
}
var unpackWorkPool sync . Pool
2020-06-23 19:29:19 +02:00
func init ( ) {
for i := 0 ; i < gomaxprocs ; i ++ {
go unpackWorker ( )
2019-05-22 23:16:55 +02:00
}
2020-06-23 19:29:19 +02:00
}
func unpackWorker ( ) {
2020-09-15 20:06:04 +02:00
var tmpBlock storage . Block
2020-06-23 19:29:19 +02:00
for upw := range unpackWorkCh {
2020-09-15 20:06:04 +02:00
upw . unpack ( & tmpBlock )
2019-05-22 23:16:55 +02:00
}
2020-06-23 19:29:19 +02:00
}
2019-05-22 23:16:55 +02:00
2020-08-06 16:42:15 +02:00
// unpackBatchSize is the maximum number of blocks that may be unpacked at once by a single goroutine.
//
// This batch is needed in order to reduce contention for upackWorkCh in multi-CPU system.
2020-08-10 14:15:37 +02:00
var unpackBatchSize = 8 * runtime . GOMAXPROCS ( - 1 )
2020-08-06 16:42:15 +02:00
2020-06-23 19:29:19 +02:00
// Unpack unpacks pts to dst.
2020-11-04 15:46:10 +01:00
func ( pts * packedTimeseries ) Unpack ( dst * Result , tbf * tmpBlocksFile , tr storage . TimeRange , fetchData bool ) error {
2020-06-23 19:29:19 +02:00
dst . reset ( )
if err := dst . MetricName . Unmarshal ( bytesutil . ToUnsafeBytes ( pts . metricName ) ) ; err != nil {
2020-06-30 21:58:18 +02:00
return fmt . Errorf ( "cannot unmarshal metricName %q: %w" , pts . metricName , err )
2019-05-22 23:16:55 +02:00
}
2020-09-24 19:16:19 +02:00
if ! fetchData {
// Do not spend resources on data reading and unpacking.
return nil
}
2019-05-22 23:16:55 +02:00
// Feed workers with work
2020-09-22 22:49:47 +02:00
brsLen := len ( pts . brs )
upws := make ( [ ] * unpackWork , 0 , 1 + brsLen / unpackBatchSize )
2020-08-06 16:42:15 +02:00
upw := getUnpackWork ( )
2020-11-04 15:46:10 +01:00
upw . tbf = tbf
2020-08-06 16:42:15 +02:00
for _ , br := range pts . brs {
if len ( upw . ws ) >= unpackBatchSize {
unpackWorkCh <- upw
upws = append ( upws , upw )
upw = getUnpackWork ( )
2020-11-04 15:46:10 +01:00
upw . tbf = tbf
2020-08-06 16:42:15 +02:00
}
upw . ws = append ( upw . ws , unpackWorkItem {
br : br ,
tr : tr ,
} )
2019-05-22 23:16:55 +02:00
}
2020-08-06 16:42:15 +02:00
unpackWorkCh <- upw
upws = append ( upws , upw )
2020-04-27 07:13:41 +02:00
pts . brs = pts . brs [ : 0 ]
2019-05-22 23:16:55 +02:00
2020-06-23 19:29:19 +02:00
// Wait until work is complete
2020-09-22 22:49:47 +02:00
sbs := make ( [ ] * sortBlock , 0 , brsLen )
2020-06-23 19:29:19 +02:00
var firstErr error
for _ , upw := range upws {
if err := <- upw . doneCh ; err != nil && firstErr == nil {
// Return the first error only, since other errors are likely the same.
firstErr = err
}
if firstErr == nil {
2020-08-06 16:42:15 +02:00
sbs = append ( sbs , upw . sbs ... )
} else {
for _ , sb := range upw . sbs {
putSortBlock ( sb )
}
2019-05-22 23:16:55 +02:00
}
2020-07-22 13:53:54 +02:00
putUnpackWork ( upw )
2019-05-22 23:16:55 +02:00
}
2020-06-23 19:29:19 +02:00
if firstErr != nil {
return firstErr
2019-05-22 23:16:55 +02:00
}
mergeSortBlocks ( dst , sbs )
return nil
}
func getSortBlock ( ) * sortBlock {
v := sbPool . Get ( )
if v == nil {
return & sortBlock { }
}
return v . ( * sortBlock )
}
func putSortBlock ( sb * sortBlock ) {
sb . reset ( )
sbPool . Put ( sb )
}
var sbPool sync . Pool
var metricRowsSkipped = metrics . NewCounter ( ` vm_metric_rows_skipped_total { name="vmselect"} ` )
func mergeSortBlocks ( dst * Result , sbh sortBlocksHeap ) {
// Skip empty sort blocks, since they cannot be passed to heap.Init.
src := sbh
sbh = sbh [ : 0 ]
for _ , sb := range src {
if len ( sb . Timestamps ) == 0 {
putSortBlock ( sb )
continue
}
sbh = append ( sbh , sb )
}
if len ( sbh ) == 0 {
return
}
heap . Init ( & sbh )
for {
top := sbh [ 0 ]
heap . Pop ( & sbh )
if len ( sbh ) == 0 {
dst . Timestamps = append ( dst . Timestamps , top . Timestamps [ top . NextIdx : ] ... )
dst . Values = append ( dst . Values , top . Values [ top . NextIdx : ] ... )
putSortBlock ( top )
2020-01-31 00:09:44 +01:00
break
2019-05-22 23:16:55 +02:00
}
sbNext := sbh [ 0 ]
tsNext := sbNext . Timestamps [ sbNext . NextIdx ]
idxNext := len ( top . Timestamps )
if top . Timestamps [ idxNext - 1 ] > tsNext {
idxNext = top . NextIdx
for top . Timestamps [ idxNext ] <= tsNext {
idxNext ++
}
}
dst . Timestamps = append ( dst . Timestamps , top . Timestamps [ top . NextIdx : idxNext ] ... )
dst . Values = append ( dst . Values , top . Values [ top . NextIdx : idxNext ] ... )
if idxNext < len ( top . Timestamps ) {
top . NextIdx = idxNext
heap . Push ( & sbh , top )
} else {
// Return top to the pool.
putSortBlock ( top )
}
}
2020-01-31 00:09:44 +01:00
2020-02-27 22:47:05 +01:00
timestamps , values := storage . DeduplicateSamples ( dst . Timestamps , dst . Values )
dedups := len ( dst . Timestamps ) - len ( timestamps )
dedupsDuringSelect . Add ( dedups )
dst . Timestamps = timestamps
dst . Values = values
2019-05-22 23:16:55 +02:00
}
2020-02-27 22:47:05 +01:00
var dedupsDuringSelect = metrics . NewCounter ( ` vm_deduplicated_samples_total { type="select"} ` )
2019-05-22 23:16:55 +02:00
type sortBlock struct {
Timestamps [ ] int64
Values [ ] float64
NextIdx int
}
func ( sb * sortBlock ) reset ( ) {
sb . Timestamps = sb . Timestamps [ : 0 ]
sb . Values = sb . Values [ : 0 ]
sb . NextIdx = 0
}
2020-11-04 15:46:10 +01:00
func ( sb * sortBlock ) unpackFrom ( tmpBlock * storage . Block , tbf * tmpBlocksFile , br blockRef , tr storage . TimeRange ) error {
2020-09-15 20:06:04 +02:00
tmpBlock . Reset ( )
2020-11-04 15:46:10 +01:00
brReal := tbf . MustReadBlockRefAt ( br . partRef , br . addr )
brReal . MustReadBlock ( tmpBlock , true )
2020-09-24 19:16:19 +02:00
if err := tmpBlock . UnmarshalData ( ) ; err != nil {
return fmt . Errorf ( "cannot unmarshal block: %w" , err )
2019-05-22 23:16:55 +02:00
}
2020-09-26 03:29:45 +02:00
sb . Timestamps , sb . Values = tmpBlock . AppendRowsWithTimeRangeFilter ( sb . Timestamps [ : 0 ] , sb . Values [ : 0 ] , tr )
skippedRows := tmpBlock . RowsCount ( ) - len ( sb . Timestamps )
2019-05-22 23:16:55 +02:00
metricRowsSkipped . Add ( skippedRows )
return nil
}
type sortBlocksHeap [ ] * sortBlock
func ( sbh sortBlocksHeap ) Len ( ) int {
return len ( sbh )
}
func ( sbh sortBlocksHeap ) Less ( i , j int ) bool {
a := sbh [ i ]
b := sbh [ j ]
return a . Timestamps [ a . NextIdx ] < b . Timestamps [ b . NextIdx ]
}
func ( sbh sortBlocksHeap ) Swap ( i , j int ) {
sbh [ i ] , sbh [ j ] = sbh [ j ] , sbh [ i ]
}
func ( sbh * sortBlocksHeap ) Push ( x interface { } ) {
* sbh = append ( * sbh , x . ( * sortBlock ) )
}
func ( sbh * sortBlocksHeap ) Pop ( ) interface { } {
a := * sbh
v := a [ len ( a ) - 1 ]
* sbh = a [ : len ( a ) - 1 ]
return v
}
// DeleteSeries deletes time series matching the given tagFilterss.
func DeleteSeries ( sq * storage . SearchQuery ) ( int , error ) {
tfss , err := setupTfss ( sq . TagFilterss )
if err != nil {
return 0 , err
}
return vmstorage . DeleteMetrics ( tfss )
}
// GetLabels returns labels until the given deadline.
2020-09-11 12:18:57 +02:00
func GetLabels ( deadline searchutils . Deadline ) ( [ ] string , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2020-09-11 12:18:57 +02:00
labels , err := vmstorage . SearchTagKeys ( * maxTagKeysPerSearch , deadline . Deadline ( ) )
2019-05-22 23:16:55 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "error during labels search: %w" , err )
2019-05-22 23:16:55 +02:00
}
// Substitute "" with "__name__"
for i := range labels {
if labels [ i ] == "" {
labels [ i ] = "__name__"
}
}
// Sort labels like Prometheus does
sort . Strings ( labels )
return labels , nil
}
// GetLabelValues returns label values for the given labelName
// until the given deadline.
2020-09-11 12:18:57 +02:00
func GetLabelValues ( labelName string , deadline searchutils . Deadline ) ( [ ] string , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2019-05-22 23:16:55 +02:00
if labelName == "__name__" {
labelName = ""
}
// Search for tag values
2020-09-11 12:18:57 +02:00
labelValues , err := vmstorage . SearchTagValues ( [ ] byte ( labelName ) , * maxTagValuesPerSearch , deadline . Deadline ( ) )
2019-05-22 23:16:55 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "error during label values search for labelName=%q: %w" , labelName , err )
2019-05-22 23:16:55 +02:00
}
// Sort labelValues like Prometheus does
sort . Strings ( labelValues )
return labelValues , nil
}
2020-09-10 23:28:19 +02:00
// GetTagValueSuffixes returns tag value suffixes for the given tagKey and the given tagValuePrefix.
//
// It can be used for implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find
2020-09-11 12:18:57 +02:00
func GetTagValueSuffixes ( tr storage . TimeRange , tagKey , tagValuePrefix string , delimiter byte , deadline searchutils . Deadline ) ( [ ] string , error ) {
2020-09-10 23:28:19 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2020-09-11 12:18:57 +02:00
suffixes , err := vmstorage . SearchTagValueSuffixes ( tr , [ ] byte ( tagKey ) , [ ] byte ( tagValuePrefix ) , delimiter , * maxTagValueSuffixesPerSearch , deadline . Deadline ( ) )
2020-09-10 23:28:19 +02:00
if err != nil {
return nil , fmt . Errorf ( "error during search for suffixes for tagKey=%q, tagValuePrefix=%q, delimiter=%c on time range %s: %w" ,
tagKey , tagValuePrefix , delimiter , tr . String ( ) , err )
}
return suffixes , nil
}
2019-06-10 17:55:20 +02:00
// GetLabelEntries returns all the label entries until the given deadline.
2020-09-11 12:18:57 +02:00
func GetLabelEntries ( deadline searchutils . Deadline ) ( [ ] storage . TagEntry , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2020-09-11 12:18:57 +02:00
labelEntries , err := vmstorage . SearchTagEntries ( * maxTagKeysPerSearch , * maxTagValuesPerSearch , deadline . Deadline ( ) )
2019-06-10 17:55:20 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "error during label entries request: %w" , err )
2019-06-10 17:55:20 +02:00
}
// Substitute "" with "__name__"
for i := range labelEntries {
e := & labelEntries [ i ]
if e . Key == "" {
e . Key = "__name__"
}
}
// Sort labelEntries by the number of label values in each entry.
sort . Slice ( labelEntries , func ( i , j int ) bool {
a , b := labelEntries [ i ] . Values , labelEntries [ j ] . Values
2019-12-14 23:07:09 +01:00
if len ( a ) != len ( b ) {
return len ( a ) > len ( b )
2019-06-10 17:55:20 +02:00
}
2019-12-14 23:07:09 +01:00
return labelEntries [ i ] . Key > labelEntries [ j ] . Key
2019-06-10 17:55:20 +02:00
} )
return labelEntries , nil
}
2020-04-22 18:57:36 +02:00
// GetTSDBStatusForDate returns tsdb status according to https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats
2020-09-11 12:18:57 +02:00
func GetTSDBStatusForDate ( deadline searchutils . Deadline , date uint64 , topN int ) ( * storage . TSDBStatus , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2020-09-11 12:18:57 +02:00
status , err := vmstorage . GetTSDBStatusForDate ( date , topN , deadline . Deadline ( ) )
2020-04-22 18:57:36 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "error during tsdb status request: %w" , err )
2020-04-22 18:57:36 +02:00
}
return status , nil
}
2019-05-22 23:16:55 +02:00
// GetSeriesCount returns the number of unique series.
2020-09-11 12:18:57 +02:00
func GetSeriesCount ( deadline searchutils . Deadline ) ( uint64 , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return 0 , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2020-09-11 12:18:57 +02:00
n , err := vmstorage . GetSeriesCount ( deadline . Deadline ( ) )
2019-05-22 23:16:55 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return 0 , fmt . Errorf ( "error during series count request: %w" , err )
2019-05-22 23:16:55 +02:00
}
return n , nil
}
func getStorageSearch ( ) * storage . Search {
v := ssPool . Get ( )
if v == nil {
return & storage . Search { }
}
return v . ( * storage . Search )
}
func putStorageSearch ( sr * storage . Search ) {
sr . MustClose ( )
ssPool . Put ( sr )
}
var ssPool sync . Pool
2020-09-26 03:29:45 +02:00
// ExportBlocks searches for time series matching sq and calls f for each found block.
//
// f is called in parallel from multiple goroutines.
2020-09-27 22:17:14 +02:00
// Data processing is immediately stopped if f returns non-nil error.
2020-09-26 03:29:45 +02:00
// It is the responsibility of f to call b.UnmarshalData before reading timestamps and values from the block.
// It is the responsibility of f to filter blocks according to the given tr.
func ExportBlocks ( sq * storage . SearchQuery , deadline searchutils . Deadline , f func ( mn * storage . MetricName , b * storage . Block , tr storage . TimeRange ) error ) error {
if deadline . Exceeded ( ) {
return fmt . Errorf ( "timeout exceeded before starting data export: %s" , deadline . String ( ) )
}
tfss , err := setupTfss ( sq . TagFilterss )
if err != nil {
return err
}
tr := storage . TimeRange {
MinTimestamp : sq . MinTimestamp ,
MaxTimestamp : sq . MaxTimestamp ,
}
if err := vmstorage . CheckTimeRange ( tr ) ; err != nil {
return err
}
vmstorage . WG . Add ( 1 )
defer vmstorage . WG . Done ( )
sr := getStorageSearch ( )
defer putStorageSearch ( sr )
sr . Init ( vmstorage . Storage , tfss , tr , * maxMetricsPerSearch , deadline . Deadline ( ) )
// Start workers that call f in parallel on available CPU cores.
gomaxprocs := runtime . GOMAXPROCS ( - 1 )
workCh := make ( chan * exportWork , gomaxprocs * 8 )
var (
errGlobal error
errGlobalLock sync . Mutex
mustStop uint32
)
var wg sync . WaitGroup
wg . Add ( gomaxprocs )
for i := 0 ; i < gomaxprocs ; i ++ {
go func ( ) {
defer wg . Done ( )
for xw := range workCh {
if err := f ( & xw . mn , & xw . b , tr ) ; err != nil {
errGlobalLock . Lock ( )
if errGlobal != nil {
errGlobal = err
atomic . StoreUint32 ( & mustStop , 1 )
}
errGlobalLock . Unlock ( )
}
xw . reset ( )
exportWorkPool . Put ( xw )
}
} ( )
}
// Feed workers with work
blocksRead := 0
for sr . NextMetricBlock ( ) {
blocksRead ++
if deadline . Exceeded ( ) {
return fmt . Errorf ( "timeout exceeded while fetching data block #%d from storage: %s" , blocksRead , deadline . String ( ) )
}
if atomic . LoadUint32 ( & mustStop ) != 0 {
break
}
xw := exportWorkPool . Get ( ) . ( * exportWork )
if err := xw . mn . Unmarshal ( sr . MetricBlockRef . MetricName ) ; err != nil {
return fmt . Errorf ( "cannot unmarshal metricName for block #%d: %w" , blocksRead , err )
}
sr . MetricBlockRef . BlockRef . MustReadBlock ( & xw . b , true )
workCh <- xw
}
close ( workCh )
// Wait for workers to finish.
wg . Wait ( )
// Check errors.
err = sr . Error ( )
if err == nil {
err = errGlobal
}
if err != nil {
if errors . Is ( err , storage . ErrDeadlineExceeded ) {
return fmt . Errorf ( "timeout exceeded during the query: %s" , deadline . String ( ) )
}
return fmt . Errorf ( "search error after reading %d data blocks: %w" , blocksRead , err )
}
return nil
}
type exportWork struct {
mn storage . MetricName
b storage . Block
}
func ( xw * exportWork ) reset ( ) {
xw . mn . Reset ( )
xw . b . Reset ( )
}
var exportWorkPool = & sync . Pool {
New : func ( ) interface { } {
return & exportWork { }
} ,
}
// ProcessSearchQuery performs sq until the given deadline.
2020-04-27 07:13:41 +02:00
//
// Results.RunParallel or Results.Cancel must be called on the returned Results.
2020-09-11 12:18:57 +02:00
func ProcessSearchQuery ( sq * storage . SearchQuery , fetchData bool , deadline searchutils . Deadline ) ( * Results , error ) {
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
return nil , fmt . Errorf ( "timeout exceeded before starting the query processing: %s" , deadline . String ( ) )
}
2019-05-22 23:16:55 +02:00
// Setup search.
tfss , err := setupTfss ( sq . TagFilterss )
if err != nil {
return nil , err
}
tr := storage . TimeRange {
MinTimestamp : sq . MinTimestamp ,
MaxTimestamp : sq . MaxTimestamp ,
}
2020-06-30 23:20:13 +02:00
if err := vmstorage . CheckTimeRange ( tr ) ; err != nil {
return nil , err
}
2019-05-22 23:16:55 +02:00
vmstorage . WG . Add ( 1 )
defer vmstorage . WG . Done ( )
sr := getStorageSearch ( )
2020-09-11 12:18:57 +02:00
maxSeriesCount := sr . Init ( vmstorage . Storage , tfss , tr , * maxMetricsPerSearch , deadline . Deadline ( ) )
2020-11-04 15:46:10 +01:00
m := make ( map [ string ] [ ] blockRef , maxSeriesCount )
2020-08-06 18:17:51 +02:00
orderedMetricNames := make ( [ ] string , 0 , maxSeriesCount )
2019-07-28 11:12:30 +02:00
blocksRead := 0
2020-11-04 15:46:10 +01:00
tbf := getTmpBlocksFile ( )
var buf [ ] byte
2019-05-22 23:16:55 +02:00
for sr . NextMetricBlock ( ) {
2019-07-28 11:12:30 +02:00
blocksRead ++
2020-07-21 17:34:59 +02:00
if deadline . Exceeded ( ) {
2020-11-04 15:46:10 +01:00
putTmpBlocksFile ( tbf )
2020-09-22 21:56:49 +02:00
putStorageSearch ( sr )
2020-01-22 14:50:34 +01:00
return nil , fmt . Errorf ( "timeout exceeded while fetching data block #%d from storage: %s" , blocksRead , deadline . String ( ) )
2019-05-22 23:16:55 +02:00
}
2020-11-04 15:46:10 +01:00
buf = sr . MetricBlockRef . BlockRef . Marshal ( buf [ : 0 ] )
addr , err := tbf . WriteBlockRefData ( buf )
if err != nil {
putTmpBlocksFile ( tbf )
putStorageSearch ( sr )
return nil , fmt . Errorf ( "cannot write %d bytes to temporary file: %w" , len ( buf ) , err )
}
2020-04-27 07:13:41 +02:00
metricName := sr . MetricBlockRef . MetricName
brs := m [ string ( metricName ) ]
2020-11-04 15:46:10 +01:00
brs = append ( brs , blockRef {
partRef : sr . MetricBlockRef . BlockRef . PartRef ( ) ,
addr : addr ,
} )
2020-07-23 13:11:48 +02:00
if len ( brs ) > 1 {
2020-07-23 12:53:30 +02:00
// An optimization: do not allocate a string for already existing metricName key in m
m [ string ( metricName ) ] = brs
} else {
// An optimization for big number of time series with long metricName values:
// use only a single copy of metricName for both orderedMetricNames and m.
2020-07-23 16:53:52 +02:00
orderedMetricNames = append ( orderedMetricNames , string ( metricName ) )
m [ orderedMetricNames [ len ( orderedMetricNames ) - 1 ] ] = brs
2020-04-26 15:25:35 +02:00
}
2019-05-22 23:16:55 +02:00
}
if err := sr . Error ( ) ; err != nil {
2020-11-04 15:46:10 +01:00
putTmpBlocksFile ( tbf )
2020-09-22 21:56:49 +02:00
putStorageSearch ( sr )
2020-08-10 12:17:12 +02:00
if errors . Is ( err , storage . ErrDeadlineExceeded ) {
return nil , fmt . Errorf ( "timeout exceeded during the query: %s" , deadline . String ( ) )
}
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "search error after reading %d data blocks: %w" , blocksRead , err )
2019-05-22 23:16:55 +02:00
}
2020-11-04 15:46:10 +01:00
if err := tbf . Finalize ( ) ; err != nil {
putTmpBlocksFile ( tbf )
putStorageSearch ( sr )
return nil , fmt . Errorf ( "cannot finalize temporary file: %w" , err )
}
2019-05-22 23:16:55 +02:00
var rss Results
rss . tr = tr
2019-08-04 21:15:33 +02:00
rss . fetchData = fetchData
2019-05-22 23:16:55 +02:00
rss . deadline = deadline
2020-04-26 15:25:35 +02:00
pts := make ( [ ] packedTimeseries , len ( orderedMetricNames ) )
for i , metricName := range orderedMetricNames {
pts [ i ] = packedTimeseries {
metricName : metricName ,
2020-04-27 07:13:41 +02:00
brs : m [ metricName ] ,
2020-04-26 15:25:35 +02:00
}
2019-05-22 23:16:55 +02:00
}
2020-04-26 15:25:35 +02:00
rss . packedTimeseries = pts
2020-04-27 07:13:41 +02:00
rss . sr = sr
2020-11-04 15:46:10 +01:00
rss . tbf = tbf
2019-05-22 23:16:55 +02:00
return & rss , nil
}
2020-11-04 15:46:10 +01:00
type blockRef struct {
partRef storage . PartRef
addr tmpBlockAddr
}
2019-05-22 23:16:55 +02:00
func setupTfss ( tagFilterss [ ] [ ] storage . TagFilter ) ( [ ] * storage . TagFilters , error ) {
tfss := make ( [ ] * storage . TagFilters , 0 , len ( tagFilterss ) )
for _ , tagFilters := range tagFilterss {
tfs := storage . NewTagFilters ( )
for i := range tagFilters {
tf := & tagFilters [ i ]
if err := tfs . Add ( tf . Key , tf . Value , tf . IsNegative , tf . IsRegexp ) ; err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "cannot parse tag filter %s: %w" , tf , err )
2019-05-22 23:16:55 +02:00
}
}
tfss = append ( tfss , tfs )
2020-03-30 17:34:51 +02:00
tfss = append ( tfss , tfs . Finalize ( ) ... )
2019-05-22 23:16:55 +02:00
}
return tfss , nil
}