2019-05-22 23:16:55 +02:00
package vmstorage
import (
2021-10-08 13:15:52 +02:00
"errors"
2019-05-22 23:16:55 +02:00
"flag"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
2020-06-30 23:20:13 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
2020-10-20 13:29:26 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2020-04-01 22:08:58 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
"github.com/VictoriaMetrics/metrics"
)
var (
2020-10-20 13:29:26 +02:00
retentionPeriod = flagutil . NewDuration ( "retentionPeriod" , 1 , "Data with timestamps outside the retentionPeriod is automatically deleted" )
2020-09-17 13:21:39 +02:00
snapshotAuthKey = flag . String ( "snapshotAuthKey" , "" , "authKey, which must be passed in query string to /snapshot* pages" )
forceMergeAuthKey = flag . String ( "forceMergeAuthKey" , "" , "authKey, which must be passed in query string to /internal/force_merge pages" )
2020-11-11 13:40:27 +01:00
forceFlushAuthKey = flag . String ( "forceFlushAuthKey" , "" , "authKey, which must be passed in query string to /internal/force_flush pages" )
2019-05-22 23:16:55 +02:00
precisionBits = flag . Int ( "precisionBits" , 64 , "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss" )
// DataPath is a path to storage data.
DataPath = flag . String ( "storageDataPath" , "victoria-metrics-data" , "Path to storage data" )
2019-10-31 15:16:53 +01:00
2021-01-07 23:09:00 +01:00
finalMergeDelay = flag . Duration ( "finalMergeDelay" , 0 , "The delay before starting final merge for per-month partition after no new data is ingested into it. " +
"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. " +
"Zero value disables final merge" )
2019-10-31 15:16:53 +01:00
bigMergeConcurrency = flag . Int ( "bigMergeConcurrency" , 0 , "The maximum number of CPU cores to use for big merges. Default value is used if set to 0" )
smallMergeConcurrency = flag . Int ( "smallMergeConcurrency" , 0 , "The maximum number of CPU cores to use for small merges. Default value is used if set to 0" )
2020-06-30 23:20:13 +02:00
2021-03-15 21:38:50 +01:00
logNewSeries = flag . Bool ( "logNewSeries" , false , "Whether to log new series. This option is for debug purposes only. It can lead to performance issues " +
"when big number of new series are ingested into VictoriaMetrics" )
2020-06-30 23:20:13 +02:00
denyQueriesOutsideRetention = flag . Bool ( "denyQueriesOutsideRetention" , false , "Whether to deny queries outside of the configured -retentionPeriod. " +
"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. " +
"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee" )
2021-05-20 13:15:19 +02:00
maxHourlySeries = flag . Int ( "storage.maxHourlySeries" , 0 , "The maximum number of unique series can be added to the storage during the last hour. " +
"Excess series are logged and dropped. This can be useful for limiting series cardinality. See also -storage.maxDailySeries" )
maxDailySeries = flag . Int ( "storage.maxDailySeries" , 0 , "The maximum number of unique series can be added to the storage during the last 24 hours. " +
"Excess series are logged and dropped. This can be useful for limiting series churn rate. See also -storage.maxHourlySeries" )
2021-10-08 11:52:56 +02:00
2021-10-08 13:15:52 +02:00
minFreeDiskSpaceBytes = flagutil . NewBytes ( "storage.minFreeDiskSpaceBytes" , 10e6 , "The minimum free disk space at -storageDataPath after which the storage stops accepting new data" )
2019-05-22 23:16:55 +02:00
)
2020-06-30 23:20:13 +02:00
// CheckTimeRange returns true if the given tr is denied for querying.
func CheckTimeRange ( tr storage . TimeRange ) error {
if ! * denyQueriesOutsideRetention {
return nil
}
2020-10-20 13:29:26 +02:00
minAllowedTimestamp := int64 ( fasttime . UnixTimestamp ( ) * 1000 ) - retentionPeriod . Msecs
2020-06-30 23:20:13 +02:00
if tr . MinTimestamp > minAllowedTimestamp {
return nil
}
return & httpserver . ErrorWithStatusCode {
2020-10-20 13:29:26 +02:00
Err : fmt . Errorf ( "the given time range %s is outside the allowed -retentionPeriod=%s according to -denyQueriesOutsideRetention" , & tr , retentionPeriod ) ,
2020-06-30 23:20:13 +02:00
StatusCode : http . StatusServiceUnavailable ,
}
}
2019-05-22 23:16:55 +02:00
// Init initializes vmstorage.
2020-12-14 12:08:22 +01:00
func Init ( resetCacheIfNeeded func ( mrs [ ] storage . MetricRow ) ) {
InitWithoutMetrics ( resetCacheIfNeeded )
2019-07-11 14:34:50 +02:00
registerStorageMetrics ( )
}
// InitWithoutMetrics must be called instead of Init inside tests.
//
// This allows multiple Init / Stop cycles.
2020-12-14 12:08:22 +01:00
func InitWithoutMetrics ( resetCacheIfNeeded func ( mrs [ ] storage . MetricRow ) ) {
2019-05-22 23:16:55 +02:00
if err := encoding . CheckPrecisionBits ( uint8 ( * precisionBits ) ) ; err != nil {
logger . Fatalf ( "invalid `-precisionBits`: %s" , err )
}
2019-10-31 15:16:53 +01:00
2020-12-14 12:08:22 +01:00
resetResponseCacheIfNeeded = resetCacheIfNeeded
2021-03-15 21:38:50 +01:00
storage . SetLogNewSeries ( * logNewSeries )
2020-10-07 16:35:42 +02:00
storage . SetFinalMergeDelay ( * finalMergeDelay )
2019-10-31 15:16:53 +01:00
storage . SetBigMergeWorkersCount ( * bigMergeConcurrency )
storage . SetSmallMergeWorkersCount ( * smallMergeConcurrency )
2021-10-08 13:15:52 +02:00
storage . SetFreeDiskSpaceLimit ( minFreeDiskSpaceBytes . N )
2019-10-31 15:16:53 +01:00
2020-10-20 13:29:26 +02:00
logger . Infof ( "opening storage at %q with -retentionPeriod=%s" , * DataPath , retentionPeriod )
2019-05-22 23:16:55 +02:00
startTime := time . Now ( )
2019-07-11 14:34:50 +02:00
WG = syncwg . WaitGroup { }
2021-05-20 13:15:19 +02:00
strg , err := storage . OpenStorage ( * DataPath , retentionPeriod . Msecs , * maxHourlySeries , * maxDailySeries )
2019-05-22 23:16:55 +02:00
if err != nil {
2020-10-20 13:29:26 +02:00
logger . Fatalf ( "cannot open a storage at %s with -retentionPeriod=%s: %s" , * DataPath , retentionPeriod , err )
2019-05-22 23:16:55 +02:00
}
Storage = strg
var m storage . Metrics
Storage . UpdateMetrics ( & m )
tm := & m . TableMetrics
partsCount := tm . SmallPartsCount + tm . BigPartsCount
blocksCount := tm . SmallBlocksCount + tm . BigBlocksCount
rowsCount := tm . SmallRowsCount + tm . BigRowsCount
2019-07-04 18:09:40 +02:00
sizeBytes := tm . SmallSizeBytes + tm . BigSizeBytes
2020-01-22 17:27:44 +01:00
logger . Infof ( "successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d" ,
* DataPath , time . Since ( startTime ) . Seconds ( ) , partsCount , blocksCount , rowsCount , sizeBytes )
2019-05-22 23:16:55 +02:00
}
// Storage is a storage.
//
// Every storage call must be wrapped into WG.Add(1) ... WG.Done()
// for proper graceful shutdown when Stop is called.
var Storage * storage . Storage
// WG must be incremented before Storage call.
//
// Use syncwg instead of sync, since Add is called from concurrent goroutines.
var WG syncwg . WaitGroup
2020-12-14 12:08:22 +01:00
// resetResponseCacheIfNeeded is a callback for automatic resetting of response cache if needed.
var resetResponseCacheIfNeeded func ( mrs [ ] storage . MetricRow )
2019-05-22 23:16:55 +02:00
// AddRows adds mrs to the storage.
func AddRows ( mrs [ ] storage . MetricRow ) error {
2021-10-08 13:15:52 +02:00
if Storage . IsReadOnly ( ) {
return errReadOnly
}
2020-12-14 12:08:22 +01:00
resetResponseCacheIfNeeded ( mrs )
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
err := Storage . AddRows ( mrs , uint8 ( * precisionBits ) )
WG . Done ( )
return err
}
2021-10-08 13:15:52 +02:00
var errReadOnly = errors . New ( "the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value" )
2020-11-15 23:42:27 +01:00
// RegisterMetricNames registers all the metrics from mrs in the storage.
func RegisterMetricNames ( mrs [ ] storage . MetricRow ) error {
WG . Add ( 1 )
err := Storage . RegisterMetricNames ( mrs )
WG . Done ( )
return err
}
2019-05-22 23:16:55 +02:00
// DeleteMetrics deletes metrics matching tfss.
//
// Returns the number of deleted metrics.
func DeleteMetrics ( tfss [ ] * storage . TagFilters ) ( int , error ) {
WG . Add ( 1 )
n , err := Storage . DeleteMetrics ( tfss )
WG . Done ( )
return n , err
}
2020-11-16 09:55:55 +01:00
// SearchMetricNames returns metric names for the given tfss on the given tr.
func SearchMetricNames ( tfss [ ] * storage . TagFilters , tr storage . TimeRange , maxMetrics int , deadline uint64 ) ( [ ] storage . MetricName , error ) {
WG . Add ( 1 )
mns , err := Storage . SearchMetricNames ( tfss , tr , maxMetrics , deadline )
WG . Done ( )
return mns , err
}
2020-11-04 23:15:43 +01:00
// SearchTagKeysOnTimeRange searches for tag keys on tr.
func SearchTagKeysOnTimeRange ( tr storage . TimeRange , maxTagKeys int , deadline uint64 ) ( [ ] string , error ) {
WG . Add ( 1 )
keys , err := Storage . SearchTagKeysOnTimeRange ( tr , maxTagKeys , deadline )
WG . Done ( )
return keys , err
}
2019-05-22 23:16:55 +02:00
// SearchTagKeys searches for tag keys
2020-07-23 19:42:57 +02:00
func SearchTagKeys ( maxTagKeys int , deadline uint64 ) ( [ ] string , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
keys , err := Storage . SearchTagKeys ( maxTagKeys , deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return keys , err
}
2020-11-04 23:15:43 +01:00
// SearchTagValuesOnTimeRange searches for tag values for the given tagKey on tr.
func SearchTagValuesOnTimeRange ( tagKey [ ] byte , tr storage . TimeRange , maxTagValues int , deadline uint64 ) ( [ ] string , error ) {
WG . Add ( 1 )
values , err := Storage . SearchTagValuesOnTimeRange ( tagKey , tr , maxTagValues , deadline )
WG . Done ( )
return values , err
}
2019-05-22 23:16:55 +02:00
// SearchTagValues searches for tag values for the given tagKey
2020-07-23 19:42:57 +02:00
func SearchTagValues ( tagKey [ ] byte , maxTagValues int , deadline uint64 ) ( [ ] string , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
values , err := Storage . SearchTagValues ( tagKey , maxTagValues , deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return values , err
}
2020-09-10 23:28:19 +02:00
// SearchTagValueSuffixes returns all the tag value suffixes for the given tagKey and tagValuePrefix on the given tr.
//
// This allows implementing https://graphite-api.readthedocs.io/en/latest/api.html#metrics-find or similar APIs.
func SearchTagValueSuffixes ( tr storage . TimeRange , tagKey , tagValuePrefix [ ] byte , delimiter byte , maxTagValueSuffixes int , deadline uint64 ) ( [ ] string , error ) {
WG . Add ( 1 )
suffixes , err := Storage . SearchTagValueSuffixes ( tr , tagKey , tagValuePrefix , delimiter , maxTagValueSuffixes , deadline )
WG . Done ( )
return suffixes , err
}
2021-02-02 23:24:05 +01:00
// SearchGraphitePaths returns all the metric names matching the given Graphite query.
func SearchGraphitePaths ( tr storage . TimeRange , query [ ] byte , maxPaths int , deadline uint64 ) ( [ ] string , error ) {
WG . Add ( 1 )
paths , err := Storage . SearchGraphitePaths ( tr , query , maxPaths , deadline )
WG . Done ( )
return paths , err
}
2019-06-10 17:55:20 +02:00
// SearchTagEntries searches for tag entries.
2020-07-23 19:42:57 +02:00
func SearchTagEntries ( maxTagKeys , maxTagValues int , deadline uint64 ) ( [ ] storage . TagEntry , error ) {
2019-06-10 17:55:20 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
tagEntries , err := Storage . SearchTagEntries ( maxTagKeys , maxTagValues , deadline )
2019-06-10 17:55:20 +02:00
WG . Done ( )
return tagEntries , err
}
2020-04-22 18:57:36 +02:00
// GetTSDBStatusForDate returns TSDB status for the given date.
2020-07-23 19:42:57 +02:00
func GetTSDBStatusForDate ( date uint64 , topN int , deadline uint64 ) ( * storage . TSDBStatus , error ) {
2020-04-22 18:57:36 +02:00
WG . Add ( 1 )
2021-05-13 07:56:02 +02:00
status , err := Storage . GetTSDBStatusWithFiltersForDate ( nil , date , topN , deadline )
2020-04-22 18:57:36 +02:00
WG . Done ( )
return status , err
}
2021-05-12 15:32:48 +02:00
// GetTSDBStatusWithFiltersForDate returns TSDB status for given filters on the given date.
func GetTSDBStatusWithFiltersForDate ( tfss [ ] * storage . TagFilters , date uint64 , topN int , deadline uint64 ) ( * storage . TSDBStatus , error ) {
2021-05-12 14:18:45 +02:00
WG . Add ( 1 )
2021-05-12 15:32:48 +02:00
status , err := Storage . GetTSDBStatusWithFiltersForDate ( tfss , date , topN , deadline )
2021-05-12 14:18:45 +02:00
WG . Done ( )
return status , err
}
2019-05-22 23:16:55 +02:00
// GetSeriesCount returns the number of time series in the storage.
2020-07-23 19:42:57 +02:00
func GetSeriesCount ( deadline uint64 ) ( uint64 , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
n , err := Storage . GetSeriesCount ( deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return n , err
}
// Stop stops the vmstorage
func Stop ( ) {
logger . Infof ( "gracefully closing the storage at %s" , * DataPath )
startTime := time . Now ( )
WG . WaitAndBlock ( )
Storage . MustClose ( )
2020-01-22 17:27:44 +01:00
logger . Infof ( "successfully closed the storage in %.3f seconds" , time . Since ( startTime ) . Seconds ( ) )
2019-05-22 23:16:55 +02:00
logger . Infof ( "the storage has been stopped" )
}
// RequestHandler is a storage request handler.
func RequestHandler ( w http . ResponseWriter , r * http . Request ) bool {
path := r . URL . Path
2020-09-17 11:01:53 +02:00
if path == "/internal/force_merge" {
2020-09-17 13:21:39 +02:00
authKey := r . FormValue ( "authKey" )
if authKey != * forceMergeAuthKey {
httpserver . Errorf ( w , r , "invalid authKey %q. It must match the value from -forceMergeAuthKey command line flag" , authKey )
return true
}
2020-09-17 11:01:53 +02:00
// Run force merge in background
partitionNamePrefix := r . FormValue ( "partition_prefix" )
go func ( ) {
activeForceMerges . Inc ( )
defer activeForceMerges . Dec ( )
logger . Infof ( "forced merge for partition_prefix=%q has been started" , partitionNamePrefix )
startTime := time . Now ( )
if err := Storage . ForceMergePartitions ( partitionNamePrefix ) ; err != nil {
logger . Errorf ( "error in forced merge for partition_prefix=%q: %s" , partitionNamePrefix , err )
}
logger . Infof ( "forced merge for partition_prefix=%q has been successfully finished in %.3f seconds" , partitionNamePrefix , time . Since ( startTime ) . Seconds ( ) )
} ( )
return true
}
2020-11-11 13:40:27 +01:00
if path == "/internal/force_flush" {
authKey := r . FormValue ( "authKey" )
if authKey != * forceFlushAuthKey {
httpserver . Errorf ( w , r , "invalid authKey %q. It must match the value from -forceFlushAuthKey command line flag" , authKey )
return true
}
logger . Infof ( "flushing storage to make pending data available for reading" )
Storage . DebugFlush ( )
return true
}
2019-05-22 23:16:55 +02:00
prometheusCompatibleResponse := false
if path == "/api/v1/admin/tsdb/snapshot" {
// Handle Prometheus API - https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot .
prometheusCompatibleResponse = true
path = "/snapshot/create"
}
if ! strings . HasPrefix ( path , "/snapshot" ) {
return false
}
authKey := r . FormValue ( "authKey" )
if authKey != * snapshotAuthKey {
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "invalid authKey %q. It must match the value from -snapshotAuthKey command line flag" , authKey )
2019-05-22 23:16:55 +02:00
return true
}
path = path [ len ( "/snapshot" ) : ]
switch path {
case "/create" :
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-05-22 23:16:55 +02:00
snapshotPath , err := Storage . CreateSnapshot ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot create snapshot: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
if prometheusCompatibleResponse {
fmt . Fprintf ( w , ` { "status":"success","data": { "name":%q}} ` , snapshotPath )
} else {
fmt . Fprintf ( w , ` { "status":"ok","snapshot":%q} ` , snapshotPath )
}
return true
case "/list" :
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-05-22 23:16:55 +02:00
snapshots , err := Storage . ListSnapshots ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot list snapshots: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
fmt . Fprintf ( w , ` { "status":"ok","snapshots":[ ` )
if len ( snapshots ) > 0 {
for _ , snapshot := range snapshots [ : len ( snapshots ) - 1 ] {
fmt . Fprintf ( w , "\n%q," , snapshot )
}
fmt . Fprintf ( w , "\n%q\n" , snapshots [ len ( snapshots ) - 1 ] )
}
fmt . Fprintf ( w , ` ]} ` )
return true
case "/delete" :
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-05-22 23:16:55 +02:00
snapshotName := r . FormValue ( "snapshot" )
if err := Storage . DeleteSnapshot ( snapshotName ) ; err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot delete snapshot %q: %w" , snapshotName , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
fmt . Fprintf ( w , ` { "status":"ok"} ` )
return true
case "/delete_all" :
2021-11-09 17:03:50 +01:00
w . Header ( ) . Set ( "Content-Type" , "application/json" )
2019-05-22 23:16:55 +02:00
snapshots , err := Storage . ListSnapshots ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot list snapshots: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
for _ , snapshotName := range snapshots {
if err := Storage . DeleteSnapshot ( snapshotName ) ; err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot delete snapshot %q: %w" , snapshotName , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
}
fmt . Fprintf ( w , ` { "status":"ok"} ` )
return true
default :
return false
}
}
2020-09-17 11:01:53 +02:00
var activeForceMerges = metrics . NewCounter ( "vm_active_force_merges" )
2019-07-11 14:34:50 +02:00
func registerStorageMetrics ( ) {
2019-05-22 23:16:55 +02:00
mCache := & storage . Metrics { }
var mCacheLock sync . Mutex
var lastUpdateTime time . Time
m := func ( ) * storage . Metrics {
mCacheLock . Lock ( )
defer mCacheLock . Unlock ( )
if time . Since ( lastUpdateTime ) < time . Second {
return mCache
}
var mc storage . Metrics
2019-07-11 14:34:50 +02:00
Storage . UpdateMetrics ( & mc )
2019-05-22 23:16:55 +02:00
mCache = & mc
lastUpdateTime = time . Now ( )
return mCache
}
tm := func ( ) * storage . TableMetrics {
sm := m ( )
return & sm . TableMetrics
}
idbm := func ( ) * storage . IndexDBMetrics {
sm := m ( )
return & sm . IndexDBMetrics
}
2020-04-01 22:08:58 +02:00
metrics . NewGauge ( fmt . Sprintf ( ` vm_free_disk_space_bytes { path=%q} ` , * DataPath ) , func ( ) float64 {
return float64 ( fs . MustGetFreeSpace ( * DataPath ) )
} )
2021-10-08 13:15:52 +02:00
metrics . NewGauge ( fmt . Sprintf ( ` vm_free_disk_space_limit_bytes { path=%q} ` , * DataPath ) , func ( ) float64 {
return float64 ( minFreeDiskSpaceBytes . N )
2021-10-08 11:52:56 +02:00
} )
2021-10-08 13:15:52 +02:00
metrics . NewGauge ( fmt . Sprintf ( ` vm_storage_is_read_only { path=%q} ` , * DataPath ) , func ( ) float64 {
if Storage . IsReadOnly ( ) {
2021-10-08 11:52:56 +02:00
return 1
}
return 0
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_active_merges { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . ActiveBigMerges )
} )
metrics . NewGauge ( ` vm_active_merges { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . ActiveSmallMerges )
} )
metrics . NewGauge ( ` vm_active_merges { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ActiveMerges )
} )
metrics . NewGauge ( ` vm_merges_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigMergesCount )
} )
metrics . NewGauge ( ` vm_merges_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallMergesCount )
} )
metrics . NewGauge ( ` vm_merges_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . MergesCount )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsMerged )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsMerged )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ItemsMerged )
} )
metrics . NewGauge ( ` vm_rows_deleted_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsDeleted )
} )
metrics . NewGauge ( ` vm_rows_deleted_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsDeleted )
} )
metrics . NewGauge ( ` vm_references { type="storage/big", name="parts"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigPartsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="storage/small", name="parts"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallPartsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="storage", name="partitions"} ` , func ( ) float64 {
return float64 ( tm ( ) . PartitionsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="indexdb", name="objects"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexDBRefCount )
} )
metrics . NewGauge ( ` vm_references { type="indexdb", name="parts"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PartsRefCount )
} )
2019-11-08 18:57:57 +01:00
metrics . NewGauge ( ` vm_new_timeseries_created_total ` , func ( ) float64 {
return float64 ( idbm ( ) . NewTimeseriesCreated )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_missing_tsids_for_metric_id_total ` , func ( ) float64 {
return float64 ( idbm ( ) . MissingTSIDsForMetricID )
} )
2019-11-06 13:24:48 +01:00
metrics . NewGauge ( ` vm_index_blocks_with_metric_ids_processed_total ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksWithMetricIDsProcessed )
} )
metrics . NewGauge ( ` vm_index_blocks_with_metric_ids_incorrect_order_total ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksWithMetricIDsIncorrectOrder )
} )
2021-02-10 15:53:26 +01:00
metrics . NewGauge ( ` vm_composite_index_min_timestamp ` , func ( ) float64 {
return float64 ( idbm ( ) . MinTimestampForCompositeIndex ) / 1e3
} )
2021-02-17 18:13:38 +01:00
metrics . NewGauge ( ` vm_composite_filter_success_conversions_total ` , func ( ) float64 {
return float64 ( idbm ( ) . CompositeFilterSuccessConversions )
} )
metrics . NewGauge ( ` vm_composite_filter_missing_conversions_total ` , func ( ) float64 {
return float64 ( idbm ( ) . CompositeFilterMissingConversions )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_assisted_merges_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallAssistedMerges )
} )
metrics . NewGauge ( ` vm_assisted_merges_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . AssistedMerges )
} )
2020-09-29 20:47:40 +02:00
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/686
2020-09-29 21:44:16 +02:00
metrics . NewGauge ( ` vm_merge_need_free_disk_space { type="storage/small"} ` , func ( ) float64 {
2020-09-29 20:47:40 +02:00
return float64 ( tm ( ) . SmallMergeNeedFreeDiskSpace )
} )
2020-09-29 21:44:16 +02:00
metrics . NewGauge ( ` vm_merge_need_free_disk_space { type="storage/big"} ` , func ( ) float64 {
2020-09-29 20:47:40 +02:00
return float64 ( tm ( ) . BigMergeNeedFreeDiskSpace )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_pending_rows { type="storage"} ` , func ( ) float64 {
return float64 ( tm ( ) . PendingRows )
} )
metrics . NewGauge ( ` vm_pending_rows { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PendingItems )
} )
metrics . NewGauge ( ` vm_parts { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigPartsCount )
} )
metrics . NewGauge ( ` vm_parts { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallPartsCount )
} )
metrics . NewGauge ( ` vm_parts { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PartsCount )
} )
metrics . NewGauge ( ` vm_blocks { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigBlocksCount )
} )
metrics . NewGauge ( ` vm_blocks { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallBlocksCount )
} )
metrics . NewGauge ( ` vm_blocks { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . BlocksCount )
} )
2019-07-04 18:09:40 +02:00
metrics . NewGauge ( ` vm_data_size_bytes { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigSizeBytes )
} )
metrics . NewGauge ( ` vm_data_size_bytes { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallSizeBytes )
} )
metrics . NewGauge ( ` vm_data_size_bytes { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . SizeBytes )
} )
2020-10-09 12:35:48 +02:00
metrics . NewGauge ( ` vm_rows_added_to_storage_total ` , func ( ) float64 {
return float64 ( m ( ) . RowsAddedTotal )
} )
2020-02-27 22:47:05 +01:00
metrics . NewGauge ( ` vm_deduplicated_samples_total { type="merge"} ` , func ( ) float64 {
return float64 ( m ( ) . DedupsDuringMerge )
} )
2019-07-26 19:00:35 +02:00
metrics . NewGauge ( ` vm_rows_ignored_total { reason="big_timestamp"} ` , func ( ) float64 {
2019-07-26 13:10:25 +02:00
return float64 ( m ( ) . TooBigTimestampRows )
} )
2019-07-26 19:00:35 +02:00
metrics . NewGauge ( ` vm_rows_ignored_total { reason="small_timestamp"} ` , func ( ) float64 {
2019-07-26 13:10:25 +02:00
return float64 ( m ( ) . TooSmallTimestampRows )
} )
2019-08-06 13:09:17 +02:00
metrics . NewGauge ( ` vm_concurrent_addrows_limit_reached_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyLimitReached )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_limit_timeout_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyLimitTimeout )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_dropped_rows_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyDroppedRows )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_capacity ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyCapacity )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_current ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyCurrent )
} )
2020-08-05 17:24:51 +02:00
metrics . NewGauge ( ` vm_concurrent_search_tsids_limit_reached_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyLimitReached )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_limit_timeout_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyLimitTimeout )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_capacity ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyCapacity )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_current ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyCurrent )
} )
2020-07-05 18:37:38 +02:00
metrics . NewGauge ( ` vm_search_delays_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchDelays )
} )
2020-05-15 12:44:23 +02:00
metrics . NewGauge ( ` vm_slow_row_inserts_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowRowInserts )
} )
metrics . NewGauge ( ` vm_slow_per_day_index_inserts_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowPerDayIndexInserts )
} )
2020-05-15 13:11:39 +02:00
metrics . NewGauge ( ` vm_slow_metric_name_loads_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowMetricNameLoads )
} )
2020-05-15 12:44:23 +02:00
2021-05-20 13:15:19 +02:00
metrics . NewGauge ( ` vm_hourly_series_limit_rows_dropped_total ` , func ( ) float64 {
return float64 ( m ( ) . HourlySeriesLimitRowsDropped )
} )
metrics . NewGauge ( ` vm_daily_series_limit_rows_dropped_total ` , func ( ) float64 {
return float64 ( m ( ) . DailySeriesLimitRowsDropped )
} )
2020-09-09 22:18:32 +02:00
metrics . NewGauge ( ` vm_timestamps_blocks_merged_total ` , func ( ) float64 {
return float64 ( m ( ) . TimestampsBlocksMerged )
} )
metrics . NewGauge ( ` vm_timestamps_bytes_saved_total ` , func ( ) float64 {
return float64 ( m ( ) . TimestampsBytesSaved )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_rows { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsCount )
} )
metrics . NewGauge ( ` vm_rows { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsCount )
} )
metrics . NewGauge ( ` vm_rows { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ItemsCount )
} )
2019-11-09 22:17:42 +01:00
metrics . NewGauge ( ` vm_date_range_search_calls_total ` , func ( ) float64 {
return float64 ( idbm ( ) . DateRangeSearchCalls )
} )
metrics . NewGauge ( ` vm_date_range_hits_total ` , func ( ) float64 {
return float64 ( idbm ( ) . DateRangeSearchHits )
} )
2021-07-30 07:37:10 +02:00
metrics . NewGauge ( ` vm_global_search_calls_total ` , func ( ) float64 {
return float64 ( idbm ( ) . GlobalSearchCalls )
} )
2019-11-09 22:17:42 +01:00
2019-12-02 19:44:18 +01:00
metrics . NewGauge ( ` vm_missing_metric_names_for_metric_id_total ` , func ( ) float64 {
return float64 ( idbm ( ) . MissingMetricNamesForMetricID )
} )
2019-11-11 12:21:05 +01:00
metrics . NewGauge ( ` vm_date_metric_id_cache_syncs_total ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSyncsCount )
} )
metrics . NewGauge ( ` vm_date_metric_id_cache_resets_total ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheResetsCount )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/date_metricID"} ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSize )
} )
2019-06-19 17:36:47 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/hour_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . HourMetricIDCacheSize )
} )
2020-05-12 00:06:17 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/next_day_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . NextDayMetricIDCacheSize )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/tagFilters"} ` , func ( ) float64 {
2021-07-06 10:01:51 +02:00
return float64 ( idbm ( ) . TagFiltersCacheSize )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheSize ( ) )
} )
2020-08-06 15:30:15 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/prefetchedMetricIDs"} ` , func ( ) float64 {
2020-01-30 00:59:43 +01:00
return float64 ( m ( ) . PrefetchedMetricIDsSize )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/tsid"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . TSIDCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/metricIDs"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . MetricIDCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/metricName"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . MetricNameCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
2021-02-08 23:34:18 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheSizeBytes )
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheSizeBytes )
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheSizeBytes )
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheSizeBytes )
} )
2019-11-13 16:58:05 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/date_metricID"} ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSizeBytes )
} )
2019-11-13 18:00:02 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/hour_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . HourMetricIDCacheSizeBytes )
} )
2020-05-12 00:06:17 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/next_day_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . NextDayMetricIDCacheSizeBytes )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="indexdb/tagFilters"} ` , func ( ) float64 {
2021-07-06 10:01:51 +02:00
return float64 ( idbm ( ) . TagFiltersCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
2020-01-30 00:59:43 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/prefetchedMetricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . PrefetchedMetricIDsSizeBytes )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/tagFilters"} ` , func ( ) float64 {
2021-07-06 10:01:51 +02:00
return float64 ( idbm ( ) . TagFiltersCacheRequests )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheRequests ( ) )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/tagFilters"} ` , func ( ) float64 {
2021-07-06 10:01:51 +02:00
return float64 ( idbm ( ) . TagFiltersCacheMisses )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheMisses ( ) )
} )
metrics . NewGauge ( ` vm_deleted_metrics_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DeletedMetricsCount )
} )
metrics . NewGauge ( ` vm_cache_collisions_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheCollisions )
} )
metrics . NewGauge ( ` vm_cache_collisions_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheCollisions )
} )
}
2020-03-10 22:51:50 +01:00
func jsonResponseError ( w http . ResponseWriter , err error ) {
logger . Errorf ( "%s" , err )
w . WriteHeader ( http . StatusInternalServerError )
fmt . Fprintf ( w , ` { "status":"error","msg":%q} ` , err )
}