2019-05-22 23:16:55 +02:00
package vmstorage
import (
"flag"
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
2020-06-30 23:20:13 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
2020-04-01 22:08:58 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
2019-05-22 23:16:55 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
"github.com/VictoriaMetrics/metrics"
)
var (
retentionPeriod = flag . Int ( "retentionPeriod" , 1 , "Retention period in months" )
snapshotAuthKey = flag . String ( "snapshotAuthKey" , "" , "authKey, which must be passed in query string to /snapshot* pages" )
precisionBits = flag . Int ( "precisionBits" , 64 , "The number of precision bits to store per each value. Lower precision bits improves data compression at the cost of precision loss" )
// DataPath is a path to storage data.
DataPath = flag . String ( "storageDataPath" , "victoria-metrics-data" , "Path to storage data" )
2019-10-31 15:16:53 +01:00
bigMergeConcurrency = flag . Int ( "bigMergeConcurrency" , 0 , "The maximum number of CPU cores to use for big merges. Default value is used if set to 0" )
smallMergeConcurrency = flag . Int ( "smallMergeConcurrency" , 0 , "The maximum number of CPU cores to use for small merges. Default value is used if set to 0" )
2020-06-30 23:20:13 +02:00
denyQueriesOutsideRetention = flag . Bool ( "denyQueriesOutsideRetention" , false , "Whether to deny queries outside of the configured -retentionPeriod. " +
"When set, then /api/v1/query_range would return '503 Service Unavailable' error for queries with 'from' value outside -retentionPeriod. " +
"This may be useful when multiple data sources with distinct retentions are hidden behind query-tee" )
2019-05-22 23:16:55 +02:00
)
2020-06-30 23:20:13 +02:00
// CheckTimeRange returns true if the given tr is denied for querying.
func CheckTimeRange ( tr storage . TimeRange ) error {
if ! * denyQueriesOutsideRetention {
return nil
}
minAllowedTimestamp := ( int64 ( fasttime . UnixTimestamp ( ) ) - int64 ( * retentionPeriod ) * 3600 * 24 * 30 ) * 1000
if tr . MinTimestamp > minAllowedTimestamp {
return nil
}
return & httpserver . ErrorWithStatusCode {
Err : fmt . Errorf ( "the given time range %s is outside the allowed retention of %d months according to -denyQueriesOutsideRetention" , & tr , * retentionPeriod ) ,
StatusCode : http . StatusServiceUnavailable ,
}
}
2019-05-22 23:16:55 +02:00
// Init initializes vmstorage.
func Init ( ) {
2019-07-11 14:34:50 +02:00
InitWithoutMetrics ( )
registerStorageMetrics ( )
}
// InitWithoutMetrics must be called instead of Init inside tests.
//
// This allows multiple Init / Stop cycles.
func InitWithoutMetrics ( ) {
2019-05-22 23:16:55 +02:00
if err := encoding . CheckPrecisionBits ( uint8 ( * precisionBits ) ) ; err != nil {
logger . Fatalf ( "invalid `-precisionBits`: %s" , err )
}
2019-10-31 15:16:53 +01:00
storage . SetBigMergeWorkersCount ( * bigMergeConcurrency )
storage . SetSmallMergeWorkersCount ( * smallMergeConcurrency )
2019-05-22 23:16:55 +02:00
logger . Infof ( "opening storage at %q with retention period %d months" , * DataPath , * retentionPeriod )
startTime := time . Now ( )
2019-07-11 14:34:50 +02:00
WG = syncwg . WaitGroup { }
2019-05-22 23:16:55 +02:00
strg , err := storage . OpenStorage ( * DataPath , * retentionPeriod )
if err != nil {
logger . Fatalf ( "cannot open a storage at %s with retention period %d months: %s" , * DataPath , * retentionPeriod , err )
}
Storage = strg
var m storage . Metrics
Storage . UpdateMetrics ( & m )
tm := & m . TableMetrics
partsCount := tm . SmallPartsCount + tm . BigPartsCount
blocksCount := tm . SmallBlocksCount + tm . BigBlocksCount
rowsCount := tm . SmallRowsCount + tm . BigRowsCount
2019-07-04 18:09:40 +02:00
sizeBytes := tm . SmallSizeBytes + tm . BigSizeBytes
2020-01-22 17:27:44 +01:00
logger . Infof ( "successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d" ,
* DataPath , time . Since ( startTime ) . Seconds ( ) , partsCount , blocksCount , rowsCount , sizeBytes )
2019-05-22 23:16:55 +02:00
}
// Storage is a storage.
//
// Every storage call must be wrapped into WG.Add(1) ... WG.Done()
// for proper graceful shutdown when Stop is called.
var Storage * storage . Storage
// WG must be incremented before Storage call.
//
// Use syncwg instead of sync, since Add is called from concurrent goroutines.
var WG syncwg . WaitGroup
// AddRows adds mrs to the storage.
func AddRows ( mrs [ ] storage . MetricRow ) error {
WG . Add ( 1 )
err := Storage . AddRows ( mrs , uint8 ( * precisionBits ) )
WG . Done ( )
return err
}
// DeleteMetrics deletes metrics matching tfss.
//
// Returns the number of deleted metrics.
func DeleteMetrics ( tfss [ ] * storage . TagFilters ) ( int , error ) {
WG . Add ( 1 )
n , err := Storage . DeleteMetrics ( tfss )
WG . Done ( )
return n , err
}
// SearchTagKeys searches for tag keys
2020-07-23 19:42:57 +02:00
func SearchTagKeys ( maxTagKeys int , deadline uint64 ) ( [ ] string , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
keys , err := Storage . SearchTagKeys ( maxTagKeys , deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return keys , err
}
// SearchTagValues searches for tag values for the given tagKey
2020-07-23 19:42:57 +02:00
func SearchTagValues ( tagKey [ ] byte , maxTagValues int , deadline uint64 ) ( [ ] string , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
values , err := Storage . SearchTagValues ( tagKey , maxTagValues , deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return values , err
}
2019-06-10 17:55:20 +02:00
// SearchTagEntries searches for tag entries.
2020-07-23 19:42:57 +02:00
func SearchTagEntries ( maxTagKeys , maxTagValues int , deadline uint64 ) ( [ ] storage . TagEntry , error ) {
2019-06-10 17:55:20 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
tagEntries , err := Storage . SearchTagEntries ( maxTagKeys , maxTagValues , deadline )
2019-06-10 17:55:20 +02:00
WG . Done ( )
return tagEntries , err
}
2020-04-22 18:57:36 +02:00
// GetTSDBStatusForDate returns TSDB status for the given date.
2020-07-23 19:42:57 +02:00
func GetTSDBStatusForDate ( date uint64 , topN int , deadline uint64 ) ( * storage . TSDBStatus , error ) {
2020-04-22 18:57:36 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
status , err := Storage . GetTSDBStatusForDate ( date , topN , deadline )
2020-04-22 18:57:36 +02:00
WG . Done ( )
return status , err
}
2019-05-22 23:16:55 +02:00
// GetSeriesCount returns the number of time series in the storage.
2020-07-23 19:42:57 +02:00
func GetSeriesCount ( deadline uint64 ) ( uint64 , error ) {
2019-05-22 23:16:55 +02:00
WG . Add ( 1 )
2020-07-23 19:42:57 +02:00
n , err := Storage . GetSeriesCount ( deadline )
2019-05-22 23:16:55 +02:00
WG . Done ( )
return n , err
}
// Stop stops the vmstorage
func Stop ( ) {
logger . Infof ( "gracefully closing the storage at %s" , * DataPath )
startTime := time . Now ( )
WG . WaitAndBlock ( )
Storage . MustClose ( )
2020-01-22 17:27:44 +01:00
logger . Infof ( "successfully closed the storage in %.3f seconds" , time . Since ( startTime ) . Seconds ( ) )
2019-05-22 23:16:55 +02:00
logger . Infof ( "the storage has been stopped" )
}
// RequestHandler is a storage request handler.
func RequestHandler ( w http . ResponseWriter , r * http . Request ) bool {
path := r . URL . Path
prometheusCompatibleResponse := false
if path == "/api/v1/admin/tsdb/snapshot" {
// Handle Prometheus API - https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot .
prometheusCompatibleResponse = true
path = "/snapshot/create"
}
if ! strings . HasPrefix ( path , "/snapshot" ) {
return false
}
authKey := r . FormValue ( "authKey" )
if authKey != * snapshotAuthKey {
2020-07-20 13:00:33 +02:00
httpserver . Errorf ( w , r , "invalid authKey %q. It must match the value from -snapshotAuthKey command line flag" , authKey )
2019-05-22 23:16:55 +02:00
return true
}
path = path [ len ( "/snapshot" ) : ]
switch path {
case "/create" :
w . Header ( ) . Set ( "Content-Type" , "application/json" )
snapshotPath , err := Storage . CreateSnapshot ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot create snapshot: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
if prometheusCompatibleResponse {
fmt . Fprintf ( w , ` { "status":"success","data": { "name":%q}} ` , snapshotPath )
} else {
fmt . Fprintf ( w , ` { "status":"ok","snapshot":%q} ` , snapshotPath )
}
return true
case "/list" :
w . Header ( ) . Set ( "Content-Type" , "application/json" )
snapshots , err := Storage . ListSnapshots ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot list snapshots: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
fmt . Fprintf ( w , ` { "status":"ok","snapshots":[ ` )
if len ( snapshots ) > 0 {
for _ , snapshot := range snapshots [ : len ( snapshots ) - 1 ] {
fmt . Fprintf ( w , "\n%q," , snapshot )
}
fmt . Fprintf ( w , "\n%q\n" , snapshots [ len ( snapshots ) - 1 ] )
}
fmt . Fprintf ( w , ` ]} ` )
return true
case "/delete" :
w . Header ( ) . Set ( "Content-Type" , "application/json" )
snapshotName := r . FormValue ( "snapshot" )
if err := Storage . DeleteSnapshot ( snapshotName ) ; err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot delete snapshot %q: %w" , snapshotName , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
fmt . Fprintf ( w , ` { "status":"ok"} ` )
return true
case "/delete_all" :
w . Header ( ) . Set ( "Content-Type" , "application/json" )
snapshots , err := Storage . ListSnapshots ( )
if err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot list snapshots: %w" , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
for _ , snapshotName := range snapshots {
if err := Storage . DeleteSnapshot ( snapshotName ) ; err != nil {
2020-06-30 21:58:18 +02:00
err = fmt . Errorf ( "cannot delete snapshot %q: %w" , snapshotName , err )
2020-03-10 22:51:50 +01:00
jsonResponseError ( w , err )
2019-05-22 23:16:55 +02:00
return true
}
}
fmt . Fprintf ( w , ` { "status":"ok"} ` )
return true
default :
return false
}
}
2019-07-11 14:34:50 +02:00
func registerStorageMetrics ( ) {
2019-05-22 23:16:55 +02:00
mCache := & storage . Metrics { }
var mCacheLock sync . Mutex
var lastUpdateTime time . Time
m := func ( ) * storage . Metrics {
mCacheLock . Lock ( )
defer mCacheLock . Unlock ( )
if time . Since ( lastUpdateTime ) < time . Second {
return mCache
}
var mc storage . Metrics
2019-07-11 14:34:50 +02:00
Storage . UpdateMetrics ( & mc )
2019-05-22 23:16:55 +02:00
mCache = & mc
lastUpdateTime = time . Now ( )
return mCache
}
tm := func ( ) * storage . TableMetrics {
sm := m ( )
return & sm . TableMetrics
}
idbm := func ( ) * storage . IndexDBMetrics {
sm := m ( )
return & sm . IndexDBMetrics
}
2020-04-01 22:08:58 +02:00
metrics . NewGauge ( fmt . Sprintf ( ` vm_free_disk_space_bytes { path=%q} ` , * DataPath ) , func ( ) float64 {
return float64 ( fs . MustGetFreeSpace ( * DataPath ) )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_active_merges { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . ActiveBigMerges )
} )
metrics . NewGauge ( ` vm_active_merges { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . ActiveSmallMerges )
} )
metrics . NewGauge ( ` vm_active_merges { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ActiveMerges )
} )
metrics . NewGauge ( ` vm_merges_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigMergesCount )
} )
metrics . NewGauge ( ` vm_merges_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallMergesCount )
} )
metrics . NewGauge ( ` vm_merges_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . MergesCount )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsMerged )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsMerged )
} )
metrics . NewGauge ( ` vm_rows_merged_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ItemsMerged )
} )
metrics . NewGauge ( ` vm_rows_deleted_total { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsDeleted )
} )
metrics . NewGauge ( ` vm_rows_deleted_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsDeleted )
} )
metrics . NewGauge ( ` vm_references { type="storage/big", name="parts"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigPartsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="storage/small", name="parts"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallPartsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="storage", name="partitions"} ` , func ( ) float64 {
return float64 ( tm ( ) . PartitionsRefCount )
} )
metrics . NewGauge ( ` vm_references { type="indexdb", name="objects"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexDBRefCount )
} )
metrics . NewGauge ( ` vm_references { type="indexdb", name="parts"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PartsRefCount )
} )
2019-11-08 18:57:57 +01:00
metrics . NewGauge ( ` vm_new_timeseries_created_total ` , func ( ) float64 {
return float64 ( idbm ( ) . NewTimeseriesCreated )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_missing_tsids_for_metric_id_total ` , func ( ) float64 {
return float64 ( idbm ( ) . MissingTSIDsForMetricID )
} )
2019-06-09 21:11:09 +02:00
metrics . NewGauge ( ` vm_recent_hour_metric_ids_search_calls_total ` , func ( ) float64 {
2019-06-09 18:06:53 +02:00
return float64 ( idbm ( ) . RecentHourMetricIDsSearchCalls )
} )
2019-06-09 21:11:09 +02:00
metrics . NewGauge ( ` vm_recent_hour_metric_ids_search_hits_total ` , func ( ) float64 {
2019-06-09 18:06:53 +02:00
return float64 ( idbm ( ) . RecentHourMetricIDsSearchHits )
} )
2019-06-09 21:11:09 +02:00
metrics . NewGauge ( ` vm_date_metric_ids_search_calls_total ` , func ( ) float64 {
2019-06-09 18:06:53 +02:00
return float64 ( idbm ( ) . DateMetricIDsSearchCalls )
} )
2019-06-09 21:11:09 +02:00
metrics . NewGauge ( ` vm_date_metric_ids_search_hits_total ` , func ( ) float64 {
2019-06-09 18:06:53 +02:00
return float64 ( idbm ( ) . DateMetricIDsSearchHits )
} )
2019-11-06 13:24:48 +01:00
metrics . NewGauge ( ` vm_index_blocks_with_metric_ids_processed_total ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksWithMetricIDsProcessed )
} )
metrics . NewGauge ( ` vm_index_blocks_with_metric_ids_incorrect_order_total ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksWithMetricIDsIncorrectOrder )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_assisted_merges_total { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallAssistedMerges )
} )
metrics . NewGauge ( ` vm_assisted_merges_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . AssistedMerges )
} )
metrics . NewGauge ( ` vm_pending_rows { type="storage"} ` , func ( ) float64 {
return float64 ( tm ( ) . PendingRows )
} )
metrics . NewGauge ( ` vm_pending_rows { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PendingItems )
} )
metrics . NewGauge ( ` vm_parts { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigPartsCount )
} )
metrics . NewGauge ( ` vm_parts { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallPartsCount )
} )
metrics . NewGauge ( ` vm_parts { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . PartsCount )
} )
metrics . NewGauge ( ` vm_blocks { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigBlocksCount )
} )
metrics . NewGauge ( ` vm_blocks { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallBlocksCount )
} )
metrics . NewGauge ( ` vm_blocks { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . BlocksCount )
} )
2019-07-04 18:09:40 +02:00
metrics . NewGauge ( ` vm_data_size_bytes { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigSizeBytes )
} )
metrics . NewGauge ( ` vm_data_size_bytes { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallSizeBytes )
} )
metrics . NewGauge ( ` vm_data_size_bytes { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . SizeBytes )
} )
2020-02-27 22:47:05 +01:00
metrics . NewGauge ( ` vm_deduplicated_samples_total { type="merge"} ` , func ( ) float64 {
return float64 ( m ( ) . DedupsDuringMerge )
} )
2019-07-26 19:00:35 +02:00
metrics . NewGauge ( ` vm_rows_ignored_total { reason="big_timestamp"} ` , func ( ) float64 {
2019-07-26 13:10:25 +02:00
return float64 ( m ( ) . TooBigTimestampRows )
} )
2019-07-26 19:00:35 +02:00
metrics . NewGauge ( ` vm_rows_ignored_total { reason="small_timestamp"} ` , func ( ) float64 {
2019-07-26 13:10:25 +02:00
return float64 ( m ( ) . TooSmallTimestampRows )
} )
2019-08-06 13:09:17 +02:00
metrics . NewGauge ( ` vm_concurrent_addrows_limit_reached_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyLimitReached )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_limit_timeout_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyLimitTimeout )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_dropped_rows_total ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyDroppedRows )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_capacity ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyCapacity )
} )
metrics . NewGauge ( ` vm_concurrent_addrows_current ` , func ( ) float64 {
return float64 ( m ( ) . AddRowsConcurrencyCurrent )
} )
2020-08-05 17:24:51 +02:00
metrics . NewGauge ( ` vm_concurrent_search_tsids_limit_reached_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyLimitReached )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_limit_timeout_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyLimitTimeout )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_capacity ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyCapacity )
} )
metrics . NewGauge ( ` vm_concurrent_search_tsids_current ` , func ( ) float64 {
return float64 ( m ( ) . SearchTSIDsConcurrencyCurrent )
} )
2020-07-05 18:37:38 +02:00
metrics . NewGauge ( ` vm_search_delays_total ` , func ( ) float64 {
return float64 ( m ( ) . SearchDelays )
} )
2020-05-15 12:44:23 +02:00
metrics . NewGauge ( ` vm_slow_row_inserts_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowRowInserts )
} )
metrics . NewGauge ( ` vm_slow_per_day_index_inserts_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowPerDayIndexInserts )
} )
2020-05-15 13:11:39 +02:00
metrics . NewGauge ( ` vm_slow_metric_name_loads_total ` , func ( ) float64 {
return float64 ( m ( ) . SlowMetricNameLoads )
} )
2020-05-15 12:44:23 +02:00
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_rows { type="storage/big"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigRowsCount )
} )
metrics . NewGauge ( ` vm_rows { type="storage/small"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallRowsCount )
} )
metrics . NewGauge ( ` vm_rows { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . ItemsCount )
} )
2019-11-09 22:17:42 +01:00
metrics . NewGauge ( ` vm_date_range_search_calls_total ` , func ( ) float64 {
return float64 ( idbm ( ) . DateRangeSearchCalls )
} )
metrics . NewGauge ( ` vm_date_range_hits_total ` , func ( ) float64 {
return float64 ( idbm ( ) . DateRangeSearchHits )
} )
2019-12-02 19:44:18 +01:00
metrics . NewGauge ( ` vm_missing_metric_names_for_metric_id_total ` , func ( ) float64 {
return float64 ( idbm ( ) . MissingMetricNamesForMetricID )
} )
2019-11-11 12:21:05 +01:00
metrics . NewGauge ( ` vm_date_metric_id_cache_syncs_total ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSyncsCount )
} )
metrics . NewGauge ( ` vm_date_metric_id_cache_resets_total ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheResetsCount )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/date_metricID"} ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSize )
} )
2019-06-19 17:36:47 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/hour_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . HourMetricIDCacheSize )
} )
2020-05-12 00:06:17 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/next_day_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . NextDayMetricIDCacheSize )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheSize )
} )
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/tagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . TagCacheSize )
} )
2019-06-10 13:02:44 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="indexdb/uselessTagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . UselessTagFiltersCacheSize )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheSize ( ) )
} )
2020-08-06 15:30:15 +02:00
metrics . NewGauge ( ` vm_cache_entries { type="storage/prefetchedMetricIDs"} ` , func ( ) float64 {
2020-01-30 00:59:43 +01:00
return float64 ( m ( ) . PrefetchedMetricIDsSize )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/tsid"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . TSIDCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/metricIDs"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . MetricIDCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/metricName"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( m ( ) . MetricNameCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
2019-11-13 16:58:05 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/date_metricID"} ` , func ( ) float64 {
return float64 ( m ( ) . DateMetricIDCacheSizeBytes )
} )
2019-11-13 18:00:02 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/hour_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . HourMetricIDCacheSizeBytes )
} )
2020-05-12 00:06:17 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/next_day_metric_ids"} ` , func ( ) float64 {
return float64 ( m ( ) . NextDayMetricIDCacheSizeBytes )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="indexdb/tagFilters"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( idbm ( ) . TagCacheSizeBytes )
2019-05-22 23:16:55 +02:00
} )
2019-06-10 13:02:44 +02:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="indexdb/uselessTagFilters"} ` , func ( ) float64 {
2019-07-09 23:47:29 +02:00
return float64 ( idbm ( ) . UselessTagFiltersCacheSizeBytes )
2019-06-10 13:02:44 +02:00
} )
2020-01-30 00:59:43 +01:00
metrics . NewGauge ( ` vm_cache_size_bytes { type="storage/prefetchedMetricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . PrefetchedMetricIDsSizeBytes )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheRequests )
} )
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/tagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . TagCacheRequests )
} )
2019-06-10 13:02:44 +02:00
metrics . NewGauge ( ` vm_cache_requests_total { type="indexdb/uselessTagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . UselessTagFiltersCacheRequests )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_requests_total { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheRequests ( ) )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/metricIDs"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricIDCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/bigIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . BigIndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/smallIndexBlocks"} ` , func ( ) float64 {
return float64 ( tm ( ) . SmallIndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/dataBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DataBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/indexBlocks"} ` , func ( ) float64 {
return float64 ( idbm ( ) . IndexBlocksCacheMisses )
} )
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/tagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . TagCacheMisses )
} )
2019-06-10 13:02:44 +02:00
metrics . NewGauge ( ` vm_cache_misses_total { type="indexdb/uselessTagFilters"} ` , func ( ) float64 {
return float64 ( idbm ( ) . UselessTagFiltersCacheMisses )
} )
2019-05-22 23:16:55 +02:00
metrics . NewGauge ( ` vm_cache_misses_total { type="storage/regexps"} ` , func ( ) float64 {
return float64 ( storage . RegexpCacheMisses ( ) )
} )
metrics . NewGauge ( ` vm_deleted_metrics_total { type="indexdb"} ` , func ( ) float64 {
return float64 ( idbm ( ) . DeletedMetricsCount )
} )
metrics . NewGauge ( ` vm_cache_collisions_total { type="storage/tsid"} ` , func ( ) float64 {
return float64 ( m ( ) . TSIDCacheCollisions )
} )
metrics . NewGauge ( ` vm_cache_collisions_total { type="storage/metricName"} ` , func ( ) float64 {
return float64 ( m ( ) . MetricNameCacheCollisions )
} )
}
2020-03-10 22:51:50 +01:00
func jsonResponseError ( w http . ResponseWriter , err error ) {
logger . Errorf ( "%s" , err )
w . WriteHeader ( http . StatusInternalServerError )
fmt . Fprintf ( w , ` { "status":"error","msg":%q} ` , err )
}