From 6ad7fe8eeb1722ae98375911c85668989ec6aa86 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 8 Nov 2019 19:57:57 +0200 Subject: [PATCH] lib/storage: export `vm_new_timeseries_created_total` metric for determining time series churn rate --- README.md | 1 + app/vmstorage/main.go | 3 +++ lib/storage/index_db.go | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/README.md b/README.md index 20547309e..107055f17 100644 --- a/README.md +++ b/README.md @@ -675,6 +675,7 @@ The most interesting metrics are: * `vm_cache_entries{type="storage/hour_metric_ids"}` - the number of time series with new data points during the last hour aka active time series. +* `rate(vm_new_timeseries_created_total[5m])` - time series churn rate. * `vm_rows{type="indexdb"}` - the number of rows in inverted index. High value for this number usually mean high churn rate for time series. * Sum of `vm_rows{type="storage/big"}` and `vm_rows{type="storage/small"}` - total number of `(timestamp, value)` data points in the database. diff --git a/app/vmstorage/main.go b/app/vmstorage/main.go index b6c8fab39..de1fb3def 100644 --- a/app/vmstorage/main.go +++ b/app/vmstorage/main.go @@ -305,6 +305,9 @@ func registerStorageMetrics() { return float64(idbm().PartsRefCount) }) + metrics.NewGauge(`vm_new_timeseries_created_total`, func() float64 { + return float64(idbm().NewTimeseriesCreated) + }) metrics.NewGauge(`vm_missing_tsids_for_metric_id_total`, func() float64 { return float64(idbm().MissingTSIDsForMetricID) }) diff --git a/lib/storage/index_db.go b/lib/storage/index_db.go index ba76026d2..faab8f1ee 100644 --- a/lib/storage/index_db.go +++ b/lib/storage/index_db.go @@ -73,6 +73,9 @@ type indexDB struct { refCount uint64 + // The counter for newly created time series. It can be used for determining time series churn rate. + newTimeseriesCreated uint64 + // The number of missing MetricID -> TSID entries. // High rate for this value means corrupted indexDB. missingTSIDsForMetricID uint64 @@ -200,6 +203,7 @@ type IndexDBMetrics struct { IndexDBRefCount uint64 + NewTimeseriesCreated uint64 MissingTSIDsForMetricID uint64 RecentHourMetricIDsSearchCalls uint64 @@ -241,6 +245,7 @@ func (db *indexDB) UpdateMetrics(m *IndexDBMetrics) { m.DeletedMetricsCount += uint64(db.getDeletedMetricIDs().Len()) m.IndexDBRefCount += atomic.LoadUint64(&db.refCount) + m.NewTimeseriesCreated += atomic.LoadUint64(&db.newTimeseriesCreated) m.MissingTSIDsForMetricID += atomic.LoadUint64(&db.missingTSIDsForMetricID) m.RecentHourMetricIDsSearchCalls += atomic.LoadUint64(&db.recentHourMetricIDsSearchCalls) m.RecentHourMetricIDsSearchHits += atomic.LoadUint64(&db.recentHourMetricIDsSearchHits) @@ -568,6 +573,7 @@ func (db *indexDB) createTSIDByName(dst *TSID, metricName []byte) error { // There is no need in invalidating tag cache, since it is invalidated // on db.tb flush via invalidateTagCache flushCallback passed to OpenTable. + atomic.AddUint64(&db.newTimeseriesCreated, 1) return nil }