app/vlstorage: expose vl_data_size_bytes metric at /metrics page for tracking the on-disk data size (both indexdb and the data itself)

app/vlstorage/main.go

@@ -121,24 +121,25 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
 		return float64(m().FileMergesTotal)
 	})
 
-	ms.NewGauge(`vl_rows{type="inmemory"}`, func() float64 {
+	ms.NewGauge(`vl_storage_rows{type="inmemory"}`, func() float64 {
 		return float64(m().InmemoryRowsCount)
 	})
-	ms.NewGauge(`vl_rows{type="file"}`, func() float64 {
+	ms.NewGauge(`vl_storage_rows{type="file"}`, func() float64 {
 		return float64(m().FileRowsCount)
 	})
-	ms.NewGauge(`vl_parts{type="inmemory"}`, func() float64 {
+	ms.NewGauge(`vl_storage_parts{type="inmemory"}`, func() float64 {
 		return float64(m().InmemoryParts)
 	})
-	ms.NewGauge(`vl_parts{type="file"}`, func() float64 {
+	ms.NewGauge(`vl_storage_parts{type="file"}`, func() float64 {
 		return float64(m().FileParts)
 	})
-	ms.NewGauge(`vl_blocks{type="inmemory"}`, func() float64 {
+	ms.NewGauge(`vl_storage_blocks{type="inmemory"}`, func() float64 {
 		return float64(m().InmemoryBlocks)
 	})
-	ms.NewGauge(`vl_blocks{type="file"}`, func() float64 {
+	ms.NewGauge(`vl_storage_blocks{type="file"}`, func() float64 {
 		return float64(m().FileBlocks)
 	})
+
 	ms.NewGauge(`vl_partitions`, func() float64 {
 		return float64(m().PartitionsCount)
 	})
@@ -146,6 +147,24 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
 		return float64(m().StreamsCreatedTotal)
 	})
 
+	ms.NewGauge(`vl_indexdb_rows`, func() float64 {
+		return float64(m().IndexdbItemsCount)
+	})
+	ms.NewGauge(`vl_indexdb_parts`, func() float64 {
+		return float64(m().IndexdbPartsCount)
+	})
+	ms.NewGauge(`vl_indexdb_blocks`, func() float64 {
+		return float64(m().IndexdbBlocksCount)
+	})
+
+	ms.NewGauge(`vl_data_size_bytes{type="indexdb"}`, func() float64 {
+		return float64(m().IndexdbSizeBytes)
+	})
+	ms.NewGauge(`vl_data_size_bytes{type="storage"}`, func() float64 {
+		dm := m()
+		return float64(dm.CompressedInmemorySize + dm.CompressedFileSize)
+	})
+
 	ms.NewGauge(`vl_compressed_data_size_bytes{type="inmemory"}`, func() float64 {
 		return float64(m().CompressedInmemorySize)
 	})

dashboards/victorialogs.json

@@ -98,7 +98,7 @@
           },
           "editorMode": "code",
           "exemplar": false,
-          "expr": "sum(vl_rows)",
+          "expr": "sum(vl_storage_rows)",
           "format": "time_series",
           "instant": true,
           "interval": "",
@@ -776,4 +776,4 @@
   "uid": "OqPIZTX4z",
   "version": 4,
   "weekStart": ""
-}
+}

docs/VictoriaLogs/CHANGELOG.md

@@ -5,6 +5,10 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta
 
 ## tip
 
+* FEATURE: expose the following metrics at [/metrics](monitoring) page:
+  * `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
+  * `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes.
+
 ## [v0.3.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v0.3.0-victorialogs)
 
 Released at 2023-07-20

lib/logstorage/indexdb.go

@@ -33,6 +33,18 @@ const (
 type IndexdbStats struct {
 	// StreamsCreatedTotal is the number of log streams created since the indexdb initialization.
 	StreamsCreatedTotal uint64
+
+	// IndexdbSizeBytes is the size of data in indexdb.
+	IndexdbSizeBytes uint64
+
+	// IndexdbItemsCount is the number of items in indexdb.
+	IndexdbItemsCount uint64
+
+	// IndexdbBlocksCount is the number of blocks in indexdb.
+	IndexdbBlocksCount uint64
+
+	// IndexdbPartsCount is the number of parts in indexdb.
+	IndexdbPartsCount uint64
 }
 
 type indexdb struct {
@@ -88,6 +100,14 @@ func (idb *indexdb) debugFlush() {
 
 func (idb *indexdb) updateStats(d *IndexdbStats) {
 	d.StreamsCreatedTotal += atomic.LoadUint64(&idb.streamsCreatedTotal)
+
+	var tm mergeset.TableMetrics
+	idb.tb.UpdateMetrics(&tm)
+
+	d.IndexdbSizeBytes += tm.InmemorySizeBytes + tm.FileSizeBytes
+	d.IndexdbItemsCount += tm.InmemoryItemsCount + tm.FileItemsCount
+	d.IndexdbPartsCount += tm.InmemoryPartsCount + tm.FilePartsCount
+	d.IndexdbBlocksCount += tm.InmemoryBlocksCount + tm.FileBlocksCount
 }
 
 func (idb *indexdb) appendStreamTagsByStreamID(dst []byte, sid *streamID) []byte {