lib/fs: add vm_filestream_read_duration_seconds_total and vm_filestream_write_duration_seconds_total metrics

These metrics help determining persistent disk saturation with `rate(vm_filestream_read_duration_seconds_total) > 0.9`

docs/CHANGELOG.md

@@ -8,7 +8,8 @@ sort: 15
 
 * FEATURE: add `now()` function to MetricsQL. This function returns the current timestamp in seconds. See [these docs](https://docs.victoriametrics.com/MetricsQL.html#now).
 * FEATURE: vmauth: allow using optional `name` field in configs. This field is then used as `username` label value for `vmauth_user_requests_total` metric. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1805).
-* FEATURE: vmagent: export `vm_persistentqueue_read_duration_seconds_total` and `vm_persistentqueue_write_duration_seconds_total` metrics, which can be used for detecting persistent queue saturation with `rate(vm_persistentqueue_write_duration_seconds_total) > 0.9s` alerting rule.
+* FEATURE: vmagent: export `vm_persistentqueue_read_duration_seconds_total` and `vm_persistentqueue_write_duration_seconds_total` metrics, which can be used for detecting persistent queue saturation with `rate(vm_persistentqueue_write_duration_seconds_total) > 0.9` alerting rule.
+* FEATURE: export `vm_filestream_read_duration_seconds_total` and `vm_filestream_write_duration_seconds_total` metrics, which can be used for detecting persistent disk saturation with `rate(vm_filestream_read_duration_seconds_total) > 0.9` alerting rule.
 * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html), [vmrestore](https://docs.victoriametrics.com/vmrestore.html): add `-s3ForcePathStyle` command-line flag, which can be used for making backups to [Aliyun OSS](https://www.aliyun.com/product/oss). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1802).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): improve data migration from OpenTSDB. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1809). Thanks to @johnseekins .
 * FEATURE: suppress `connection reset by peer` errors when remote client resets TCP connection to VictoriaMetrics / vmagent while ingesting the data via InfluxDB line protocol, Graphite protocol or OpenTSDB protocol. This error is expected, so there is no need in logging it.

lib/filestream/filestream.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
@@ -108,6 +109,7 @@ func (r *Reader) MustClose() {
 }
 
 var (
+	readDuration      = metrics.NewFloatCounter(`vm_filestream_read_duration_seconds_total`)
 	readCallsBuffered = metrics.NewCounter(`vm_filestream_buffered_read_calls_total`)
 	readCallsReal     = metrics.NewCounter(`vm_filestream_real_read_calls_total`)
 	readBytesBuffered = metrics.NewCounter(`vm_filestream_buffered_read_bytes_total`)
@@ -117,6 +119,11 @@ var (
 
 // Read reads file contents to p.
 func (r *Reader) Read(p []byte) (int, error) {
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime).Seconds()
+		readDuration.Add(d)
+	}()
 	readCallsBuffered.Inc()
 	n, err := r.br.Read(p)
 	readBytesBuffered.Add(n)
@@ -232,6 +239,7 @@ func (w *Writer) MustClose() {
 }
 
 var (
+	writeDuration        = metrics.NewFloatCounter(`vm_filestream_write_duration_seconds_total`)
 	writeCallsBuffered   = metrics.NewCounter(`vm_filestream_buffered_write_calls_total`)
 	writeCallsReal       = metrics.NewCounter(`vm_filestream_real_write_calls_total`)
 	writtenBytesBuffered = metrics.NewCounter(`vm_filestream_buffered_written_bytes_total`)
@@ -241,6 +249,11 @@ var (
 
 // Write writes p to the underlying file.
 func (w *Writer) Write(p []byte) (int, error) {
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime).Seconds()
+		writeDuration.Add(d)
+	}()
 	writeCallsBuffered.Inc()
 	n, err := w.bw.Write(p)
 	writtenBytesBuffered.Add(n)
@@ -257,6 +270,11 @@ func (w *Writer) Write(p []byte) (int, error) {
 //
 // if isSync is true, then the flushed data is fsynced to the underlying storage.
 func (w *Writer) MustFlush(isSync bool) {
+	startTime := time.Now()
+	defer func() {
+		d := time.Since(startTime).Seconds()
+		writeDuration.Add(d)
+	}()
 	if err := w.bw.Flush(); err != nil {
 		logger.Panicf("FATAL: cannot flush buffered data to file %q: %s", w.f.Name(), err)
 	}