From 9df5b2d1c35fb7f57ea11b42f36f05cefe48ac09 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 25 Jan 2020 19:19:23 +0200 Subject: [PATCH] app/victoria-metrics: add `-selfScrapeInterval` flag for self-scraping `/metrics` page Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/30 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/180 --- README.md | 7 +- app/victoria-metrics/main.go | 3 + app/victoria-metrics/self_scraper.go | 99 ++++++++++++++++++++++++++++ lib/httpserver/httpserver.go | 2 +- lib/httpserver/metrics.go | 3 +- 5 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 app/victoria-metrics/self_scraper.go diff --git a/README.md b/README.md index 9108229ab..5cdf4977d 100644 --- a/README.md +++ b/README.md @@ -786,8 +786,11 @@ mkfs.ext4 ... -O 64bit,huge_file,extent -T huge ### Monitoring -VictoriaMetrics exports internal metrics in Prometheus format on the `/metrics` page. -Add this page to Prometheus' scrape config in order to collect VictoriaMetrics metrics. +VictoriaMetrics exports internal metrics in Prometheus format at `/metrics` page. +These metrics may be collected either via Prometheus by adding the corresponding scrape config to it. +Alternatively they can be self-scraped by setting `-selfScrapeInterval` command-line flag to duration greater than 0. +For example, `-scrapeInterval=10s` would enable self-scraping of `/metrics` page with 10 seconds interval. + There are officials Grafana dashboards for [single-node VictoriaMetrics](https://grafana.com/dashboards/10229) and [clustered VictoriaMetrics](https://grafana.com/grafana/dashboards/11176). The most interesting metrics are: diff --git a/app/victoria-metrics/main.go b/app/victoria-metrics/main.go index 9a8e067b5..9cfcdad1b 100644 --- a/app/victoria-metrics/main.go +++ b/app/victoria-metrics/main.go @@ -26,6 +26,7 @@ func main() { vmstorage.Init() vmselect.Init() vminsert.Init() + startSelfScraper() go httpserver.Serve(*httpListenAddr, requestHandler) logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds()) @@ -33,6 +34,8 @@ func main() { sig := procutil.WaitForSigterm() logger.Infof("received signal %s", sig) + stopSelfScraper() + logger.Infof("gracefully shutting down webservice at %q", *httpListenAddr) startTime = time.Now() if err := httpserver.Stop(*httpListenAddr); err != nil { diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go new file mode 100644 index 000000000..624a168c8 --- /dev/null +++ b/app/victoria-metrics/self_scraper.go @@ -0,0 +1,99 @@ +package main + +import ( + "flag" + "sync" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" +) + +var selfScrapeInterval = flag.Duration("selfScrapeInterval", 0, "Interval for self-scraping own metrics at `/metrics` page") + +var selfScraperStopCh chan struct{} +var selfScraperWG sync.WaitGroup + +func startSelfScraper() { + selfScraperStopCh = make(chan struct{}) + selfScraperWG.Add(1) + go func() { + defer selfScraperWG.Done() + selfScraper(*selfScrapeInterval) + }() +} + +func stopSelfScraper() { + close(selfScraperStopCh) + selfScraperWG.Wait() +} + +func selfScraper(scrapeInterval time.Duration) { + if scrapeInterval <= 0 { + // Self-scrape is disabled. + return + } + logger.Infof("started self-scraping `/metrics` page with interval %.3f seconds", scrapeInterval.Seconds()) + + var bb bytesutil.ByteBuffer + var rows prometheus.Rows + var mrs []storage.MetricRow + var labels []prompb.Label + t := time.NewTicker(scrapeInterval) + var currentTimestamp int64 + for { + select { + case <-selfScraperStopCh: + t.Stop() + logger.Infof("stopped self-scraping `/metrics` page") + return + case currentTime := <-t.C: + currentTimestamp = currentTime.UnixNano() / 1e6 + } + bb.Reset() + httpserver.WritePrometheusMetrics(&bb) + s := bytesutil.ToUnsafeString(bb.B) + rows.Reset() + rows.Unmarshal(s) + mrs = mrs[:0] + for i := range rows.Rows { + r := &rows.Rows[i] + labels = labels[:0] + labels = addLabel(labels, "", r.Metric) + labels = addLabel(labels, "job", "victoria-metrics") + labels = addLabel(labels, "instance", "self") + for j := range r.Tags { + t := &r.Tags[j] + labels = addLabel(labels, t.Key, t.Value) + } + if len(mrs) < cap(mrs) { + mrs = mrs[:len(mrs)+1] + } else { + mrs = append(mrs, storage.MetricRow{}) + } + mr := &mrs[len(mrs)-1] + mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels) + mr.Timestamp = currentTimestamp + mr.Value = r.Value + } + logger.Infof("writing %d rows at timestamp %d", len(mrs), currentTimestamp) + vmstorage.AddRows(mrs) + } +} + +func addLabel(dst []prompb.Label, key, value string) []prompb.Label { + if len(dst) < cap(dst) { + dst = dst[:len(dst)+1] + } else { + dst = append(dst, prompb.Label{}) + } + lb := &dst[len(dst)-1] + lb.Name = bytesutil.ToUnsafeBytes(key) + lb.Value = bytesutil.ToUnsafeBytes(value) + return dst +} diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index b736508a6..743559f5a 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -181,7 +181,7 @@ func handlerWrapper(w http.ResponseWriter, r *http.Request, rh RequestHandler) { } startTime := time.Now() w.Header().Set("Content-Type", "text/plain") - writePrometheusMetrics(w) + WritePrometheusMetrics(w) metricsHandlerDuration.UpdateDuration(startTime) return default: diff --git a/lib/httpserver/metrics.go b/lib/httpserver/metrics.go index 1ff7a211e..4dae6fb73 100644 --- a/lib/httpserver/metrics.go +++ b/lib/httpserver/metrics.go @@ -12,7 +12,8 @@ import ( "github.com/VictoriaMetrics/metrics" ) -func writePrometheusMetrics(w io.Writer) { +// WritePrometheusMetrics writes all the registered metrics to w in Prometheus exposition format. +func WritePrometheusMetrics(w io.Writer) { metrics.WritePrometheus(w, true) fmt.Fprintf(w, "vm_app_version{version=%q} 1\n", buildinfo.Version)