diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go index ed12efeb4..fb544a009 100644 --- a/app/victoria-metrics/self_scraper.go +++ b/app/victoria-metrics/self_scraper.go @@ -6,8 +6,8 @@ import ( "time" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" @@ -60,7 +60,7 @@ func selfScraper(scrapeInterval time.Duration) { currentTimestamp = currentTime.UnixNano() / 1e6 } bb.Reset() - httpserver.WritePrometheusMetrics(&bb) + appmetrics.WritePrometheusMetrics(&bb) s := bytesutil.ToUnsafeString(bb.B) rows.Reset() rows.Unmarshal(s) diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 90836da57..7f3705ff2 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -31,6 +31,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" "github.com/VictoriaMetrics/metrics" @@ -63,6 +64,7 @@ var staticServer = http.FileServer(http.FS(staticFiles)) // Init initializes vminsert. func Init() { + pushmetrics.Init() relabel.Init() storage.SetMaxLabelsPerTimeseries(*maxLabelsPerTimeseries) storage.SetMaxLabelValueLen(*maxLabelValueLen) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c21d352a7..c347be171 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): execute left and right sides of certain operations in parallel. For example, `q1 or q2`, `aggr_func(q1) q2`, `q1 aggr_func(q1)`. This may improve query performance if VictoriaMetrics has enough free resources for parallel processing of both sides of the operation. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmagent.html): allow duplicate username records with different passwords at configuration file. It should allow password rotation without username change. +* FEATURE: add ability to push internal metrics (e.g. metrics exposed at `/metrics` page) to the configured remote storage from all the VictoriaMetrics components. See [these docs](https://docs.victoriametrics.com/#push-metrics). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): restart all the scrape jobs during [config reload](https://docs.victoriametrics.com/vmagent.html#configuration-update) after `global` section is changed inside `-promscrape.config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly assume role with AWS ECS credentials. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2875). Thanks to @transacid for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2876). diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index bd48a9fb7..7c5d762ce 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1636,6 +1636,20 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet See also [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html). +## Push metrics + +All the VictoriaMetrics apps support pushing their metrics exposed at `/metrics` page to remote storage in Prometheus text exposition format. This can be done by specifying the following command-line flags: + +* `-pushmetrics.url` - the url to push metrics to. For example, `-pushmetrics.url=http://victoria-metrics:8428/api/v1/import/prometheus` instructs to push internal metrics to `/api/v1/import/prometheus` endpoint according to [these docs](#how-to-import-data-in-prometheus-exposition-format). The `-pushmetrics.url` can be specified multiple times. In this case metrics are pushed to all the specified urls. The url can contain basic auth params in the form http://user:pass@hostname/api/v1/import/prometheus . +* `-pushmetrics.interval` - the interval between pushes. By default it is set to 10 seconds. +* `-pushmetrics.extraLabels` - the list of labels to add to all the metrics before sending them to `-pushmetrics.url`. + +For example, the following command instructs VictoriaMetrics to push metrics from `/metrics` page to `https://maas.victoriametrics.com/api/v1/import/prometheus` with `user:pass` [Basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication): + +```console +/path/to/victoria-metrics -pushmetrics.url=https://user:pass@maas.victoriametrics.com/api/v1/import/prometheus +``` + ## Cache removal VictoriaMetrics uses various internal caches. These caches are stored to `<-storageDataPath>/cache` directory during graceful shutdown (e.g. when VictoriaMetrics is stopped by sending `SIGINT` signal). The caches are read on the next VictoriaMetrics startup. Sometimes it is needed to remove such caches on the next startup. This can be performed by placing `reset_cache_on_startup` file inside the `<-storageDataPath>/cache` directory before the restart of VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447) for details. diff --git a/go.mod b/go.mod index 9bb1a1c7a..ff1bed669 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // Do not use the original github.com/valyala/fasthttp because of issues // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.1.0 - github.com/VictoriaMetrics/metrics v1.18.1 + github.com/VictoriaMetrics/metrics v1.19.3 github.com/VictoriaMetrics/metricsql v0.44.1 github.com/aws/aws-sdk-go v1.44.56 github.com/cespare/xxhash/v2 v2.1.2 diff --git a/go.sum b/go.sum index 820db6f81..acd25aee5 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,9 @@ github.com/VictoriaMetrics/fastcache v1.10.0 h1:5hDJnLsKLpnUEToub7ETuRu8RCkb40wo github.com/VictoriaMetrics/fastcache v1.10.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8= github.com/VictoriaMetrics/fasthttp v1.1.0 h1:3crd4YWHsMwu60GUXRH6OstowiFvqrwS4a/ueoLdLL0= github.com/VictoriaMetrics/fasthttp v1.1.0/go.mod h1:/7DMcogqd+aaD3G3Hg5kFgoFwlR2uydjiWvoLp5ZTqQ= -github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= +github.com/VictoriaMetrics/metrics v1.19.3 h1:cr7yyS6fHSzjvwCAYsJbvh8qaRfFzilkcqgHgO97e6Y= +github.com/VictoriaMetrics/metrics v1.19.3/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/VictoriaMetrics/metricsql v0.44.1 h1:qGoRt0g84uMUscVjS7P3uDZKmjJubWKaIx9v0iHKgck= github.com/VictoriaMetrics/metricsql v0.44.1/go.mod h1:6pP1ZeLVJHqJrHlF6Ij3gmpQIznSsgktEcZgsAWYel0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= diff --git a/lib/httpserver/metrics.go b/lib/appmetrics/appmetrics.go similarity index 99% rename from lib/httpserver/metrics.go rename to lib/appmetrics/appmetrics.go index 4ad6781a9..ba6885e29 100644 --- a/lib/httpserver/metrics.go +++ b/lib/appmetrics/appmetrics.go @@ -1,4 +1,4 @@ -package httpserver +package appmetrics import ( "flag" diff --git a/lib/httpserver/httpserver.go b/lib/httpserver/httpserver.go index ddd3352e9..eaa4bc645 100644 --- a/lib/httpserver/httpserver.go +++ b/lib/httpserver/httpserver.go @@ -20,6 +20,7 @@ import ( "sync/atomic" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -281,7 +282,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques } startTime := time.Now() w.Header().Set("Content-Type", "text/plain; charset=utf-8") - WritePrometheusMetrics(w) + appmetrics.WritePrometheusMetrics(w) metricsHandlerDuration.UpdateDuration(startTime) return case "/flags": diff --git a/lib/pushmetrics/pushmetrics.go b/lib/pushmetrics/pushmetrics.go new file mode 100644 index 000000000..05cf22007 --- /dev/null +++ b/lib/pushmetrics/pushmetrics.go @@ -0,0 +1,32 @@ +package pushmetrics + +import ( + "flag" + "strings" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" + "github.com/VictoriaMetrics/metrics" +) + +var ( + pushURL = flagutil.NewArray("pushmetrics.url", "Optional URL to push metrics exposed at /metrics page. See https://docs.victoriametrics.com/#push-metrics . "+ + "By default metrics exposed at /metrics page aren't pushed to any remote storage") + pushInterval = flag.Duration("pushmetrics.interval", 10*time.Second, "Interval for pushing metrics to -pushmetrics.url") + pushExtraLabels = flagutil.NewArray("pushmetrics.extraLabels", "Optional labels to add to metrics pushed to -pushmetrics.url . "+ + `For example, -pushmetrics.extraLabels='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url`) +) + +func init() { + // The -pushmetrics.url flag can contain basic auth creds, so it mustn't be visible when exposing the flags. + flagutil.RegisterSecretFlag("pushmetrics.url") +} + +// Init must be called after flag.Parse. +func Init() { + extraLabels := strings.Join(*pushExtraLabels, ",") + for _, pu := range *pushURL { + metrics.InitPushExt(pu, *pushInterval, extraLabels, appmetrics.WritePrometheusMetrics) + } +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/README.md b/vendor/github.com/VictoriaMetrics/metrics/README.md index 5eef96a66..e1a2537cb 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/README.md +++ b/vendor/github.com/VictoriaMetrics/metrics/README.md @@ -16,6 +16,9 @@ * Allows exporting distinct metric sets via distinct endpoints. See [Set](http://godoc.org/github.com/VictoriaMetrics/metrics#Set). * Supports [easy-to-use histograms](http://godoc.org/github.com/VictoriaMetrics/metrics#Histogram), which just work without any tuning. Read more about VictoriaMetrics histograms at [this article](https://medium.com/@valyala/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350). +* Can push metrics to VictoriaMetrics or to any other remote storage, which accepts metrics + in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format). + See [these docs](http://godoc.org/github.com/VictoriaMetrics/metrics#InitPush). ### Limitations @@ -28,8 +31,8 @@ ```go import "github.com/VictoriaMetrics/metrics" -// Register various time series. -// Time series name may contain labels in Prometheus format - see below. +// Register various metrics. +// Metric name may contain labels in Prometheus format - see below. var ( // Register counter without labels. requestsTotal = metrics.NewCounter("requests_total") @@ -64,6 +67,10 @@ func requestHandler() { http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { metrics.WritePrometheus(w, true) }) + +// ... or push registered metrics every 10 seconds to http://victoria-metrics:8428/api/v1/import/prometheus +// with the added `instance="foobar"` label to all the pushed metrics. +metrics.InitPush("http://victoria-metrics:8428/api/v1/import/prometheus", 10*time.Second, `instance="foobar"`, true) ``` See [docs](http://godoc.org/github.com/VictoriaMetrics/metrics) for more info. @@ -86,8 +93,8 @@ Because the `github.com/prometheus/client_golang` is too complex and is hard to #### Why the `metrics.WritePrometheus` doesn't expose documentation for each metric? Because this documentation is ignored by Prometheus. The documentation is for users. -Just give meaningful names to the exported metrics or add comments in the source code -or in other suitable place explaining each metric exposed from your application. +Just give [meaningful names to the exported metrics](https://prometheus.io/docs/practices/naming/#metric-names) +or add comments in the source code or in other suitable place explaining each metric exposed from your application. #### How to implement [CounterVec](https://godoc.org/github.com/prometheus/client_golang/prometheus#CounterVec) in `metrics`? diff --git a/vendor/github.com/VictoriaMetrics/metrics/metrics.go b/vendor/github.com/VictoriaMetrics/metrics/metrics.go index c28c03613..57dcd3f0a 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/metrics.go +++ b/vendor/github.com/VictoriaMetrics/metrics/metrics.go @@ -110,3 +110,8 @@ func WriteFDMetrics(w io.Writer) { func UnregisterMetric(name string) bool { return defaultSet.UnregisterMetric(name) } + +// ListMetricNames returns a list of all the metric names from default set. +func ListMetricNames() []string { + return defaultSet.ListMetricNames() +} diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go index 12b5de8e3..005af82f3 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_linux.go @@ -45,13 +45,13 @@ func writeProcessMetrics(w io.Writer) { statFilepath := "/proc/self/stat" data, err := ioutil.ReadFile(statFilepath) if err != nil { - log.Printf("ERROR: cannot open %s: %s", statFilepath, err) + log.Printf("ERROR: metrics: cannot open %s: %s", statFilepath, err) return } // Search for the end of command. n := bytes.LastIndex(data, []byte(") ")) if n < 0 { - log.Printf("ERROR: cannot find command in parentheses in %q read from %s", data, statFilepath) + log.Printf("ERROR: metrics: cannot find command in parentheses in %q read from %s", data, statFilepath) return } data = data[n+2:] @@ -62,7 +62,7 @@ func writeProcessMetrics(w io.Writer) { &p.State, &p.Ppid, &p.Pgrp, &p.Session, &p.TtyNr, &p.Tpgid, &p.Flags, &p.Minflt, &p.Cminflt, &p.Majflt, &p.Cmajflt, &p.Utime, &p.Stime, &p.Cutime, &p.Cstime, &p.Priority, &p.Nice, &p.NumThreads, &p.ItrealValue, &p.Starttime, &p.Vsize, &p.Rss) if err != nil { - log.Printf("ERROR: cannot parse %q read from %s: %s", data, statFilepath, err) + log.Printf("ERROR: metrics: cannot parse %q read from %s: %s", data, statFilepath, err) return } @@ -89,17 +89,17 @@ func writeIOMetrics(w io.Writer) { ioFilepath := "/proc/self/io" data, err := ioutil.ReadFile(ioFilepath) if err != nil { - log.Printf("ERROR: cannot open %q: %s", ioFilepath, err) + log.Printf("ERROR: metrics: cannot open %q: %s", ioFilepath, err) } getInt := func(s string) int64 { n := strings.IndexByte(s, ' ') if n < 0 { - log.Printf("ERROR: cannot find whitespace in %q at %q", s, ioFilepath) + log.Printf("ERROR: metrics: cannot find whitespace in %q at %q", s, ioFilepath) return 0 } v, err := strconv.ParseInt(s[n+1:], 10, 64) if err != nil { - log.Printf("ERROR: cannot parse %q at %q: %s", s, ioFilepath, err) + log.Printf("ERROR: metrics: cannot parse %q at %q: %s", s, ioFilepath, err) return 0 } return v @@ -137,12 +137,12 @@ var startTimeSeconds = time.Now().Unix() func writeFDMetrics(w io.Writer) { totalOpenFDs, err := getOpenFDsCount("/proc/self/fd") if err != nil { - log.Printf("ERROR: cannot determine open file descriptors count: %s", err) + log.Printf("ERROR: metrics: cannot determine open file descriptors count: %s", err) return } maxOpenFDs, err := getMaxFilesLimit("/proc/self/limits") if err != nil { - log.Printf("ERROR: cannot determine the limit on open file descritors: %s", err) + log.Printf("ERROR: metrics: cannot determine the limit on open file descritors: %s", err) return } fmt.Fprintf(w, "process_max_fds %d\n", maxOpenFDs) @@ -211,7 +211,7 @@ type memStats struct { func writeProcessMemMetrics(w io.Writer) { ms, err := getMemStats("/proc/self/status") if err != nil { - log.Printf("ERROR: cannot determine memory status: %s", err) + log.Printf("ERROR: metrics: cannot determine memory status: %s", err) return } fmt.Fprintf(w, "process_virtual_memory_peak_bytes %d\n", ms.vmPeak) diff --git a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go index 5e6ac935d..ca7167f80 100644 --- a/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go +++ b/vendor/github.com/VictoriaMetrics/metrics/process_metrics_other.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package metrics diff --git a/vendor/github.com/VictoriaMetrics/metrics/push.go b/vendor/github.com/VictoriaMetrics/metrics/push.go new file mode 100644 index 000000000..a63c93693 --- /dev/null +++ b/vendor/github.com/VictoriaMetrics/metrics/push.go @@ -0,0 +1,184 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "net/url" + "time" +) + +// InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval. +// +// extraLabels may contain comma-separated list of `label="value"` labels, which will be added +// to all the metrics before pushing them to pushURL. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushProcessMetrics multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func InitPushProcessMetrics(pushURL string, interval time.Duration, extraLabels string) error { + writeMetrics := func(w io.Writer) { + WriteProcessMetrics(w) + } + return InitPushExt(pushURL, interval, extraLabels, writeMetrics) +} + +// InitPush sets up periodic push for globally registered metrics to the given pushURL with the given interval. +// +// extraLabels may contain comma-separated list of `label="value"` labels, which will be added +// to all the metrics before pushing them to pushURL. +// +// If pushProcessMetrics is set to true, then 'process_*' metrics are also pushed to pushURL. +// +// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPush multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func InitPush(pushURL string, interval time.Duration, extraLabels string, pushProcessMetrics bool) error { + writeMetrics := func(w io.Writer) { + WritePrometheus(w, pushProcessMetrics) + } + return InitPushExt(pushURL, interval, extraLabels, writeMetrics) +} + +// InitPush sets up periodic push for metrics from s to the given pushURL with the given interval. +// +// extraLabels may contain comma-separated list of `label="value"` labels, which will be added +// to all the metrics before pushing them to pushURL. +// +/// The metrics are pushed to pushURL in Prometheus text exposition format. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPush multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels string) error { + writeMetrics := func(w io.Writer) { + s.WritePrometheus(w) + } + return InitPushExt(pushURL, interval, extraLabels, writeMetrics) +} + +// InitPushExt sets up periodic push for metrics obtained by calling writeMetrics with the given interval. +// +// extraLabels may contain comma-separated list of `label="value"` labels, which will be added +// to all the metrics before pushing them to pushURL. +// +// The writeMetrics callback must write metrics to w in Prometheus text exposition format without timestamps and trailing comments. +// See https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format +// +// It is recommended pushing metrics to /api/v1/import/prometheus endpoint according to +// https://docs.victoriametrics.com/#how-to-import-data-in-prometheus-exposition-format +// +// It is OK calling InitPushExt multiple times with different pushURL - +// in this case metrics are pushed to all the provided pushURL urls. +func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error { + if interval <= 0 { + return fmt.Errorf("interval must be positive; got %s", interval) + } + if err := validateTags(extraLabels); err != nil { + return fmt.Errorf("invalid extraLabels=%q: %w", extraLabels, err) + } + pu, err := url.Parse(pushURL) + if err != nil { + return fmt.Errorf("cannot parse pushURL=%q: %w", pushURL, err) + } + if pu.Scheme != "http" && pu.Scheme != "https" { + return fmt.Errorf("unsupported scheme in pushURL=%q; expecting 'http' or 'https'", pushURL) + } + if pu.Host == "" { + return fmt.Errorf("missing host in pushURL=%q", pushURL) + } + pushURLRedacted := pu.Redacted() + c := &http.Client{ + Timeout: interval, + } + go func() { + ticker := time.NewTicker(interval) + var bb bytes.Buffer + var tmpBuf []byte + for range ticker.C { + bb.Reset() + writeMetrics(&bb) + if len(extraLabels) > 0 { + tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels) + bb.Reset() + bb.Write(tmpBuf) + } + resp, err := c.Post(pushURL, "text/plain", &bb) + if err != nil { + log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err) + continue + } + if resp.StatusCode/100 != 2 { + body, _ := ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + log.Printf("ERROR: metrics.push: unexpected status code in response from %q: %d; expecting 2xx; response body: %q", + pushURLRedacted, resp.StatusCode, body) + continue + } + _ = resp.Body.Close() + } + }() + return nil +} + +func addExtraLabels(dst, src []byte, extraLabels string) []byte { + for len(src) > 0 { + var line []byte + n := bytes.IndexByte(src, '\n') + if n >= 0 { + line = src[:n] + src = src[n+1:] + } else { + line = src + src = nil + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + // Skip empy lines + continue + } + if bytes.HasPrefix(line, bashBytes) { + // Copy comments as is + dst = append(dst, line...) + dst = append(dst, '\n') + continue + } + n = bytes.IndexByte(line, '{') + if n >= 0 { + dst = append(dst, line[:n+1]...) + dst = append(dst, extraLabels...) + dst = append(dst, ',') + dst = append(dst, line[n+1:]...) + } else { + n = bytes.LastIndexByte(line, ' ') + if n < 0 { + panic(fmt.Errorf("BUG: missing whitespace between metric name and metric value in Prometheus text exposition line %q", line)) + } + dst = append(dst, line[:n]...) + dst = append(dst, '{') + dst = append(dst, extraLabels...) + dst = append(dst, '}') + dst = append(dst, line[n:]...) + } + dst = append(dst, '\n') + } + return dst +} + +var bashBytes = []byte("#") diff --git a/vendor/modules.txt b/vendor/modules.txt index 3d72fd062..707c7f5e7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -24,7 +24,7 @@ github.com/VictoriaMetrics/fastcache github.com/VictoriaMetrics/fasthttp github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless -# github.com/VictoriaMetrics/metrics v1.18.1 +# github.com/VictoriaMetrics/metrics v1.19.3 ## explicit; go 1.12 github.com/VictoriaMetrics/metrics # github.com/VictoriaMetrics/metricsql v0.44.1