mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-29 23:30:04 +01:00
090cb2c9de
* app/vmagent: allow to disabled on-disk queue Previously, it wasn't possible to build data processing pipeline with a chain of vmagents. In case when remoteWrite for the last vmagent in the chain wasn't accessible, it persisted data only when it has enough disk capacity. If disk queue is full, it started to silently drop ingested metrics. New flags allows to disable on-disk persistent and immediatly return an error if remoteWrite is not accessible anymore. It blocks any writes and notify client, that data ingestion isn't possible. Main use case for this feature - use external queue such as kafka for data persistence. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2110 * adds test, updates readme * apply review suggestions * update docs for vmagent * makes linter happy --------- Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
89 lines
2.8 KiB
Go
89 lines
2.8 KiB
Go
package newrelic
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/VictoriaMetrics/metrics"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/newrelic/stream"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
|
)
|
|
|
|
var (
|
|
rowsInserted = metrics.NewCounter(`vmagent_rows_inserted_total{type="newrelic"}`)
|
|
rowsTenantInserted = tenantmetrics.NewCounterMap(`vmagent_tenant_inserted_rows_total{type="newrelic"}`)
|
|
rowsPerInsert = metrics.NewHistogram(`vmagent_rows_per_insert{type="newrelic"}`)
|
|
)
|
|
|
|
// InsertHandlerForHTTP processes remote write for NewRelic POST /infra/v2/metrics/events/bulk request.
|
|
func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
|
extraLabels, err := parserCommon.GetExtraLabels(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ce := req.Header.Get("Content-Encoding")
|
|
isGzip := ce == "gzip"
|
|
return stream.Parse(req.Body, isGzip, func(rows []newrelic.Row) error {
|
|
return insertRows(at, rows, extraLabels)
|
|
})
|
|
}
|
|
|
|
func insertRows(at *auth.Token, rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
|
|
ctx := common.GetPushCtx()
|
|
defer common.PutPushCtx(ctx)
|
|
|
|
samplesCount := 0
|
|
tssDst := ctx.WriteRequest.Timeseries[:0]
|
|
labels := ctx.Labels[:0]
|
|
samples := ctx.Samples[:0]
|
|
for i := range rows {
|
|
r := &rows[i]
|
|
tags := r.Tags
|
|
srcSamples := r.Samples
|
|
for j := range srcSamples {
|
|
s := &srcSamples[j]
|
|
labelsLen := len(labels)
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: "__name__",
|
|
Value: bytesutil.ToUnsafeString(s.Name),
|
|
})
|
|
for k := range tags {
|
|
t := &tags[k]
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: bytesutil.ToUnsafeString(t.Key),
|
|
Value: bytesutil.ToUnsafeString(t.Value),
|
|
})
|
|
}
|
|
samples = append(samples, prompbmarshal.Sample{
|
|
Value: s.Value,
|
|
Timestamp: r.Timestamp,
|
|
})
|
|
tssDst = append(tssDst, prompbmarshal.TimeSeries{
|
|
Labels: labels[labelsLen:],
|
|
Samples: samples[len(samples)-1:],
|
|
})
|
|
labels = append(labels, extraLabels...)
|
|
}
|
|
samplesCount += len(srcSamples)
|
|
}
|
|
ctx.WriteRequest.Timeseries = tssDst
|
|
ctx.Labels = labels
|
|
ctx.Samples = samples
|
|
if !remotewrite.Push(at, &ctx.WriteRequest) {
|
|
return remotewrite.ErrQueueFullHTTPRetry
|
|
}
|
|
rowsInserted.Add(len(rows))
|
|
if at != nil {
|
|
rowsTenantInserted.Get(at).Add(samplesCount)
|
|
}
|
|
rowsPerInsert.Update(float64(samplesCount))
|
|
return nil
|
|
}
|