mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-14 16:12:15 +01:00
app/vminsert: remove support for undocumented VictoriaMetrics_AccountID and VictoriaMetrics_ProjectID labels in tcp-based data ingestion endpoints
These labels are substituted by documented vm_account_id and vm_project_id labels.
See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels
This is a follow up for 505d359b39
This commit is contained in:
Aliaksandr Valialkin
parent
505d359b39
commit
c8000c029a
@ -10,7 +10,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson/fastfloat"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -35,10 +34,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
defer netstorage.PutInsertCtx(ctx)
|
||||
|
||||
ctx.Reset() // This line is required for initializing ctx internals.
|
||||
var atCopy auth.Token
|
||||
if at != nil {
|
||||
atCopy = *at
|
||||
}
|
||||
perTenantRows := make(map[auth.Token]int)
|
||||
hasRelabeling := relabel.HasRelabeling()
|
||||
for i := range rows {
|
||||
@ -47,16 +42,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
ctx.AddLabel("", r.Metric)
|
||||
for j := range r.Tags {
|
||||
tag := &r.Tags[j]
|
||||
if atCopy.AccountID == 0 {
|
||||
// Multi-tenancy support via custom tags.
|
||||
// Do not allow overriding AccountID and ProjectID from atCopy for security reasons.
|
||||
if tag.Key == "VictoriaMetrics_AccountID" {
|
||||
atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" {
|
||||
atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
}
|
||||
ctx.AddLabel(tag.Key, tag.Value)
|
||||
}
|
||||
if hasRelabeling {
|
||||
@ -67,7 +52,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
continue
|
||||
}
|
||||
ctx.SortLabelsIfNeeded()
|
||||
atLocal := ctx.GetLocalAuthToken(&atCopy)
|
||||
atLocal := ctx.GetLocalAuthToken(at)
|
||||
if err := ctx.WriteDataPoint(atLocal, ctx.Labels, r.Timestamp, r.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -19,7 +19,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson/fastfloat"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -41,7 +40,7 @@ var (
|
||||
func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
|
||||
return writeconcurrencylimiter.Do(func() error {
|
||||
return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error {
|
||||
return insertRows(at, db, rows, nil, true)
|
||||
return insertRows(at, db, rows, nil)
|
||||
})
|
||||
})
|
||||
}
|
||||
@ -61,22 +60,18 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
|
||||
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
|
||||
db := q.Get("db")
|
||||
return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error {
|
||||
return insertRows(at, db, rows, extraLabels, false)
|
||||
return insertRows(at, db, rows, extraLabels)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label, mayOverrideAccountProjectID bool) error {
|
||||
func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
|
||||
ctx := getPushCtx()
|
||||
defer putPushCtx(ctx)
|
||||
|
||||
ic := &ctx.Common
|
||||
ic.Reset() // This line is required for initializing ic internals.
|
||||
rowsTotal := 0
|
||||
var atCopy auth.Token
|
||||
if at != nil {
|
||||
atCopy = *at
|
||||
}
|
||||
perTenantRows := make(map[auth.Token]int)
|
||||
hasRelabeling := relabel.HasRelabeling()
|
||||
for i := range rows {
|
||||
@ -86,15 +81,6 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
|
||||
hasDBKey := false
|
||||
for j := range r.Tags {
|
||||
tag := &r.Tags[j]
|
||||
if mayOverrideAccountProjectID {
|
||||
// Multi-tenancy support via custom tags.
|
||||
if tag.Key == "VictoriaMetrics_AccountID" {
|
||||
atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
if tag.Key == "VictoriaMetrics_ProjectID" {
|
||||
atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
}
|
||||
if tag.Key == *dbLabel {
|
||||
hasDBKey = true
|
||||
}
|
||||
@ -133,7 +119,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
|
||||
continue
|
||||
}
|
||||
ic.SortLabelsIfNeeded()
|
||||
atLocal := ic.GetLocalAuthToken(&atCopy)
|
||||
atLocal := ic.GetLocalAuthToken(at)
|
||||
ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], atLocal.AccountID, atLocal.ProjectID, nil)
|
||||
for i := range ic.Labels {
|
||||
ic.MetricNameBuf = storage.MarshalMetricLabelRaw(ic.MetricNameBuf, &ic.Labels[i])
|
||||
@ -146,7 +132,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom
|
||||
}
|
||||
} else {
|
||||
ic.SortLabelsIfNeeded()
|
||||
atLocal := ic.GetLocalAuthToken(&atCopy)
|
||||
atLocal := ic.GetLocalAuthToken(at)
|
||||
ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], atLocal.AccountID, atLocal.ProjectID, ic.Labels)
|
||||
metricNameBufLen := len(ic.MetricNameBuf)
|
||||
labelsLen := len(ic.Labels)
|
||||
|
||||
@ -10,7 +10,6 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
"github.com/valyala/fastjson/fastfloat"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -35,10 +34,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
defer netstorage.PutInsertCtx(ctx)
|
||||
|
||||
ctx.Reset() // This line is required for initializing ctx internals.
|
||||
var atCopy auth.Token
|
||||
if at != nil {
|
||||
atCopy = *at
|
||||
}
|
||||
perTenantRows := make(map[auth.Token]int)
|
||||
hasRelabeling := relabel.HasRelabeling()
|
||||
for i := range rows {
|
||||
@ -47,16 +42,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
ctx.AddLabel("", r.Metric)
|
||||
for j := range r.Tags {
|
||||
tag := &r.Tags[j]
|
||||
if atCopy.AccountID == 0 {
|
||||
// Multi-tenancy support via custom tags.
|
||||
// Do not allow overriding AccountID and ProjectID from atCopy for security reasons.
|
||||
if tag.Key == "VictoriaMetrics_AccountID" {
|
||||
atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" {
|
||||
atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value))
|
||||
}
|
||||
}
|
||||
ctx.AddLabel(tag.Key, tag.Value)
|
||||
}
|
||||
if hasRelabeling {
|
||||
@ -67,7 +52,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error {
|
||||
continue
|
||||
}
|
||||
ctx.SortLabelsIfNeeded()
|
||||
atLocal := ctx.GetLocalAuthToken(&atCopy)
|
||||
atLocal := ctx.GetLocalAuthToken(at)
|
||||
if err := ctx.WriteDataPoint(atLocal, ctx.Labels, r.Timestamp, r.Value); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -19,6 +19,11 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
||||
|
||||
**Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval.
|
||||
|
||||
**Update note 3:** `vm_account_id` and `vm_project_id` labels must be passed to tcp-based `Graphite`, `InfluxDB` and `OpenTSDB` endpoints
|
||||
at [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) instead of undocumented
|
||||
`VictoriaMetrics_AccountID` and `VictoriaMetrics_ProjectID` labels when writing samples to the needed tenant.
|
||||
See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) for details.
|
||||
|
||||
* FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): support specifying tenant ids via `vm_account_id` and `vm_project_id` labels. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970).
|
||||
* FEATURE: improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x if non-trivial `regex` values are used.
|
||||
* FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105).
|
||||
|
||||
@ -604,7 +604,7 @@ Cluster version of VictoriaMetrics:
|
||||
<div class="with-copy" markdown="1">
|
||||
|
||||
```console
|
||||
echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2 VictoriaMetrics_AccountID=0" | nc -N http://<vminsert> 4242
|
||||
echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N http://<vminsert> 4242
|
||||
```
|
||||
|
||||
</div>
|
||||
@ -651,14 +651,12 @@ Cluster version of VictoriaMetrics:
|
||||
<div class="with-copy" markdown="1">
|
||||
|
||||
```console
|
||||
echo "foo.bar.baz;tag1=value1;tag2=value2;VictoriaMetrics_AccountID=42 123 `date +%s`" | nc -N http://<vminsert> 2003
|
||||
echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N http://<vminsert> 2003
|
||||
```
|
||||
|
||||
</div>
|
||||
|
||||
Additional information:
|
||||
|
||||
`VictoriaMetrics_AccountID=42` - [tenant ID](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) in cluster version of VictoriaMetrics
|
||||
|
||||
* [How to send Graphite data to VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd)
|
||||
* [Multitenancy in cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user