From c8000c029a86c35618ca6a996d91d67c0496f838 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 30 Sep 2022 18:04:48 +0300 Subject: [PATCH] app/vminsert: remove support for undocumented VictoriaMetrics_AccountID and VictoriaMetrics_ProjectID labels in tcp-based data ingestion endpoints These labels are substituted by documented vm_account_id and vm_project_id labels. See https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels This is a follow up for 505d359b397d3594a996253d49635ec83fd949d4 --- app/vminsert/graphite/request_handler.go | 17 +---------------- app/vminsert/influx/request_handler.go | 24 +++++------------------- app/vminsert/opentsdb/request_handler.go | 17 +---------------- docs/CHANGELOG.md | 5 +++++ docs/url-examples.md | 6 ++---- 5 files changed, 14 insertions(+), 55 deletions(-) diff --git a/app/vminsert/graphite/request_handler.go b/app/vminsert/graphite/request_handler.go index 9cbf79afea..484886bcbd 100644 --- a/app/vminsert/graphite/request_handler.go +++ b/app/vminsert/graphite/request_handler.go @@ -10,7 +10,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" "github.com/VictoriaMetrics/metrics" - "github.com/valyala/fastjson/fastfloat" ) var ( @@ -35,10 +34,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error { defer netstorage.PutInsertCtx(ctx) ctx.Reset() // This line is required for initializing ctx internals. - var atCopy auth.Token - if at != nil { - atCopy = *at - } perTenantRows := make(map[auth.Token]int) hasRelabeling := relabel.HasRelabeling() for i := range rows { @@ -47,16 +42,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error { ctx.AddLabel("", r.Metric) for j := range r.Tags { tag := &r.Tags[j] - if atCopy.AccountID == 0 { - // Multi-tenancy support via custom tags. - // Do not allow overriding AccountID and ProjectID from atCopy for security reasons. - if tag.Key == "VictoriaMetrics_AccountID" { - atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" { - atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - } ctx.AddLabel(tag.Key, tag.Value) } if hasRelabeling { @@ -67,7 +52,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error { continue } ctx.SortLabelsIfNeeded() - atLocal := ctx.GetLocalAuthToken(&atCopy) + atLocal := ctx.GetLocalAuthToken(at) if err := ctx.WriteDataPoint(atLocal, ctx.Labels, r.Timestamp, r.Value); err != nil { return err } diff --git a/app/vminsert/influx/request_handler.go b/app/vminsert/influx/request_handler.go index a3cd05194c..72eebb5ad6 100644 --- a/app/vminsert/influx/request_handler.go +++ b/app/vminsert/influx/request_handler.go @@ -19,7 +19,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" "github.com/VictoriaMetrics/metrics" - "github.com/valyala/fastjson/fastfloat" ) var ( @@ -41,7 +40,7 @@ var ( func InsertHandlerForReader(at *auth.Token, r io.Reader) error { return writeconcurrencylimiter.Do(func() error { return parser.ParseStream(r, false, "", "", func(db string, rows []parser.Row) error { - return insertRows(at, db, rows, nil, true) + return insertRows(at, db, rows, nil) }) }) } @@ -61,22 +60,18 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error { // Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint db := q.Get("db") return parser.ParseStream(req.Body, isGzipped, precision, db, func(db string, rows []parser.Row) error { - return insertRows(at, db, rows, extraLabels, false) + return insertRows(at, db, rows, extraLabels) }) }) } -func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label, mayOverrideAccountProjectID bool) error { +func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prompbmarshal.Label) error { ctx := getPushCtx() defer putPushCtx(ctx) ic := &ctx.Common ic.Reset() // This line is required for initializing ic internals. rowsTotal := 0 - var atCopy auth.Token - if at != nil { - atCopy = *at - } perTenantRows := make(map[auth.Token]int) hasRelabeling := relabel.HasRelabeling() for i := range rows { @@ -86,15 +81,6 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom hasDBKey := false for j := range r.Tags { tag := &r.Tags[j] - if mayOverrideAccountProjectID { - // Multi-tenancy support via custom tags. - if tag.Key == "VictoriaMetrics_AccountID" { - atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - if tag.Key == "VictoriaMetrics_ProjectID" { - atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - } if tag.Key == *dbLabel { hasDBKey = true } @@ -133,7 +119,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom continue } ic.SortLabelsIfNeeded() - atLocal := ic.GetLocalAuthToken(&atCopy) + atLocal := ic.GetLocalAuthToken(at) ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], atLocal.AccountID, atLocal.ProjectID, nil) for i := range ic.Labels { ic.MetricNameBuf = storage.MarshalMetricLabelRaw(ic.MetricNameBuf, &ic.Labels[i]) @@ -146,7 +132,7 @@ func insertRows(at *auth.Token, db string, rows []parser.Row, extraLabels []prom } } else { ic.SortLabelsIfNeeded() - atLocal := ic.GetLocalAuthToken(&atCopy) + atLocal := ic.GetLocalAuthToken(at) ic.MetricNameBuf = storage.MarshalMetricNameRaw(ic.MetricNameBuf[:0], atLocal.AccountID, atLocal.ProjectID, ic.Labels) metricNameBufLen := len(ic.MetricNameBuf) labelsLen := len(ic.Labels) diff --git a/app/vminsert/opentsdb/request_handler.go b/app/vminsert/opentsdb/request_handler.go index 76ff9e4ef6..0e111c0e1e 100644 --- a/app/vminsert/opentsdb/request_handler.go +++ b/app/vminsert/opentsdb/request_handler.go @@ -10,7 +10,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter" "github.com/VictoriaMetrics/metrics" - "github.com/valyala/fastjson/fastfloat" ) var ( @@ -35,10 +34,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error { defer netstorage.PutInsertCtx(ctx) ctx.Reset() // This line is required for initializing ctx internals. - var atCopy auth.Token - if at != nil { - atCopy = *at - } perTenantRows := make(map[auth.Token]int) hasRelabeling := relabel.HasRelabeling() for i := range rows { @@ -47,16 +42,6 @@ func insertRows(at *auth.Token, rows []parser.Row) error { ctx.AddLabel("", r.Metric) for j := range r.Tags { tag := &r.Tags[j] - if atCopy.AccountID == 0 { - // Multi-tenancy support via custom tags. - // Do not allow overriding AccountID and ProjectID from atCopy for security reasons. - if tag.Key == "VictoriaMetrics_AccountID" { - atCopy.AccountID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - if atCopy.ProjectID == 0 && tag.Key == "VictoriaMetrics_ProjectID" { - atCopy.ProjectID = uint32(fastfloat.ParseUint64BestEffort(tag.Value)) - } - } ctx.AddLabel(tag.Key, tag.Value) } if hasRelabeling { @@ -67,7 +52,7 @@ func insertRows(at *auth.Token, rows []parser.Row) error { continue } ctx.SortLabelsIfNeeded() - atLocal := ctx.GetLocalAuthToken(&atCopy) + atLocal := ctx.GetLocalAuthToken(at) if err := ctx.WriteDataPoint(atLocal, ctx.Labels, r.Timestamp, r.Value); err != nil { return err } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5a7d17c170..170a237d4f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,11 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update note 2:** [vmalert](https://docs.victoriametrics.com/vmalert.html) changes default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +**Update note 3:** `vm_account_id` and `vm_project_id` labels must be passed to tcp-based `Graphite`, `InfluxDB` and `OpenTSDB` endpoints +at [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) instead of undocumented +`VictoriaMetrics_AccountID` and `VictoriaMetrics_ProjectID` labels when writing samples to the needed tenant. +See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) for details. + * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): support specifying tenant ids via `vm_account_id` and `vm_project_id` labels. See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy-via-labels) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2970). * FEATURE: improve [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) performance by up to 3x if non-trivial `regex` values are used. * FEATURE: sanitize metric names for data ingested via [DataDog protocol](https://docs.victoriametrics.com/#how-to-send-data-from-datadog-agent) according to [DataDog metric naming](https://docs.datadoghq.com/metrics/custom_metrics/#naming-custom-metrics). The behaviour can be disabled by passing `-datadog.sanitizeMetricName=false` command-line flag. Thanks to @PerGon for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3105). diff --git a/docs/url-examples.md b/docs/url-examples.md index 35e6107651..9f0ac3e571 100644 --- a/docs/url-examples.md +++ b/docs/url-examples.md @@ -604,7 +604,7 @@ Cluster version of VictoriaMetrics:
```console -echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2 VictoriaMetrics_AccountID=0" | nc -N http:// 4242 +echo "put foo.bar.baz `date +%s` 123 tag1=value1 tag2=value2" | nc -N http:// 4242 ```
@@ -651,14 +651,12 @@ Cluster version of VictoriaMetrics:
```console -echo "foo.bar.baz;tag1=value1;tag2=value2;VictoriaMetrics_AccountID=42 123 `date +%s`" | nc -N http:// 2003 +echo "foo.bar.baz;tag1=value1;tag2=value2 123 `date +%s`" | nc -N http:// 2003 ```
Additional information: -`VictoriaMetrics_AccountID=42` - [tenant ID](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy) in cluster version of VictoriaMetrics - * [How to send Graphite data to VictoriaMetrics](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-send-data-from-graphite-compatible-agents-such-as-statsd) * [Multitenancy in cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenancy)