diff --git a/app/vlinsert/insertutils/common_params.go b/app/vlinsert/insertutils/common_params.go index 97bf5c2807..a42cb3b1f9 100644 --- a/app/vlinsert/insertutils/common_params.go +++ b/app/vlinsert/insertutils/common_params.go @@ -3,12 +3,13 @@ package insertutils import ( "net/http" + "github.com/VictoriaMetrics/metrics" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage" - "github.com/VictoriaMetrics/metrics" ) // CommonParams contains common HTTP parameters used by log ingestion APIs. @@ -73,12 +74,20 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) { // GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr. func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) { return func(timestamp int64, fields []logstorage.Field) { + if len(fields) > *MaxFieldsPerLine { + var rf logstorage.RowFormatter + rf = append(rf[:0], fields...) + logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf.String()) + rowsDroppedTotalTooManyFields.Inc() + return + } + lr.MustAdd(cp.TenantID, timestamp, fields) if cp.Debug { s := lr.GetRowString(0) lr.ResetKeepSettings() logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s) - rowsDroppedTotal.Inc() + rowsDroppedTotalDebug.Inc() return } if lr.NeedFlush() { @@ -88,4 +97,5 @@ func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(ti } } -var rowsDroppedTotal = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`) +var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`) +var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`) diff --git a/app/vlinsert/insertutils/flags.go b/app/vlinsert/insertutils/flags.go index 8f070488c9..f1b28a88c3 100644 --- a/app/vlinsert/insertutils/flags.go +++ b/app/vlinsert/insertutils/flags.go @@ -1,10 +1,15 @@ package insertutils import ( + "flag" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" ) var ( // MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers") + + // MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers + MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of fields per line, which can be read by /insert/* handlers") ) diff --git a/docs/VictoriaLogs/CHANGELOG.md b/docs/VictoriaLogs/CHANGELOG.md index e4977cc510..0ba8008530 100644 --- a/docs/VictoriaLogs/CHANGELOG.md +++ b/docs/VictoriaLogs/CHANGELOG.md @@ -9,6 +9,7 @@ according to [these docs](https://docs.victoriametrics.com/VictoriaLogs/QuickSta * FEATURE: expose the following metrics at [/metrics](monitoring) page: * `vl_data_size_bytes{type="storage"}` - on-disk size for data excluding [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes. * `vl_data_size_bytes{type="indexdb"}` - on-disk size for [log stream](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields) indexes. +* FEATURE: add `-insert.maxFieldsPerLine` command-line flag, which can be used for limiting the number of fields per line in logs sent to VictoriaLogs via ingestion protocols. This helps to avoid issues like [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4762). * BUGFIX: fix possible panic when no data is written to VictoriaLogs for a long time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4895). Thanks to @crossoverJie for filing and fixing the issue. * BUGFIX: add `/insert/loky/ready` endpoint, which is used by Promtail for healthchecks. This should remove `unsupported path requested: /insert/loki/ready` warning logs. See [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4762#issuecomment-1690966722). diff --git a/docs/VictoriaLogs/README.md b/docs/VictoriaLogs/README.md index 2fb4d919c9..0990d2e187 100644 --- a/docs/VictoriaLogs/README.md +++ b/docs/VictoriaLogs/README.md @@ -171,6 +171,8 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing -inmemoryDataFlushInterval duration The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s) + -insert.maxFieldsPerLine int + The maximum number of fields per line, which can be read by /insert/* handlers (default 1000) -insert.maxLineSizeBytes size The maximum size of a single line, which can be read by /insert/* handlers Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 262144)