From fbe2795670db951ba9a9e4f81f6fec6bffb0d9c0 Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 30 Aug 2023 16:24:24 +0200 Subject: [PATCH] app/vminsert: fixes readonly check (#4892) * app/vminsert: fixes readonly check previously vminsert doesn't check readOnly state for vmstorage, since check was never performed for nil buffer In this case every 30 second storage node loss readonly state and received some data. It caused re-routing and possible slow down for ingestion https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4870 * wip --------- Co-authored-by: Aliaksandr Valialkin --- app/vminsert/netstorage/netstorage.go | 7 +++---- docs/CHANGELOG.md | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/vminsert/netstorage/netstorage.go b/app/vminsert/netstorage/netstorage.go index 4b2584a637..996e275d65 100644 --- a/app/vminsert/netstorage/netstorage.go +++ b/app/vminsert/netstorage/netstorage.go @@ -325,10 +325,9 @@ var cannotCloseStorageNodeConnLogger = logger.WithThrottler("cannotCloseStorageN var cannotSendBufsLogger = logger.WithThrottler("cannotSendBufRows", 5*time.Second) func sendToConn(bc *handshake.BufferedConn, buf []byte) error { - if len(buf) == 0 { - // Nothing to send - return nil - } + // if len(buf) == 0, it must be sent to the vmstorage too in order to check for vmstorage health + // See checkReadOnlyMode() and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4870 + timeoutSeconds := len(buf) / 3e5 if timeoutSeconds < 60 { timeoutSeconds = 60 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f38f7b0626..304ed980de 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -42,7 +42,8 @@ The following `tip` changes can be tested by building VictoriaMetrics components * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): consistently set timeout for scraping with enabled or disabled [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4847). * BUGFIX: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): fix display of ingested rows rate for `Samples ingested/s` and `Samples rate` panels for vmagent's dasbhoard. Previously, not all ingested protocols were accounted in these panels. An extra panel `Rows rate` was added to `Ingestion` section to display the split for rows ingested rate by protocol. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): correctly re-use HTTP request object on `EOF` retries when querying the configured datasource. Previously, there was a small chance that query retry wouldn't succeed. -* BUGFIX: vmselect: correctly handle requests with `/select/multitenant` prefix. Such requests must be rejected since vmselect does not support multitenancy endpoint. Previously, such requests were causing panic. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4910). +* BUGFIX: [vmselect](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): correctly handle requests with `/select/multitenant` prefix. Such requests must be rejected since vmselect does not support multitenancy endpoint. Previously, such requests were causing panic. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4910). +* BUGFIX: [vminsert](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): properly check for [read-only state](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode) at `vmstorage`. Previously it wasn't properly checked, which could lead to increased resource usage and data ingestion slowdown when some of `vmstorage` nodes are in read-only mode. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4870). ## [v1.93.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.1)