From 8c3e9adf7fc5d8193b6ca582c17bc4cacf244a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=82=96=E8=B4=9D=E8=B4=9D?= Date: Thu, 14 May 2020 01:58:56 +0800 Subject: [PATCH] Feat/vmalert add max queue size (#472) * feat: add remoteWrite.maxQueueSize to reduce queue full * rename remote(write|read) flags to remote(Write|Read) for the sake of consistency Co-authored-by: xiaobeibei --- app/vmalert/README.md | 3 +++ app/vmalert/main.go | 16 +++++++++------- app/vmalert/remotewrite/remotewrite.go | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index f4dc3bc1cc..4417a67467 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -94,6 +94,9 @@ Usage of vmalert: Optional basic auth password for -remotewrite.url -remotewrite.basicAuth.username string Optional basic auth username for -remotewrite.url + -remoteWrite.maxQueueSize + Optional Defines the max number of pending datapoints to remote write endpoint + -remotewrite.url string Optional URL to Victoria Metrics or VMInsert where to persist alerts state in form of timeseries. E.g. http://127.0.0.1:8428 -rule value diff --git a/app/vmalert/main.go b/app/vmalert/main.go index 0bdba96bbc..c121cf407a 100644 --- a/app/vmalert/main.go +++ b/app/vmalert/main.go @@ -37,17 +37,18 @@ absolute path to all .yaml files in root.`) basicAuthUsername = flag.String("datasource.basicAuth.username", "", "Optional basic auth username for -datasource.url") basicAuthPassword = flag.String("datasource.basicAuth.password", "", "Optional basic auth password for -datasource.url") - remoteWriteURL = flag.String("remotewrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+ + remoteWriteURL = flag.String("remoteWrite.url", "", "Optional URL to Victoria Metrics or VMInsert where to persist alerts state"+ " in form of timeseries. E.g. http://127.0.0.1:8428") - remoteWriteUsername = flag.String("remotewrite.basicAuth.username", "", "Optional basic auth username for -remotewrite.url") - remoteWritePassword = flag.String("remotewrite.basicAuth.password", "", "Optional basic auth password for -remotewrite.url") + remoteWriteUsername = flag.String("remoteWrite.basicAuth.username", "", "Optional basic auth username for -remotewrite.url") + remoteWritePassword = flag.String("remoteWrite.basicAuth.password", "", "Optional basic auth password for -remotewrite.url") + remoteWriteMaxQueueSize = flag.Int("remoteWrite.maxQueueSize", 10000, "Optional Defines the max number of pending datapoints to remote write endpoint") - remoteReadURL = flag.String("remoteread.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+ + remoteReadURL = flag.String("remoteRead.url", "", "Optional URL to Victoria Metrics or VMSelect that will be used to restore alerts"+ " state. This configuration makes sense only if `vmalert` was configured with `remotewrite.url` before and has been successfully persisted its state."+ " E.g. http://127.0.0.1:8428") - remoteReadUsername = flag.String("remoteread.basicAuth.username", "", "Optional basic auth username for -remoteread.url") - remoteReadPassword = flag.String("remoteread.basicAuth.password", "", "Optional basic auth password for -remoteread.url") - remoteReadLookBack = flag.Duration("remoteread.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+ + remoteReadUsername = flag.String("remoteRead.basicAuth.username", "", "Optional basic auth username for -remoteread.url") + remoteReadPassword = flag.String("remoteRead.basicAuth.password", "", "Optional basic auth password for -remoteread.url") + remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+ " For example, if lookback=1h then range from now() to now()-1h will be scanned.") evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules. Default 1m") @@ -77,6 +78,7 @@ func main() { if *remoteWriteURL != "" { c, err := remotewrite.NewClient(ctx, remotewrite.Config{ Addr: *remoteWriteURL, + MaxQueueSize: *remoteWriteMaxQueueSize, FlushInterval: *evaluationInterval, BasicAuthUser: *remoteWriteUsername, BasicAuthPass: *remoteWritePassword, diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go index 10586b029b..812c7a178e 100644 --- a/app/vmalert/remotewrite/remotewrite.go +++ b/app/vmalert/remotewrite/remotewrite.go @@ -103,7 +103,7 @@ func (c *Client) Push(s prompbmarshal.TimeSeries) error { case c.input <- s: return nil default: - return fmt.Errorf("failed to push timeseries - queue is full (%d entries)", + return fmt.Errorf("failed to push timeseries - queue is full (%d entries), hint from description and add recommendation to increaseremoteWrite.maxQueueSize", c.maxQueueSize) } }