diff --git a/app/vmstorage/transport/server.go b/app/vmstorage/transport/server.go index 8df77c5575..ae2497b510 100644 --- a/app/vmstorage/transport/server.go +++ b/app/vmstorage/transport/server.go @@ -1,6 +1,7 @@ package transport import ( + "errors" "flag" "fmt" "io" @@ -381,6 +382,9 @@ func (s *Server) processVMSelectConn(bc *handshake.BufferedConn) error { // Remote client gracefully closed the connection. return nil } + if errors.Is(err, storage.ErrDeadlineExceeded) { + return fmt.Errorf("cannot process vmselect request in %d seconds: %w", ctx.timeout, err) + } return fmt.Errorf("cannot process vmselect request: %w", err) } if err := bc.Flush(); err != nil { @@ -399,6 +403,9 @@ type vmselectRequestCtx struct { sr storage.Search mb storage.MetricBlock + // timeout in seconds for the current request + timeout uint64 + // deadline in unix timestamp seconds for the current request. deadline uint64 } @@ -466,6 +473,9 @@ func (ctx *vmselectRequestCtx) writeDataBufBytes() error { const maxErrorMessageSize = 64 * 1024 func (ctx *vmselectRequestCtx) writeErrorMessage(err error) error { + if errors.Is(err, storage.ErrDeadlineExceeded) { + err = fmt.Errorf("cannot execute request in %d seconds: %w", ctx.timeout, err) + } errMsg := err.Error() if len(errMsg) > maxErrorMessageSize { // Trim too long error message. @@ -520,6 +530,7 @@ func (s *Server) processVMSelectRequest(ctx *vmselectRequestCtx) error { if err != nil { return fmt.Errorf("cannot read timeout for the request %q: %w", rpcName, err) } + ctx.timeout = uint64(timeout) ctx.deadline = fasttime.UnixTimestamp() + uint64(timeout) switch rpcName { diff --git a/lib/storage/search.go b/lib/storage/search.go index 6736da5837..ffe8c7ba6e 100644 --- a/lib/storage/search.go +++ b/lib/storage/search.go @@ -416,7 +416,7 @@ func (sq *SearchQuery) Unmarshal(src []byte) ([]byte, error) { func checkSearchDeadlineAndPace(deadline uint64) error { if fasttime.UnixTimestamp() > deadline { - return errDeadlineExceeded + return ErrDeadlineExceeded } storagepacelimiter.Search.WaitIfNeeded() return nil diff --git a/lib/storage/storage.go b/lib/storage/storage.go index acac917bcb..a219dffd6e 100644 --- a/lib/storage/storage.go +++ b/lib/storage/storage.go @@ -957,7 +957,8 @@ func (s *Storage) prefetchMetricNames(tsids []TSID, deadline uint64) error { return nil } -var errDeadlineExceeded = fmt.Errorf("deadline exceeded") +// ErrDeadlineExceeded is returned when the request times out. +var ErrDeadlineExceeded = fmt.Errorf("deadline exceeded") // DeleteMetrics deletes all the metrics matching the given tfss. //