app/vmstorage: prevent from serving conns from vminsert and vmselect after the server is closed

Previously it was possible that the connection is served after the server is closed if the following
steps are performed:

1) Server accepts new connection.
2) Server.MustClose() is called and successfully finished.
3) Server starts processing the connection accepted at step 1. There could be various crashes
   like in https://github.com/VictoriaMetrics/VictoriaMetrics/issues/534 since the storage may be already closed.

Now the server closes the connection at step 3 without processing it.
This commit is contained in:
Aliaksandr Valialkin 2020-06-05 11:50:11 +03:00
parent 58069f5a6a
commit e4cef1b678

View File

@ -48,18 +48,24 @@ type Server struct {
}
type connsMap struct {
mu sync.Mutex
m map[net.Conn]struct{}
mu sync.Mutex
m map[net.Conn]struct{}
isClosed bool
}
func (cm *connsMap) Init() {
cm.m = make(map[net.Conn]struct{})
cm.isClosed = false
}
func (cm *connsMap) Add(c net.Conn) {
func (cm *connsMap) Add(c net.Conn) bool {
cm.mu.Lock()
cm.m[c] = struct{}{}
ok := !cm.isClosed
if ok {
cm.m[c] = struct{}{}
}
cm.mu.Unlock()
return ok
}
func (cm *connsMap) Delete(c net.Conn) {
@ -73,6 +79,7 @@ func (cm *connsMap) CloseAll() {
for c := range cm.m {
_ = c.Close()
}
cm.isClosed = true
cm.mu.Unlock()
}
@ -116,8 +123,12 @@ func (s *Server) RunVMInsert() {
}
logger.Infof("accepted vminsert conn from %s", c.RemoteAddr())
if !s.vminsertConnsMap.Add(c) {
// The server is closed.
_ = c.Close()
return
}
vminsertConns.Inc()
s.vminsertConnsMap.Add(c)
s.vminsertWG.Add(1)
go func() {
defer func() {
@ -179,8 +190,12 @@ func (s *Server) RunVMSelect() {
}
logger.Infof("accepted vmselect conn from %s", c.RemoteAddr())
if !s.vmselectConnsMap.Add(c) {
// The server is closed.
_ = c.Close()
return
}
vmselectConns.Inc()
s.vmselectConnsMap.Add(c)
s.vmselectWG.Add(1)
go func() {
defer func() {