2019-05-22 23:16:55 +02:00
|
|
|
package concurrencylimiter
|
|
|
|
|
|
|
|
import (
|
2019-05-29 11:35:47 +02:00
|
|
|
"flag"
|
2019-05-22 23:16:55 +02:00
|
|
|
"fmt"
|
|
|
|
"runtime"
|
|
|
|
"time"
|
2019-05-28 16:17:19 +02:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
2019-05-29 11:35:47 +02:00
|
|
|
"github.com/VictoriaMetrics/metrics"
|
2019-05-22 23:16:55 +02:00
|
|
|
)
|
|
|
|
|
2019-05-29 11:35:47 +02:00
|
|
|
var maxConcurrentInserts = flag.Int("maxConcurrentInserts", runtime.GOMAXPROCS(-1)*4, "The maximum number of concurrent inserts")
|
|
|
|
|
2019-05-22 23:16:55 +02:00
|
|
|
var (
|
2019-05-29 11:35:47 +02:00
|
|
|
// ch is the channel for limiting concurrent calls to Do.
|
|
|
|
ch chan struct{}
|
2019-05-22 23:16:55 +02:00
|
|
|
|
|
|
|
// waitDuration is the amount of time to wait until at least a single
|
2019-05-29 11:35:47 +02:00
|
|
|
// concurrent Do call out of cap(ch) inserts is complete.
|
2019-05-22 23:16:55 +02:00
|
|
|
waitDuration = time.Second * 30
|
|
|
|
)
|
|
|
|
|
2019-05-29 11:35:47 +02:00
|
|
|
// Init initializes concurrencylimiter.
|
|
|
|
//
|
|
|
|
// Init must be called after flag.Parse call.
|
|
|
|
func Init() {
|
|
|
|
ch = make(chan struct{}, *maxConcurrentInserts)
|
|
|
|
}
|
|
|
|
|
2019-05-22 23:16:55 +02:00
|
|
|
// Do calls f with the limited concurrency.
|
|
|
|
func Do(f func() error) error {
|
2019-05-29 11:35:47 +02:00
|
|
|
// Limit the number of conurrent f calls in order to prevent from excess
|
2019-05-22 23:16:55 +02:00
|
|
|
// memory usage and CPU trashing.
|
2019-05-28 16:17:19 +02:00
|
|
|
t := timerpool.Get(waitDuration)
|
2019-05-22 23:16:55 +02:00
|
|
|
select {
|
|
|
|
case ch <- struct{}{}:
|
2019-05-28 16:17:19 +02:00
|
|
|
timerpool.Put(t)
|
2019-05-22 23:16:55 +02:00
|
|
|
err := f()
|
|
|
|
<-ch
|
|
|
|
return err
|
|
|
|
case <-t.C:
|
2019-05-28 16:17:19 +02:00
|
|
|
timerpool.Put(t)
|
2019-05-29 11:35:47 +02:00
|
|
|
concurrencyLimitErrors.Inc()
|
2019-06-08 21:43:08 +02:00
|
|
|
return fmt.Errorf("the server is overloaded with %d concurrent inserts; either increase -maxConcurrentInserts or reduce the load", cap(ch))
|
2019-05-22 23:16:55 +02:00
|
|
|
}
|
|
|
|
}
|
2019-05-29 11:35:47 +02:00
|
|
|
|
|
|
|
var concurrencyLimitErrors = metrics.NewCounter(`vm_concurrency_limit_errors_total`)
|