lib/logger: prevent from blocking when log output isn't consumed in timely manner

Drop log messages instead of blocking and increment `vm_log_messages_dropped_total` metric.

lib/logger/logger.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"runtime"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -30,6 +29,7 @@ func Init() {
 	validateLoggerLevel()
 	validateLoggerFormat()
 	go errorsLoggedCleaner()
+	go logMessageWriter()
 	logAllFlags()
 }
 
@@ -137,10 +137,13 @@ func logMessage(level, msg string, skipframes int) {
 		logMsg = fmt.Sprintf("%s\t%s\t%s:%d\t%s\n", timestamp, levelLowercase, file, line, msg)
 	}
 
-	// Serialize writes to log.
-	mu.Lock()
-	fmt.Fprint(os.Stderr, logMsg)
-	mu.Unlock()
+	select {
+	case logMessageCh <- logMsg:
+	default:
+		// Writing to log can stuck if the log output isn't consumed in timely manner.
+		// Handle this case via `vm_log_messages_dropped_total`
+		logMessagesDropped.Inc()
+	}
 
 	// Increment vm_log_messages_total
 	location := fmt.Sprintf("%s:%d", file, line)
@@ -155,7 +158,15 @@ func logMessage(level, msg string, skipframes int) {
 	}
 }
 
-var mu sync.Mutex
+var logMessagesDropped = metrics.NewCounter(`vm_log_messages_dropped_total`)
+
+func logMessageWriter() {
+	for msg := range logMessageCh {
+		fmt.Fprint(os.Stderr, msg)
+	}
+}
+
+var logMessageCh = make(chan string, 100)
 
 func shouldSkipLog(level string) bool {
 	switch *loggerLevel {