diff --git a/lib/logger/logger.go b/lib/logger/logger.go index f886427f0f..8f820b6229 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -8,7 +8,6 @@ import ( "os" "runtime" "strings" - "sync" "sync/atomic" "time" @@ -30,6 +29,7 @@ func Init() { validateLoggerLevel() validateLoggerFormat() go errorsLoggedCleaner() + go logMessageWriter() logAllFlags() } @@ -137,10 +137,13 @@ func logMessage(level, msg string, skipframes int) { logMsg = fmt.Sprintf("%s\t%s\t%s:%d\t%s\n", timestamp, levelLowercase, file, line, msg) } - // Serialize writes to log. - mu.Lock() - fmt.Fprint(os.Stderr, logMsg) - mu.Unlock() + select { + case logMessageCh <- logMsg: + default: + // Writing to log can stuck if the log output isn't consumed in timely manner. + // Handle this case via `vm_log_messages_dropped_total` + logMessagesDropped.Inc() + } // Increment vm_log_messages_total location := fmt.Sprintf("%s:%d", file, line) @@ -155,7 +158,15 @@ func logMessage(level, msg string, skipframes int) { } } -var mu sync.Mutex +var logMessagesDropped = metrics.NewCounter(`vm_log_messages_dropped_total`) + +func logMessageWriter() { + for msg := range logMessageCh { + fmt.Fprint(os.Stderr, msg) + } +} + +var logMessageCh = make(chan string, 100) func shouldSkipLog(level string) bool { switch *loggerLevel {