From fb26b9fc72c3b18d6720303db747101907ed6ad1 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Tue, 29 Nov 2022 03:01:58 -0800 Subject: [PATCH] Default GOMAXPROCS to 1 (#2530) Avoid running on all CPUs by limiting the Go runtime to one CPU by default. Avoids having Go routines schedule on every CPU, driving up the visible run queue length on high CPU count systems. This also helps workaround a kernel deadlock issue with reading from sysfs concurrently. See: * https://github.com/prometheus/node_exporter/issues/1880 * https://github.com/prometheus/node_exporter/issues/2500 Signed-off-by: Ben Kochie --- CHANGELOG.md | 7 +++++++ node_exporter.go | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 060d5838..f63a260f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ * [ENHANCEMENT] * [BUGFIX] +NOTE: This changes the Go runtime "GOMAXPROCS" to 1. This is done to limit the + concurrency of the exporter to 1 CPU thread at a time in order to avoid a + race condition problem in the Linux kernel (#2500) and parallel IO issues + on nodes with high numbers of CPUs/CPU threads (#1880). + +* [CHANGE] Default GOMAXPROCS to 1 + ## 1.4.0 / 2022-09-24 * [CHANGE] Merge metrics descriptions in textfile collector #2475 diff --git a/node_exporter.go b/node_exporter.go index 2453153e..2f1f0c62 100644 --- a/node_exporter.go +++ b/node_exporter.go @@ -20,6 +20,7 @@ import ( _ "net/http/pprof" "os" "os/user" + "runtime" "sort" "github.com/prometheus/common/promlog" @@ -159,6 +160,9 @@ func main() { "collector.disable-defaults", "Set all collectors to disabled by default.", ).Default("false").Bool() + maxProcs = kingpin.Flag( + "runtime.gomaxprocs", "The target number of CPUs Go will run on (GOMAXPROCS)", + ).Envar("GOMAXPROCS").Default("1").Int() toolkitFlags = kingpinflag.AddFlags(kingpin.CommandLine, ":9100") ) @@ -178,6 +182,8 @@ func main() { if user, err := user.Current(); err == nil && user.Uid == "0" { level.Warn(logger).Log("msg", "Node Exporter is running as root user. This exporter is designed to run as unprivileged user, root is not required.") } + runtime.GOMAXPROCS(*maxProcs) + level.Debug(logger).Log("msg", "Go MAXPROCS", "procs", *maxProcs) http.Handle(*metricsPath, newHandler(!*disableExporterMetrics, *maxRequests, logger)) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {