mirror of
https://github.com/prometheus/node_exporter.git
synced 2024-11-23 20:36:21 +01:00
ffefc8e74d
In order to avoid stuck collectors using up all system resources, add a limit to the number of parallel in-flight scrape requests. This will return a 503 error. Default to 40 requests, this seems like a reasonable number based on: * Two Prometheus servers scraping every 15 seconds. * Failing scrapes after 5 minutes of stuckness. Signed-off-by: Ben Kochie <superq@gmail.com>
175 lines
5.4 KiB
Go
175 lines
5.4 KiB
Go
// Copyright 2015 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
_ "net/http/pprof"
|
|
"sort"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
"github.com/prometheus/common/log"
|
|
"github.com/prometheus/common/version"
|
|
"github.com/prometheus/node_exporter/collector"
|
|
"gopkg.in/alecthomas/kingpin.v2"
|
|
)
|
|
|
|
// handler wraps an unfiltered http.Handler but uses a filtered handler,
|
|
// created on the fly, if filtering is requested. Create instances with
|
|
// newHandler.
|
|
type handler struct {
|
|
unfilteredHandler http.Handler
|
|
// exporterMetricsRegistry is a separate registry for the metrics about
|
|
// the exporter itself.
|
|
exporterMetricsRegistry *prometheus.Registry
|
|
includeExporterMetrics bool
|
|
maxRequests int
|
|
}
|
|
|
|
func newHandler(includeExporterMetrics bool, maxRequests int) *handler {
|
|
h := &handler{
|
|
exporterMetricsRegistry: prometheus.NewRegistry(),
|
|
includeExporterMetrics: includeExporterMetrics,
|
|
maxRequests: maxRequests,
|
|
}
|
|
if h.includeExporterMetrics {
|
|
h.exporterMetricsRegistry.MustRegister(
|
|
prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
|
|
prometheus.NewGoCollector(),
|
|
)
|
|
}
|
|
if innerHandler, err := h.innerHandler(); err != nil {
|
|
log.Fatalf("Couldn't create metrics handler: %s", err)
|
|
} else {
|
|
h.unfilteredHandler = innerHandler
|
|
}
|
|
return h
|
|
}
|
|
|
|
// ServeHTTP implements http.Handler.
|
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
filters := r.URL.Query()["collect[]"]
|
|
log.Debugln("collect query:", filters)
|
|
|
|
if len(filters) == 0 {
|
|
// No filters, use the prepared unfiltered handler.
|
|
h.unfilteredHandler.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
// To serve filtered metrics, we create a filtering handler on the fly.
|
|
filteredHandler, err := h.innerHandler(filters...)
|
|
if err != nil {
|
|
log.Warnln("Couldn't create filtered metrics handler:", err)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(fmt.Sprintf("Couldn't create filtered metrics handler: %s", err)))
|
|
return
|
|
}
|
|
filteredHandler.ServeHTTP(w, r)
|
|
}
|
|
|
|
// innerHandler is used to create buth the one unfiltered http.Handler to be
|
|
// wrapped by the outer handler and also the filtered handlers created on the
|
|
// fly. The former is accomplished by calling innerHandler without any arguments
|
|
// (in which case it will log all the collectors enabled via command-line
|
|
// flags).
|
|
func (h *handler) innerHandler(filters ...string) (http.Handler, error) {
|
|
nc, err := collector.NewNodeCollector(filters...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("couldn't create collector: %s", err)
|
|
}
|
|
|
|
// Only log the creation of an unfiltered handler, which should happen
|
|
// only once upon startup.
|
|
if len(filters) == 0 {
|
|
log.Infof("Enabled collectors:")
|
|
collectors := []string{}
|
|
for n := range nc.Collectors {
|
|
collectors = append(collectors, n)
|
|
}
|
|
sort.Strings(collectors)
|
|
for _, n := range collectors {
|
|
log.Infof(" - %s", n)
|
|
}
|
|
}
|
|
|
|
r := prometheus.NewRegistry()
|
|
r.MustRegister(version.NewCollector("node_exporter"))
|
|
if err := r.Register(nc); err != nil {
|
|
return nil, fmt.Errorf("couldn't register node collector: %s", err)
|
|
}
|
|
handler := promhttp.HandlerFor(
|
|
prometheus.Gatherers{h.exporterMetricsRegistry, r},
|
|
promhttp.HandlerOpts{
|
|
ErrorLog: log.NewErrorLogger(),
|
|
ErrorHandling: promhttp.ContinueOnError,
|
|
MaxRequestsInFlight: h.maxRequests,
|
|
},
|
|
)
|
|
if h.includeExporterMetrics {
|
|
// Note that we have to use h.exporterMetricsRegistry here to
|
|
// use the same promhttp metrics for all expositions.
|
|
handler = promhttp.InstrumentMetricHandler(
|
|
h.exporterMetricsRegistry, handler,
|
|
)
|
|
}
|
|
return handler, nil
|
|
}
|
|
|
|
func main() {
|
|
var (
|
|
listenAddress = kingpin.Flag(
|
|
"web.listen-address",
|
|
"Address on which to expose metrics and web interface.",
|
|
).Default(":9100").String()
|
|
metricsPath = kingpin.Flag(
|
|
"web.telemetry-path",
|
|
"Path under which to expose metrics.",
|
|
).Default("/metrics").String()
|
|
disableExporterMetrics = kingpin.Flag(
|
|
"web.disable-exporter-metrics",
|
|
"Exclude metrics about the exporter itself (promhttp_*, process_*, go_*).",
|
|
).Bool()
|
|
maxRequests = kingpin.Flag(
|
|
"web.max-requests",
|
|
"Maximum number of parallel scrape requests. Use 0 to disable.",
|
|
).Default("40").Int()
|
|
)
|
|
|
|
log.AddFlags(kingpin.CommandLine)
|
|
kingpin.Version(version.Print("node_exporter"))
|
|
kingpin.HelpFlag.Short('h')
|
|
kingpin.Parse()
|
|
|
|
log.Infoln("Starting node_exporter", version.Info())
|
|
log.Infoln("Build context", version.BuildContext())
|
|
|
|
http.Handle(*metricsPath, newHandler(!*disableExporterMetrics, *maxRequests))
|
|
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Write([]byte(`<html>
|
|
<head><title>Node Exporter</title></head>
|
|
<body>
|
|
<h1>Node Exporter</h1>
|
|
<p><a href="` + *metricsPath + `">Metrics</a></p>
|
|
</body>
|
|
</html>`))
|
|
})
|
|
|
|
log.Infoln("Listening on", *listenAddress)
|
|
if err := http.ListenAndServe(*listenAddress, nil); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|