diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 03e5a6971d..28ceda4d56 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -97,16 +97,16 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) { } // loadConfig loads Prometheus config from the given path. -func loadConfig(path string) (cfg *Config, err error) { - data, err := ioutil.ReadFile(path) +func loadConfig(path string) (cfg *Config, data []byte, err error) { + data, err = ioutil.ReadFile(path) if err != nil { - return nil, fmt.Errorf("cannot read Prometheus config from %q: %s", path, err) + return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %s", path, err) } var cfgObj Config if err := cfgObj.parse(data, path); err != nil { - return nil, fmt.Errorf("cannot parse Prometheus config from %q: %s", path, err) + return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %s", path, err) } - return &cfgObj, nil + return &cfgObj, data, nil } func (cfg *Config) parse(data []byte, path string) error { diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index f7612146d9..aa696f0568 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -42,7 +42,7 @@ func TestLoadStaticConfigs(t *testing.T) { } func TestLoadConfig(t *testing.T) { - cfg, err := loadConfig("testdata/prometheus.yml") + cfg, _, err := loadConfig("testdata/prometheus.yml") if err != nil { t.Fatalf("unexpected error: %s", err) } @@ -51,7 +51,7 @@ func TestLoadConfig(t *testing.T) { } // Try loading non-existing file - cfg, err = loadConfig("testdata/non-existing-file") + cfg, _, err = loadConfig("testdata/non-existing-file") if err == nil { t.Fatalf("expecting non-nil error") } @@ -60,7 +60,7 @@ func TestLoadConfig(t *testing.T) { } // Try loading invalid file - cfg, err = loadConfig("testdata/file_sd_1.yml") + cfg, _, err = loadConfig("testdata/file_sd_1.yml") if err == nil { t.Fatalf("expecting non-nil error") } diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index b86488394b..643d55d743 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -1,6 +1,7 @@ package promscrape import ( + "bytes" "flag" "os" "os/signal" @@ -14,6 +15,8 @@ import ( ) var ( + configCheckInterval = flag.Duration("promscrape.configCheckInterval", 0, "Interval for checking for changes in '-promscrape.config' file. "+ + "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes") fileSDCheckInterval = flag.Duration("promscrape.fileSDCheckInterval", 30*time.Second, "Interval for checking for changes in 'file_sd_config'. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config") kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+ @@ -55,11 +58,18 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) signal.Notify(sighupCh, syscall.SIGHUP) logger.Infof("reading Prometheus configs from %q", configFile) - cfg, err := loadConfig(configFile) + cfg, data, err := loadConfig(configFile) if err != nil { logger.Fatalf("cannot read %q: %s", configFile, err) } + var tickerCh <-chan time.Time + if *configCheckInterval > 0 { + ticker := time.NewTicker(*configCheckInterval) + tickerCh = ticker.C + defer ticker.Stop() + } + mustStop := false for !mustStop { stopCh := make(chan struct{}) @@ -84,16 +94,36 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) select { case <-sighupCh: logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile) - cfgNew, err := loadConfig(configFile) + cfgNew, dataNew, err := loadConfig(configFile) + if err != nil { + logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err) + goto waitForChans + } + if bytes.Equal(data, dataNew) { + logger.Infof("nothing changed in %q", configFile) + goto waitForChans + } + cfg = cfgNew + data = dataNew + case <-tickerCh: + cfgNew, dataNew, err := loadConfig(configFile) if err != nil { logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err) goto waitForChans } + if bytes.Equal(data, dataNew) { + // Nothing changed since the previous loadConfig + goto waitForChans + } cfg = cfgNew + data = dataNew case <-globalStopCh: mustStop = true } + if !mustStop { + logger.Infof("found changes in %q; applying these changes", configFile) + } logger.Infof("stopping Prometheus scrapers") startTime := time.Now() close(stopCh)