lib/promscrape: add -promscrape.configCheckInterval command-line flag for automating config checking

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/431

lib/promscrape/config.go

@@ -97,16 +97,16 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
 }
 
 // loadConfig loads Prometheus config from the given path.
-func loadConfig(path string) (cfg *Config, err error) {
-	data, err := ioutil.ReadFile(path)
+func loadConfig(path string) (cfg *Config, data []byte, err error) {
+	data, err = ioutil.ReadFile(path)
 	if err != nil {
-		return nil, fmt.Errorf("cannot read Prometheus config from %q: %s", path, err)
+		return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %s", path, err)
 	}
 	var cfgObj Config
 	if err := cfgObj.parse(data, path); err != nil {
-		return nil, fmt.Errorf("cannot parse Prometheus config from %q: %s", path, err)
+		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %s", path, err)
 	}
-	return &cfgObj, nil
+	return &cfgObj, data, nil
 }
 
 func (cfg *Config) parse(data []byte, path string) error {

lib/promscrape/config_test.go

@@ -42,7 +42,7 @@ func TestLoadStaticConfigs(t *testing.T) {
 }
 
 func TestLoadConfig(t *testing.T) {
-	cfg, err := loadConfig("testdata/prometheus.yml")
+	cfg, _, err := loadConfig("testdata/prometheus.yml")
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -51,7 +51,7 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	// Try loading non-existing file
-	cfg, err = loadConfig("testdata/non-existing-file")
+	cfg, _, err = loadConfig("testdata/non-existing-file")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}
@@ -60,7 +60,7 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	// Try loading invalid file
-	cfg, err = loadConfig("testdata/file_sd_1.yml")
+	cfg, _, err = loadConfig("testdata/file_sd_1.yml")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}

lib/promscrape/scraper.go

@@ -1,6 +1,7 @@
 package promscrape
 
 import (
+	"bytes"
 	"flag"
 	"os"
 	"os/signal"
@@ -14,6 +15,8 @@ import (
 )
 
 var (
+	configCheckInterval = flag.Duration("promscrape.configCheckInterval", 0, "Interval for checking for changes in '-promscrape.config' file. "+
+		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes")
 	fileSDCheckInterval = flag.Duration("promscrape.fileSDCheckInterval", 30*time.Second, "Interval for checking for changes in 'file_sd_config'. "+
 		"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config")
 	kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+
@@ -55,11 +58,18 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	signal.Notify(sighupCh, syscall.SIGHUP)
 
 	logger.Infof("reading Prometheus configs from %q", configFile)
-	cfg, err := loadConfig(configFile)
+	cfg, data, err := loadConfig(configFile)
 	if err != nil {
 		logger.Fatalf("cannot read %q: %s", configFile, err)
 	}
 
+	var tickerCh <-chan time.Time
+	if *configCheckInterval > 0 {
+		ticker := time.NewTicker(*configCheckInterval)
+		tickerCh = ticker.C
+		defer ticker.Stop()
+	}
+
 	mustStop := false
 	for !mustStop {
 		stopCh := make(chan struct{})
@@ -84,16 +94,36 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 		select {
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile)
-			cfgNew, err := loadConfig(configFile)
+			cfgNew, dataNew, err := loadConfig(configFile)
+			if err != nil {
+				logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err)
+				goto waitForChans
+			}
+			if bytes.Equal(data, dataNew) {
+				logger.Infof("nothing changed in %q", configFile)
+				goto waitForChans
+			}
+			cfg = cfgNew
+			data = dataNew
+		case <-tickerCh:
+			cfgNew, dataNew, err := loadConfig(configFile)
 			if err != nil {
 				logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err)
 				goto waitForChans
 			}
+			if bytes.Equal(data, dataNew) {
+				// Nothing changed since the previous loadConfig
+				goto waitForChans
+			}
 			cfg = cfgNew
+			data = dataNew
 		case <-globalStopCh:
 			mustStop = true
 		}
 
+		if !mustStop {
+			logger.Infof("found changes in %q; applying these changes", configFile)
+		}
 		logger.Infof("stopping Prometheus scrapers")
 		startTime := time.Now()
 		close(stopCh)