lib/promscrape: add ability to load scrape configs from multiple files

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559

docs/CHANGELOG.md

@@ -6,6 +6,7 @@ sort: 15
 
 ## tip
 
+* FEATURE: vmagent: add ability to read scrape configs from multiple files specified in `scrape_config_files` section. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559).
 * FEATURE: vmagent: reduce memory usage and CPU usage when Prometheus staleness tracking is enabled for metrics exported from the deleted or disappeared scrape targets.
 * FEATURE: take into account failed queries in `vm_request_duration_seconds` summary at `/metrics`. Previously only successful queries were taken into account. This could result in skewed summary. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1537).
 * FEATURE: vmalert: add `-disableAlertgroupLabel` command-line flag for disabling the label with alert group name. This may be needed for proper deduplication in Alertmanager. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1532).

lib/promscrape/config.go

@@ -56,13 +56,22 @@ var (
 
 // Config represents essential parts from Prometheus config defined at https://prometheus.io/docs/prometheus/latest/configuration/configuration/
 type Config struct {
-	Global        GlobalConfig   `yaml:"global"`
-	ScrapeConfigs []ScrapeConfig `yaml:"scrape_configs"`
+	Global            GlobalConfig   `yaml:"global,omitempty"`
+	ScrapeConfigs     []ScrapeConfig `yaml:"scrape_configs"`
+	ScrapeConfigFiles []string       `yaml:"scrape_config_files"`
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
 }
 
+func (cfg *Config) marshal() []byte {
+	data, err := yaml.Marshal(cfg)
+	if err != nil {
+		logger.Panicf("BUG: cannot marshal Config: %s", err)
+	}
+	return data
+}
+
 func (cfg *Config) mustStart() {
 	startTime := time.Now()
 	logger.Infof("starting service discovery routines...")
@@ -229,16 +238,45 @@ func loadStaticConfigs(path string) ([]StaticConfig, error) {
 }
 
 // loadConfig loads Prometheus config from the given path.
-func loadConfig(path string) (cfg *Config, data []byte, err error) {
-	data, err = ioutil.ReadFile(path)
+func loadConfig(path string) (*Config, error) {
+	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
+		return nil, fmt.Errorf("cannot read Prometheus config from %q: %w", path, err)
 	}
-	var cfgObj Config
-	if err := cfgObj.parse(data, path); err != nil {
-		return nil, nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
+	var c Config
+	if err := c.parseData(data, path); err != nil {
+		return nil, fmt.Errorf("cannot parse Prometheus config from %q: %w", path, err)
 	}
-	return &cfgObj, data, nil
+	return &c, nil
+}
+
+func loadScrapeConfigFiles(baseDir string, scrapeConfigFiles []string) ([]ScrapeConfig, error) {
+	var scrapeConfigs []ScrapeConfig
+	for _, filePath := range scrapeConfigFiles {
+		filePath := getFilepath(baseDir, filePath)
+		paths := []string{filePath}
+		if strings.Contains(filePath, "*") {
+			ps, err := filepath.Glob(filePath)
+			if err != nil {
+				return nil, fmt.Errorf("invalid pattern %q in `scrape_config_files`: %w", filePath, err)
+			}
+			sort.Strings(ps)
+			paths = ps
+		}
+		for _, path := range paths {
+			data, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, fmt.Errorf("cannot load %q from `scrape_config_files`: %w", filePath, err)
+			}
+			data = envtemplate.Replace(data)
+			var scs []ScrapeConfig
+			if err = yaml.UnmarshalStrict(data, &scs); err != nil {
+				return nil, fmt.Errorf("cannot parse %q from `scrape_config_files`: %w", filePath, err)
+			}
+			scrapeConfigs = append(scrapeConfigs, scs...)
+		}
+	}
+	return scrapeConfigs, nil
 }
 
 // IsDryRun returns true if -promscrape.config.dryRun command-line flag is set
@@ -246,7 +284,7 @@ func IsDryRun() bool {
 	return *dryRun
 }
 
-func (cfg *Config) parse(data []byte, path string) error {
+func (cfg *Config) parseData(data []byte, path string) error {
 	if err := unmarshalMaybeStrict(data, cfg); err != nil {
 		return fmt.Errorf("cannot unmarshal data: %w", err)
 	}
@@ -255,6 +293,26 @@ func (cfg *Config) parse(data []byte, path string) error {
 		return fmt.Errorf("cannot obtain abs path for %q: %w", path, err)
 	}
 	cfg.baseDir = filepath.Dir(absPath)
+
+	// Load cfg.ScrapeConfigFiles into c.ScrapeConfigs
+	scs, err := loadScrapeConfigFiles(cfg.baseDir, cfg.ScrapeConfigFiles)
+	if err != nil {
+		return fmt.Errorf("cannot load `scrape_config_files` from %q: %w", path, err)
+	}
+	cfg.ScrapeConfigFiles = nil
+	cfg.ScrapeConfigs = append(cfg.ScrapeConfigs, scs...)
+
+	// Check that all the scrape configs have unique JobName
+	m := make(map[string]struct{}, len(cfg.ScrapeConfigs))
+	for i := range cfg.ScrapeConfigs {
+		jobName := cfg.ScrapeConfigs[i].JobName
+		if _, ok := m[jobName]; ok {
+			return fmt.Errorf("duplicate `job_name` in `scrape_configs` loaded from %q: %q", path, jobName)
+		}
+		m[jobName] = struct{}{}
+	}
+
+	// Initialize cfg.ScrapeConfigs
 	for i := range cfg.ScrapeConfigs {
 		sc := &cfg.ScrapeConfigs[i]
 		swc, err := getScrapeWorkConfig(sc, cfg.baseDir, &cfg.Global)

lib/promscrape/config_test.go

@@ -67,7 +67,15 @@ func TestLoadStaticConfigs(t *testing.T) {
 }
 
 func TestLoadConfig(t *testing.T) {
-	cfg, _, err := loadConfig("testdata/prometheus.yml")
+	cfg, err := loadConfig("testdata/prometheus.yml")
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if cfg == nil {
+		t.Fatalf("expecting non-nil config")
+	}
+
+	cfg, err = loadConfig("testdata/prometheus-with-scrape-config-files.yml")
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
@@ -76,7 +84,7 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	// Try loading non-existing file
-	cfg, _, err = loadConfig("testdata/non-existing-file")
+	cfg, err = loadConfig("testdata/non-existing-file")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}
@@ -85,7 +93,7 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	// Try loading invalid file
-	cfg, _, err = loadConfig("testdata/file_sd_1.yml")
+	cfg, err = loadConfig("testdata/file_sd_1.yml")
 	if err == nil {
 		t.Fatalf("expecting non-nil error")
 	}
@@ -114,7 +122,7 @@ scrape_configs:
         replacement: black:9115  # The blackbox exporter's real hostname:port.%
 `
 	var cfg Config
-	if err := cfg.parse([]byte(data), "sss"); err != nil {
+	if err := cfg.parseData([]byte(data), "sss"); err != nil {
 		t.Fatalf("cannot parase data: %s", err)
 	}
 	sws := cfg.getStaticScrapeWork()
@@ -170,7 +178,7 @@ scrape_configs:
   - files: [testdata/file_sd.json]
 `
 	var cfg Config
-	if err := cfg.parse([]byte(data), "sss"); err != nil {
+	if err := cfg.parseData([]byte(data), "sss"); err != nil {
 		t.Fatalf("cannot parase data: %s", err)
 	}
 	sws := cfg.getFileSDScrapeWork(nil)
@@ -186,7 +194,7 @@ scrape_configs:
   - files: [testdata/file_sd_1.yml]
 `
 	var cfgNew Config
-	if err := cfgNew.parse([]byte(dataNew), "sss"); err != nil {
+	if err := cfgNew.parseData([]byte(dataNew), "sss"); err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
 	swsNew := cfgNew.getFileSDScrapeWork(sws)
@@ -201,7 +209,7 @@ scrape_configs:
   file_sd_configs:
   - files: [testdata/prometheus.yml]
 `
-	if err := cfg.parse([]byte(data), "sss"); err != nil {
+	if err := cfg.parseData([]byte(data), "sss"); err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
 	sws = cfg.getFileSDScrapeWork(swsNew)
@@ -216,7 +224,7 @@ scrape_configs:
   file_sd_configs:
   - files: [testdata/empty_target_file_sd.yml]
 `
-	if err := cfg.parse([]byte(data), "sss"); err != nil {
+	if err := cfg.parseData([]byte(data), "sss"); err != nil {
 		t.Fatalf("cannot parse data: %s", err)
 	}
 	sws = cfg.getFileSDScrapeWork(swsNew)
@@ -227,7 +235,7 @@ scrape_configs:
 
 func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
 	var cfg Config
-	if err := cfg.parse(data, path); err != nil {
+	if err := cfg.parseData(data, path); err != nil {
 		return nil, fmt.Errorf("cannot parse data: %w", err)
 	}
 	return cfg.getFileSDScrapeWork(nil), nil
@@ -235,7 +243,7 @@ func getFileSDScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
 
 func getStaticScrapeWork(data []byte, path string) ([]*ScrapeWork, error) {
 	var cfg Config
-	if err := cfg.parse(data, path); err != nil {
+	if err := cfg.parseData(data, path); err != nil {
 		return nil, fmt.Errorf("cannot parse data: %w", err)
 	}
 	return cfg.getStaticScrapeWork(), nil
@@ -263,6 +271,17 @@ scrape_configs:
   - targets: ["foo"]
 `)
 
+	// Duplicate job_name
+	f(`
+scrape_configs:
+- job_name: foo
+  static_configs:
+    targets: ["foo"]
+- job_name: foo
+  static_configs:
+    targets: ["bar"]
+`)
+
 	// Invalid scheme
 	f(`
 scrape_configs:
@@ -487,6 +506,14 @@ scrape_configs:
   static_configs:
   - targets: ["s"]
 `)
+
+	// Invalid scrape_config_files contents
+	f(`
+scrape_config_files:
+- job_name: aa
+  static_configs:
+  - targets: ["s"]
+`)
 }
 
 func resetNonEssentialFields(sws []*ScrapeWork) {

lib/promscrape/scraper.go

@@ -42,7 +42,7 @@ func CheckConfig() error {
 	if *promscrapeConfigFile == "" {
 		return fmt.Errorf("missing -promscrape.config option")
 	}
-	_, _, err := loadConfig(*promscrapeConfigFile)
+	_, err := loadConfig(*promscrapeConfigFile)
 	return err
 }
 
@@ -84,10 +84,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	sighupCh := procutil.NewSighupChan()
 
 	logger.Infof("reading Prometheus configs from %q", configFile)
-	cfg, data, err := loadConfig(configFile)
+	cfg, err := loadConfig(configFile)
 	if err != nil {
 		logger.Fatalf("cannot read %q: %s", configFile, err)
 	}
+	data := cfg.marshal()
 	cfg.mustStart()
 
 	scs := newScrapeConfigs(pushData)
@@ -117,11 +118,12 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 		select {
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; reloading Prometheus configs from %q", configFile)
-			cfgNew, dataNew, err := loadConfig(configFile)
+			cfgNew, err := loadConfig(configFile)
 			if err != nil {
 				logger.Errorf("cannot read %q on SIGHUP: %s; continuing with the previous config", configFile, err)
 				goto waitForChans
 			}
+			dataNew := cfgNew.marshal()
 			if bytes.Equal(data, dataNew) {
 				logger.Infof("nothing changed in %q", configFile)
 				goto waitForChans
@@ -131,11 +133,12 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 			cfg = cfgNew
 			data = dataNew
 		case <-tickerCh:
-			cfgNew, dataNew, err := loadConfig(configFile)
+			cfgNew, err := loadConfig(configFile)
 			if err != nil {
 				logger.Errorf("cannot read %q: %s; continuing with the previous config", configFile, err)
 				goto waitForChans
 			}
+			dataNew := cfgNew.marshal()
 			if bytes.Equal(data, dataNew) {
 				// Nothing changed since the previous loadConfig
 				goto waitForChans

lib/promscrape/testdata/prometheus-with-scrape-config-files.yml

@@ -0,0 +1,8 @@
+scrape_configs:
+- job_name: foo
+  kubernetes_sd_configs:
+  - role: pod
+
+scrape_config_files:
+- scrape_configs.yml
+- scrape_config_files/*.yml

lib/promscrape/testdata/scrape_config_files/1.yml

@@ -0,0 +1,3 @@
+- job_name: job1
+  static_configs:
+  - targets: [foo, bar]

lib/promscrape/testdata/scrape_config_files/2.yml

@@ -0,0 +1,3 @@
+- job_name: job2
+  static_configs:
+  - targets: [foo, bar]

lib/promscrape/testdata/scrape_configs.yml

@@ -0,0 +1,3 @@
+- job_name: bar
+  static_configs:
+  - targets: [foo, bar]