vmalert-tool: support file path with hierarchical patterns and regexp… (#6501)

…es, and http url in unittest cmd-line flag `-files`

app/vmalert-tool/main.go

@@ -27,7 +27,13 @@ func main() {
 				Flags: []cli.Flag{
 					&cli.StringSliceFlag{
 						Name: "files",
-						Usage:    "files to run unittest with. Supports an array of values separated by comma or specified via multiple flags.",
+						Usage: `File path or http url with test files. Supports an array of values separated by comma or specified via multiple flags.
+						Supports hierarchical patterns and regexpes.
+Examples:
+ -files="/path/to/file". Path to a single test file.
+ -files="http://<some-server-addr>/path/to/test.yaml". HTTP URL to a test file.
+ -files="dir/**/*.yaml". Includes all the .yaml files in "dir" subfolders recursively.
+ `,
 						Required: true,
 					},
 					&cli.BoolFlag{

app/vmalert-tool/unittest/unittest.go

@@ -9,10 +9,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strings"
 	"time"
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	vmalertconfig "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -66,36 +68,38 @@ func UnitTest(files []string, disableGroupLabel bool) bool {
 	defer vminsert.Stop()
 	defer vmselect.Stop()
 	disableAlertgroupLabel = disableGroupLabel
-	return rulesUnitTest(files)
-}
 
-func rulesUnitTest(files []string) bool {
-	var failed bool
-	for _, f := range files {
-		if err := ruleUnitTest(f); err != nil {
+	testfiles, err := config.ReadFromFS(files)
+	if err != nil {
 		fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", f, err)
+		fmt.Printf("\nfailed to read test files: \n%v", err)
+	}
+	var failed bool
+	for fileName, file := range testfiles {
+		if err := ruleUnitTest(fileName, file); err != nil {
+			fmt.Println("  FAILED")
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
 		}
 	}
+
 	return failed
 }
 
-func ruleUnitTest(filename string) []error {
+func ruleUnitTest(filename string, content []byte) []error {
 	fmt.Println("\nUnit Testing: ", filename)
-	b, err := os.ReadFile(filename)
-	if err != nil {
-		return []error{fmt.Errorf("failed to read file: %w", err)}
-	}
-
 	var unitTestInp unitTestFile
-	if err := yaml.UnmarshalStrict(b, &unitTestInp); err != nil {
+	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
 		return []error{fmt.Errorf("failed to unmarshal file: %w", err)}
 	}
-	if err := resolveAndGlobFilepaths(filepath.Dir(filename), &unitTestInp); err != nil {
-		return []error{fmt.Errorf("failed to resolve path for `rule_files`: %w", err)}
+
+	// add file directory for rule files if needed
+	for i, rf := range unitTestInp.RuleFiles {
+		if rf != "" && !filepath.IsAbs(rf) && !strings.HasPrefix(rf, "http") {
+			unitTestInp.RuleFiles[i] = filepath.Join(filepath.Dir(filename), rf)
+		}
 	}
 
 	if unitTestInp.EvaluationInterval.Duration() == 0 {
@@ -235,30 +239,6 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-// resolveAndGlobFilepaths joins all relative paths in a configuration
-// with a given base directory and replaces all globs with matching files.
-func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
-	for i, rf := range utf.RuleFiles {
-		if rf != "" && !filepath.IsAbs(rf) {
-			utf.RuleFiles[i] = filepath.Join(baseDir, rf)
-		}
-	}
-
-	var globbedFiles []string
-	for _, rf := range utf.RuleFiles {
-		m, err := filepath.Glob(rf)
-		if err != nil {
-			return err
-		}
-		if len(m) == 0 {
-			fmt.Fprintln(os.Stderr, "  WARNING: no file match pattern", rf)
-		}
-		globbedFiles = append(globbedFiles, m...)
-	}
-	utf.RuleFiles = globbedFiles
-	return nil
-}
-
 func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()

app/vmalert/config/config.go

@@ -234,7 +234,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 	cLogger.Suppress(true)
 	defer cLogger.Suppress(false)
 
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}
@@ -243,7 +243,7 @@ func ParseSilent(pathPatterns []string, validateTplFn ValidateTplFn, validateExp
 
 // Parse parses rule configs from given file patterns
 func Parse(pathPatterns []string, validateTplFn ValidateTplFn, validateExpressions bool) ([]Group, error) {
-	files, err := readFromFS(pathPatterns)
+	files, err := ReadFromFS(pathPatterns)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read from the config: %w", err)
 	}

app/vmalert/config/fs.go

@@ -32,14 +32,14 @@ var (
 	fsRegistry   = make(map[string]FS)
 )
 
-// readFromFS parses the given path list and inits FS for each item.
-// Once initialed, readFromFS will try to read and return files from each FS.
-// readFromFS returns an error if at least one FS failed to init.
+// ReadFromFS parses the given path list and inits FS for each item.
+// Once initialed, ReadFromFS will try to read and return files from each FS.
+// ReadFromFS returns an error if at least one FS failed to init.
 // The function can be called multiple times but each unique path
 // will be initialed only once.
 //
 // It is allowed to mix different FS types in path list.
-func readFromFS(paths []string) (map[string][]byte, error) {
+func ReadFromFS(paths []string) (map[string][]byte, error) {
 	var err error
 	result := make(map[string][]byte)
 	for _, path := range paths {

docs/CHANGELOG.md

@@ -39,6 +39,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): increase default value of `-promscrape.maxDroppedTargets` command-line flag to 10_000 from 1000. This makes it easier to track down large number of dropped targets.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): check for ranged vector arguments in non-rollup expressions when `-search.disableImplicitConversion` or `-search.logImplicitConversion` are enabled. For example, `sum(up[5m])` or `absent(up[5m])` will fail to execute if these flags are set.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): validate that rollup expressions has ranged vector arguments passed when `-search.disableImplicitConversion` or `-search.logImplicitConversion` are enabled. For example, `rate(metric)` or `count_over_time(metric)` will fail to execute if these flags are set.
+* FEATURE: [vmalert-tool](https://docs.victoriametrics.com/vmalert-tool/): support file path with hierarchical patterns and regexpes, and http url in unittest cmd-line flag `-files`, e.g. `-files="http://<some-server-addr>/path/to/rules"` or `-files="dir/**/*.yaml"`.
 
 * BUGFIX: all VictoriaMetrics components: prioritize `-configAuthKey` and `-reloadAuthKey` over `-httpAuth.*` settings. This change aligns behavior of mentioned flags with other auth flags like `-metricsAuthKey`, `-flagsAuthKey`, `-pprofAuthKey`. Check [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6329).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): add `--disable-progress-bar` global command-line flag. It can be used for disabling dynamic progress bar for all migration modes. `--vm-disable-progress-bar`  command-line flag is deprecated and will be removed in the future releases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6367).

docs/vmalert-tool.md

@@ -26,8 +26,9 @@ It will perform the following actions:
 See how to run vmalert-tool for unit test below:
 
 ```
-# Run vmalert-tool with one or multiple test files via --files cmd-line flag
-./vmalert-tool unittest --files test1.yaml --files test2.yaml
+# Run vmalert-tool with one or multiple test files via `--files` cmd-line flag
+# Supports file path with hierarchical patterns and regexpes, and http url.
+./vmalert-tool unittest --files /path/to/file --files http://<some-server-addr>/path/to/test.yaml
 ```
 
 vmalert-tool unittest is compatible with [Prometheus config format for tests](https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/#test-file-format)