VictoriaMetrics/app/vmalert/config/fs.go

package config

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fsurl"
)

// FS represent a file system abstract for reading files.
type FS interface {
	// Init initializes FS.
	Init() error

	// String must return human-readable representation of FS.
	String() string

	// List returns the list of file names which will be read via Read fn
	List() ([]string, error)

	// Read returns a list of read files in form of a map
	// where key is a file name and value is a content of read file.
	// Read must be called only after the successful Init call.
	Read(files []string) (map[string][]byte, error)
}

var (
	fsRegistryMu sync.Mutex
	fsRegistry   = make(map[string]FS)
)

// readFromFS parses the given path list and inits FS for each item.
// Once initialed, readFromFS will try to read and return files from each FS.
// readFromFS returns an error if at least one FS failed to init.
// The function can be called multiple times but each unique path
// will be initialed only once.
//
// It is allowed to mix different FS types in path list.
func readFromFS(paths []string) (map[string][]byte, error) {
	var err error
	result := make(map[string][]byte)
	for _, path := range paths {
		fsRegistryMu.Lock()
		fs, ok := fsRegistry[path]
		if !ok {
			fs, err = newFS(path)
			if err != nil {
				fsRegistryMu.Unlock()
				return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
			}
			if err := fs.Init(); err != nil {
				fsRegistryMu.Unlock()
				return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
			}
			fsRegistry[path] = fs
		}
		fsRegistryMu.Unlock()

		list, err := fs.List()
		if err != nil {
			return nil, fmt.Errorf("failed to list files from %q", fs)
		}

		cLogger.Infof("found %d files to read from %q", len(list), fs)

		if len(list) < 1 {
			continue
		}

		ts := time.Now()
		files, err := fs.Read(list)
		if err != nil {
			return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
		}
		cLogger.Infof("finished reading %d files in %v from %q", len(list), time.Since(ts), fs)

		for k, v := range files {
			if _, ok := result[k]; ok {
				return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
			}
			result[k] = v
		}
	}
	return result, nil
}

// newFS creates FS based on the give path.
// Supported file systems are: fs
func newFS(originPath string) (FS, error) {
	scheme := "fs"
	path := originPath
	n := strings.Index(path, "://")
	if n >= 0 {
		scheme = path[:n]
		path = path[n+len("://"):]
	}
	if len(path) == 0 {
		return nil, fmt.Errorf("path cannot be empty")
	}
	switch scheme {
	case "fs":
		return &fslocal.FS{Pattern: path}, nil
	case "http", "https":
		return &fsurl.FS{Path: originPath}, nil
	default:
		return nil, fmt.Errorf("unsupported scheme %q", scheme)
	}
}