mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-18 22:52:11 +01:00
5be2f2c4e4
…es, and http url in unittest cmd-line flag `-files`
(cherry picked from commit 3b8970802e)
Signed-off-by: hagen1778 <roman@victoriametrics.com>
112 lines
2.9 KiB
Go
112 lines
2.9 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fslocal"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config/fsurl"
|
|
)
|
|
|
|
// FS represent a file system abstract for reading files.
|
|
type FS interface {
|
|
// Init initializes FS.
|
|
Init() error
|
|
|
|
// String must return human-readable representation of FS.
|
|
String() string
|
|
|
|
// List returns the list of file names which will be read via Read fn
|
|
List() ([]string, error)
|
|
|
|
// Read returns a list of read files in form of a map
|
|
// where key is a file name and value is a content of read file.
|
|
// Read must be called only after the successful Init call.
|
|
Read(files []string) (map[string][]byte, error)
|
|
}
|
|
|
|
var (
|
|
fsRegistryMu sync.Mutex
|
|
fsRegistry = make(map[string]FS)
|
|
)
|
|
|
|
// ReadFromFS parses the given path list and inits FS for each item.
|
|
// Once initialed, ReadFromFS will try to read and return files from each FS.
|
|
// ReadFromFS returns an error if at least one FS failed to init.
|
|
// The function can be called multiple times but each unique path
|
|
// will be initialed only once.
|
|
//
|
|
// It is allowed to mix different FS types in path list.
|
|
func ReadFromFS(paths []string) (map[string][]byte, error) {
|
|
var err error
|
|
result := make(map[string][]byte)
|
|
for _, path := range paths {
|
|
fsRegistryMu.Lock()
|
|
fs, ok := fsRegistry[path]
|
|
if !ok {
|
|
fs, err = newFS(path)
|
|
if err != nil {
|
|
fsRegistryMu.Unlock()
|
|
return nil, fmt.Errorf("error while parsing path %q: %w", path, err)
|
|
}
|
|
if err := fs.Init(); err != nil {
|
|
fsRegistryMu.Unlock()
|
|
return nil, fmt.Errorf("error while initializing path %q: %w", path, err)
|
|
}
|
|
fsRegistry[path] = fs
|
|
}
|
|
fsRegistryMu.Unlock()
|
|
|
|
list, err := fs.List()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list files from %q", fs)
|
|
}
|
|
|
|
cLogger.Infof("found %d files to read from %q", len(list), fs)
|
|
|
|
if len(list) < 1 {
|
|
continue
|
|
}
|
|
|
|
ts := time.Now()
|
|
files, err := fs.Read(list)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while reading files from %q: %w", fs, err)
|
|
}
|
|
cLogger.Infof("finished reading %d files in %v from %q", len(list), time.Since(ts), fs)
|
|
|
|
for k, v := range files {
|
|
if _, ok := result[k]; ok {
|
|
return nil, fmt.Errorf("duplicate found for file name %q: file names must be unique", k)
|
|
}
|
|
result[k] = v
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// newFS creates FS based on the give path.
|
|
// Supported file systems are: fs
|
|
func newFS(originPath string) (FS, error) {
|
|
scheme := "fs"
|
|
path := originPath
|
|
n := strings.Index(path, "://")
|
|
if n >= 0 {
|
|
scheme = path[:n]
|
|
path = path[n+len("://"):]
|
|
}
|
|
if len(path) == 0 {
|
|
return nil, fmt.Errorf("path cannot be empty")
|
|
}
|
|
switch scheme {
|
|
case "fs":
|
|
return &fslocal.FS{Pattern: path}, nil
|
|
case "http", "https":
|
|
return &fsurl.FS{Path: originPath}, nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported scheme %q", scheme)
|
|
}
|
|
}
|