mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-15 08:23:34 +01:00
b9ff61811d
vmctl: fix vmctl blocking on process interrupt This change prevents vmctl from indefinite blocking on receiving the interrupt signal. The update touches all import modes and suppose to improve tool reliability. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2491
160 lines
3.2 KiB
Go
160 lines
3.2 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"sync"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
|
)
|
|
|
|
type influxProcessor struct {
|
|
ic *influx.Client
|
|
im *vm.Importer
|
|
cc int
|
|
separator string
|
|
}
|
|
|
|
func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator string) *influxProcessor {
|
|
if cc < 1 {
|
|
cc = 1
|
|
}
|
|
return &influxProcessor{
|
|
ic: ic,
|
|
im: im,
|
|
cc: cc,
|
|
separator: separator,
|
|
}
|
|
}
|
|
|
|
func (ip *influxProcessor) run(silent, verbose bool) error {
|
|
series, err := ip.ic.Explore()
|
|
if err != nil {
|
|
return fmt.Errorf("explore query failed: %s", err)
|
|
}
|
|
if len(series) < 1 {
|
|
return fmt.Errorf("found no timeseries to import")
|
|
}
|
|
|
|
question := fmt.Sprintf("Found %d timeseries to import. Continue?", len(series))
|
|
if !silent && !prompt(question) {
|
|
return nil
|
|
}
|
|
|
|
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing series"), len(series))
|
|
if err := barpool.Start(); err != nil {
|
|
return err
|
|
}
|
|
|
|
seriesCh := make(chan *influx.Series)
|
|
errCh := make(chan error)
|
|
ip.im.ResetStats()
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(ip.cc)
|
|
for i := 0; i < ip.cc; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
for s := range seriesCh {
|
|
if err := ip.do(s); err != nil {
|
|
errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
|
|
return
|
|
}
|
|
bar.Increment()
|
|
}
|
|
}()
|
|
}
|
|
|
|
// any error breaks the import
|
|
for _, s := range series {
|
|
select {
|
|
case infErr := <-errCh:
|
|
return fmt.Errorf("influx error: %s", infErr)
|
|
case vmErr := <-ip.im.Errors():
|
|
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
|
|
case seriesCh <- s:
|
|
}
|
|
}
|
|
|
|
close(seriesCh)
|
|
wg.Wait()
|
|
ip.im.Close()
|
|
close(errCh)
|
|
// drain import errors channel
|
|
for vmErr := range ip.im.Errors() {
|
|
if vmErr.Err != nil {
|
|
return fmt.Errorf("import process failed: %s", wrapErr(vmErr, verbose))
|
|
}
|
|
}
|
|
for err := range errCh {
|
|
return fmt.Errorf("import process failed: %s", err)
|
|
}
|
|
barpool.Stop()
|
|
log.Println("Import finished!")
|
|
log.Print(ip.im.Stats())
|
|
return nil
|
|
}
|
|
|
|
const dbLabel = "db"
|
|
|
|
func (ip *influxProcessor) do(s *influx.Series) error {
|
|
cr, err := ip.ic.FetchDataPoints(s)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch datapoints: %s", err)
|
|
}
|
|
defer func() {
|
|
_ = cr.Close()
|
|
}()
|
|
var name string
|
|
if s.Measurement != "" {
|
|
name = fmt.Sprintf("%s%s%s", s.Measurement, ip.separator, s.Field)
|
|
} else {
|
|
name = s.Field
|
|
}
|
|
|
|
labels := make([]vm.LabelPair, len(s.LabelPairs))
|
|
var containsDBLabel bool
|
|
for i, lp := range s.LabelPairs {
|
|
if lp.Name == dbLabel {
|
|
containsDBLabel = true
|
|
break
|
|
}
|
|
labels[i] = vm.LabelPair{
|
|
Name: lp.Name,
|
|
Value: lp.Value,
|
|
}
|
|
}
|
|
if !containsDBLabel {
|
|
labels = append(labels, vm.LabelPair{
|
|
Name: dbLabel,
|
|
Value: ip.ic.Database(),
|
|
})
|
|
}
|
|
|
|
for {
|
|
time, values, err := cr.Next()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
// skip empty results
|
|
if len(time) < 1 {
|
|
continue
|
|
}
|
|
ts := vm.TimeSeries{
|
|
Name: name,
|
|
LabelPairs: labels,
|
|
Timestamps: time,
|
|
Values: values,
|
|
}
|
|
if err := ip.im.Input(&ts); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|