2021-02-01 00:10:16 +01:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2022-05-03 07:03:41 +02:00
|
|
|
"context"
|
2021-02-01 00:10:16 +01:00
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"log"
|
|
|
|
"net/http"
|
2022-09-06 09:09:34 +02:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/dmitryk-dk/pb/v3"
|
2021-02-01 00:10:16 +01:00
|
|
|
|
2021-12-24 11:18:07 +01:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
|
2022-09-06 09:09:34 +02:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
|
2021-12-24 11:18:07 +01:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
2021-02-01 00:10:16 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
type vmNativeProcessor struct {
|
2021-12-24 11:18:07 +01:00
|
|
|
filter filter
|
|
|
|
rateLimit int64
|
2021-02-01 00:10:16 +01:00
|
|
|
|
|
|
|
dst *vmNativeClient
|
|
|
|
src *vmNativeClient
|
|
|
|
}
|
|
|
|
|
|
|
|
type vmNativeClient struct {
|
|
|
|
addr string
|
|
|
|
user string
|
|
|
|
password string
|
|
|
|
extraLabels []string
|
|
|
|
}
|
|
|
|
|
|
|
|
type filter struct {
|
|
|
|
match string
|
|
|
|
timeStart string
|
|
|
|
timeEnd string
|
2022-09-06 09:09:34 +02:00
|
|
|
chunk string
|
2021-02-01 00:10:16 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
func (f filter) String() string {
|
|
|
|
s := fmt.Sprintf("\n\tfilter: match[]=%s", f.match)
|
|
|
|
if f.timeStart != "" {
|
|
|
|
s += fmt.Sprintf("\n\tstart: %s", f.timeStart)
|
|
|
|
}
|
|
|
|
if f.timeEnd != "" {
|
|
|
|
s += fmt.Sprintf("\n\tend: %s", f.timeEnd)
|
|
|
|
}
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
|
|
|
const (
|
|
|
|
nativeExportAddr = "api/v1/export/native"
|
|
|
|
nativeImportAddr = "api/v1/import/native"
|
|
|
|
|
2022-05-02 09:06:34 +02:00
|
|
|
nativeBarTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}`
|
2021-02-01 00:10:16 +01:00
|
|
|
)
|
|
|
|
|
2022-05-03 07:03:41 +02:00
|
|
|
func (p *vmNativeProcessor) run(ctx context.Context) error {
|
2022-09-06 09:09:34 +02:00
|
|
|
if p.filter.chunk == "" {
|
|
|
|
return p.runSingle(ctx, p.filter)
|
|
|
|
}
|
|
|
|
|
|
|
|
startOfRange, err := time.Parse(time.RFC3339, p.filter.timeStart)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeStart, p.filter.timeStart, time.RFC3339, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
var endOfRange time.Time
|
|
|
|
if p.filter.timeEnd != "" {
|
|
|
|
endOfRange, err = time.Parse(time.RFC3339, p.filter.timeEnd)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %v", vmNativeFilterTimeEnd, p.filter.timeEnd, time.RFC3339, err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
endOfRange = time.Now()
|
|
|
|
}
|
|
|
|
|
|
|
|
ranges, err := stepper.SplitDateRange(startOfRange, endOfRange, p.filter.chunk)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to create date ranges for the given time filters: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
for rangeIdx, r := range ranges {
|
|
|
|
formattedStartTime := r[0].Format(time.RFC3339)
|
|
|
|
formattedEndTime := r[1].Format(time.RFC3339)
|
|
|
|
log.Printf("Processing range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
|
|
|
|
f := filter{
|
|
|
|
match: p.filter.match,
|
|
|
|
timeStart: formattedStartTime,
|
|
|
|
timeEnd: formattedEndTime,
|
|
|
|
}
|
|
|
|
err := p.runSingle(ctx, f)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("processing failed for range %d/%d: %s - %s \n", rangeIdx+1, len(ranges), formattedStartTime, formattedEndTime)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *vmNativeProcessor) runSingle(ctx context.Context, f filter) error {
|
2021-02-01 00:10:16 +01:00
|
|
|
pr, pw := io.Pipe()
|
|
|
|
|
2022-09-06 09:09:34 +02:00
|
|
|
log.Printf("Initing export pipe from %q with filters: %s\n", p.src.addr, f)
|
|
|
|
exportReader, err := p.exportPipe(ctx, f)
|
2021-02-01 00:10:16 +01:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to init export pipe: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
nativeImportAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.extraLabels)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2022-05-03 07:03:41 +02:00
|
|
|
sync := make(chan struct{})
|
2021-02-01 00:10:16 +01:00
|
|
|
go func() {
|
|
|
|
defer func() { close(sync) }()
|
|
|
|
u := fmt.Sprintf("%s/%s", p.dst.addr, nativeImportAddr)
|
2022-05-03 07:03:41 +02:00
|
|
|
req, err := http.NewRequestWithContext(ctx, "POST", u, pr)
|
2021-02-01 00:10:16 +01:00
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("cannot create import request to %q: %s", p.dst.addr, err)
|
|
|
|
}
|
|
|
|
importResp, err := p.dst.do(req, http.StatusNoContent)
|
|
|
|
if err != nil {
|
|
|
|
log.Fatalf("import request failed: %s", err)
|
|
|
|
}
|
2021-02-01 00:31:25 +01:00
|
|
|
if err := importResp.Body.Close(); err != nil {
|
|
|
|
log.Fatalf("cannot close import response body: %s", err)
|
|
|
|
}
|
2021-02-01 00:10:16 +01:00
|
|
|
}()
|
|
|
|
|
|
|
|
fmt.Printf("Initing import process to %q:\n", p.dst.addr)
|
2022-09-06 09:09:34 +02:00
|
|
|
pool := pb.NewPool()
|
|
|
|
bar := pb.ProgressBarTemplate(nativeBarTpl).New(0)
|
|
|
|
pool.Add(bar)
|
2021-02-01 00:10:16 +01:00
|
|
|
barReader := bar.NewProxyReader(exportReader)
|
2022-09-06 09:09:34 +02:00
|
|
|
if err := pool.Start(); err != nil {
|
2022-05-02 09:06:34 +02:00
|
|
|
log.Printf("error start process bars pool: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
2022-09-06 09:09:34 +02:00
|
|
|
defer func() {
|
|
|
|
bar.Finish()
|
|
|
|
if err := pool.Stop(); err != nil {
|
|
|
|
fmt.Printf("failed to stop barpool: %+v\n", err)
|
|
|
|
}
|
|
|
|
}()
|
2021-02-01 00:10:16 +01:00
|
|
|
|
2021-12-24 11:18:07 +01:00
|
|
|
w := io.Writer(pw)
|
|
|
|
if p.rateLimit > 0 {
|
|
|
|
rl := limiter.NewLimiter(p.rateLimit)
|
|
|
|
w = limiter.NewWriteLimiter(pw, rl)
|
|
|
|
}
|
2022-05-03 07:03:41 +02:00
|
|
|
|
2021-12-24 11:18:07 +01:00
|
|
|
_, err = io.Copy(w, barReader)
|
2021-02-01 00:10:16 +01:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed to write into %q: %s", p.dst.addr, err)
|
|
|
|
}
|
2022-05-03 07:03:41 +02:00
|
|
|
|
2021-02-01 00:10:16 +01:00
|
|
|
if err := pw.Close(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
<-sync
|
|
|
|
|
2022-05-03 07:03:41 +02:00
|
|
|
log.Println("Import finished!")
|
2021-02-01 00:10:16 +01:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-09-06 09:09:34 +02:00
|
|
|
func (p *vmNativeProcessor) exportPipe(ctx context.Context, f filter) (io.ReadCloser, error) {
|
2021-02-01 00:10:16 +01:00
|
|
|
u := fmt.Sprintf("%s/%s", p.src.addr, nativeExportAddr)
|
2022-05-03 07:03:41 +02:00
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", u, nil)
|
2021-02-01 00:10:16 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("cannot create request to %q: %s", p.src.addr, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
params := req.URL.Query()
|
2022-09-06 09:09:34 +02:00
|
|
|
params.Set("match[]", f.match)
|
|
|
|
if f.timeStart != "" {
|
|
|
|
params.Set("start", f.timeStart)
|
2021-02-01 00:10:16 +01:00
|
|
|
}
|
2022-09-06 09:09:34 +02:00
|
|
|
if f.timeEnd != "" {
|
|
|
|
params.Set("end", f.timeEnd)
|
2021-02-01 00:10:16 +01:00
|
|
|
}
|
|
|
|
req.URL.RawQuery = params.Encode()
|
|
|
|
|
|
|
|
// disable compression since it is meaningless for native format
|
|
|
|
req.Header.Set("Accept-Encoding", "identity")
|
|
|
|
resp, err := p.src.do(req, http.StatusOK)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("export request failed: %s", err)
|
|
|
|
}
|
|
|
|
return resp.Body, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *vmNativeClient) do(req *http.Request, expSC int) (*http.Response, error) {
|
|
|
|
if c.user != "" {
|
|
|
|
req.SetBasicAuth(c.user, c.password)
|
|
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("unexpected error when performing request: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if resp.StatusCode != expSC {
|
2022-08-21 23:13:44 +02:00
|
|
|
body, err := io.ReadAll(resp.Body)
|
2021-02-01 00:10:16 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("failed to read response body for status code %d: %s", resp.StatusCode, err)
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("unexpected response code %d: %s", resp.StatusCode, string(body))
|
|
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|