From 54f035d4ce617bcff38917e2e8bacd5e7d869534 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 23 Jul 2019 19:54:50 +0300 Subject: [PATCH] all: small updates after PR #114 --- Makefile | 22 ++++++---- app/vminsert/Makefile | 6 +++ app/vmselect/Makefile | 6 +++ app/vmstorage/Makefile | 6 +++ deployment/docker/Makefile | 3 +- lib/encoding/zstd/stream_cgo.go | 25 ++++++++++++ lib/encoding/zstd/stream_pure.go | 70 ++++++++++++++++++++++++++++++++ lib/handshake/buffered_conn.go | 10 ++--- 8 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 lib/encoding/zstd/stream_cgo.go create mode 100644 lib/encoding/zstd/stream_pure.go diff --git a/Makefile b/Makefile index 3518c47c9e..d3573589ee 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,11 @@ all: \ vmselect \ vmstorage +all-pure: \ + vminsert-pure \ + vmselect-pure \ + vmstorage-pure + include app/*/Makefile include deployment/*/Makefile include deployment/*/helm/Makefile @@ -59,15 +64,13 @@ install-errcheck: check_all: fmt vet lint errcheck golangci-lint test: - GO111MODULE=on go test -mod=vendor ./lib/... - GO111MODULE=on go test -mod=vendor ./app/... - -test_full: - GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... + GO111MODULE=on go test -mod=vendor ./lib/... ./app/... test-pure: - GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... - GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./app/... + GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/... + +test-full: + GO111MODULE=on go test -mod=vendor -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/... benchmark: GO111MODULE=on go test -mod=vendor -bench=. ./lib/... @@ -84,7 +87,10 @@ vendor-update: GO111MODULE=on go mod vendor app-local: - GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME) + CGO_ENABLED=1 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)$(RACE) $(PKG_PREFIX)/app/$(APP_NAME) + +app-local-pure: + CGO_ENABLED=0 GO111MODULE=on go build $(RACE) -mod=vendor -ldflags "$(GO_BUILDINFO)" -o bin/$(APP_NAME)-pure$(RACE) $(PKG_PREFIX)/app/$(APP_NAME) quicktemplate-gen: install-qtc qtc diff --git a/app/vminsert/Makefile b/app/vminsert/Makefile index 5f836d9144..8a2ebc0ffa 100644 --- a/app/vminsert/Makefile +++ b/app/vminsert/Makefile @@ -17,6 +17,12 @@ vminsert-prod: vminsert-prod-race: APP_NAME=vminsert RACE=-race $(MAKE) app-via-docker +vminsert-pure: + APP_NAME=vminsert $(MAKE) app-local-pure + +vminsert-pure-prod: + APP_NAME=vminsert APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker + package-vminsert: APP_NAME=vminsert $(MAKE) package-via-docker diff --git a/app/vmselect/Makefile b/app/vmselect/Makefile index 8822a69f7c..41de873a70 100644 --- a/app/vmselect/Makefile +++ b/app/vmselect/Makefile @@ -19,6 +19,12 @@ vmselect-prod: vmselect-prod-race: APP_NAME=vmselect RACE=-race $(MAKE) app-via-docker +vmselect-pure: + APP_NAME=vmselect $(MAKE) app-local-pure + +vmselect-pure-prod: + APP_NAME=vmselect APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker + package-vmselect: APP_NAME=vmselect $(MAKE) package-via-docker diff --git a/app/vmstorage/Makefile b/app/vmstorage/Makefile index 670e2420f2..94f23aca39 100644 --- a/app/vmstorage/Makefile +++ b/app/vmstorage/Makefile @@ -19,6 +19,12 @@ vmstorage-prod: vmstorage-prod-race: APP_NAME=vmstorage RACE=-race $(MAKE) app-via-docker +vmstorage-pure: + APP_NAME=vmstorage $(MAKE) app-local-pure + +vmstorage-pure-prod: + APP_NAME=vmstorage APP_SUFFIX='-pure' DOCKER_OPTS='--env CGO_ENABLED=0' $(MAKE) app-via-docker + package-vmstorage: APP_NAME=vmstorage $(MAKE) package-via-docker diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 587f8145f4..e476772911 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -21,8 +21,9 @@ app-via-docker: package-certs package-builder --mount type=bind,src="$(shell pwd)/gocache-for-docker",dst=/gocache \ --env GOCACHE=/gocache \ --env GO111MODULE=on \ + $(DOCKER_OPTS) \ $(BUILDER_IMAGE) \ - go build $(RACE) -mod=vendor -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' -o bin/$(APP_NAME)-prod $(PKG_PREFIX)/app/$(APP_NAME) + go build $(RACE) -mod=vendor -ldflags "-s -w -extldflags '-static' $(GO_BUILDINFO)" -tags 'netgo osusergo' -o bin/$(APP_NAME)$(APP_SUFFIX)-prod $(PKG_PREFIX)/app/$(APP_NAME) package-via-docker: (docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q '$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)') || (\ diff --git a/lib/encoding/zstd/stream_cgo.go b/lib/encoding/zstd/stream_cgo.go new file mode 100644 index 0000000000..ca6e4f4474 --- /dev/null +++ b/lib/encoding/zstd/stream_cgo.go @@ -0,0 +1,25 @@ +// +build cgo + +package zstd + +import ( + "io" + + "github.com/valyala/gozstd" +) + +// Reader is zstd reader +type Reader = gozstd.Reader + +// NewReader returns zstd reader for the given r. +func NewReader(r io.Reader) *Reader { + return gozstd.NewReader(r) +} + +// Writer is zstd writer +type Writer = gozstd.Writer + +// NewWriterLevel returns zstd writer for the given w and level. +func NewWriterLevel(w io.Writer, level int) *Writer { + return gozstd.NewWriterLevel(w, level) +} diff --git a/lib/encoding/zstd/stream_pure.go b/lib/encoding/zstd/stream_pure.go new file mode 100644 index 0000000000..50afb7ccf4 --- /dev/null +++ b/lib/encoding/zstd/stream_pure.go @@ -0,0 +1,70 @@ +// +build !cgo + +package zstd + +import ( + "io" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/klauspost/compress/zstd" +) + +// Reader is zstd reader +type Reader struct { + d *zstd.Decoder +} + +// NewReader returns zstd reader for the given r. +func NewReader(r io.Reader) *Reader { + d, err := zstd.NewReader(r) + if err != nil { + logger.Panicf("BUG: failed to create ZSTD reader: %s", err) + } + return &Reader{ + d: d, + } +} + +// Read reads up to len(p) bytes to p from r. +func (r *Reader) Read(p []byte) (int, error) { + return r.d.Read(p) +} + +// Release releases r. +func (r *Reader) Release() { + r.d.Reset(nil) + r.d = nil +} + +// Writer is zstd writer +type Writer struct { + e *zstd.Encoder +} + +// NewWriterLevel returns zstd writer for the given w and level. +func NewWriterLevel(w io.Writer, level int) *Writer { + l := zstd.EncoderLevelFromZstd(level) + e, err := zstd.NewWriter(w, zstd.WithEncoderLevel(l)) + if err != nil { + logger.Panicf("BUG: failed to create ZSTD writer: %s", err) + } + return &Writer{ + e: e, + } +} + +// Write writes p to w. +func (w *Writer) Write(p []byte) (int, error) { + return w.e.Write(p) +} + +// Flush flushes all the pending data from w to the underlying writer. +func (w *Writer) Flush() error { + return w.e.Flush() +} + +// Release releases w. +func (w *Writer) Release() { + w.e.Reset(nil) + w.e = nil +} diff --git a/lib/handshake/buffered_conn.go b/lib/handshake/buffered_conn.go index 774adf0366..50fbafb4b4 100644 --- a/lib/handshake/buffered_conn.go +++ b/lib/handshake/buffered_conn.go @@ -5,7 +5,7 @@ import ( "io" "net" - "github.com/valyala/gozstd" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd" ) type bufferedWriter interface { @@ -31,12 +31,12 @@ func newBufferedConn(c net.Conn, compressionLevel int, isReadCompressed bool) *B if compressionLevel <= 0 { bc.bw = bufio.NewWriterSize(c, bufferSize) } else { - bc.bw = gozstd.NewWriterLevel(c, compressionLevel) + bc.bw = zstd.NewWriterLevel(c, compressionLevel) } if !isReadCompressed { bc.br = bufio.NewReaderSize(c, bufferSize) } else { - bc.br = gozstd.NewReader(c) + bc.br = zstd.NewReader(c) } return bc } @@ -60,12 +60,12 @@ func (bc *BufferedConn) Close() error { err := bc.Conn.Close() bc.Conn = nil - if zr, ok := bc.br.(*gozstd.Reader); ok { + if zr, ok := bc.br.(*zstd.Reader); ok { zr.Release() } bc.br = nil - if zw, ok := bc.bw.(*gozstd.Writer); ok { + if zw, ok := bc.bw.(*zstd.Writer); ok { // Do not call zw.Close(), since we already closed the underlying conn. zw.Release() }