From 0d893eff36e3c8ad4a393a2558cd89d00728d1c0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 5 Mar 2020 12:02:06 +0200 Subject: [PATCH] Makefile: add build and test rules with enabled race detector. These rules have `-race` suffix Fix also `unsafe pointer conversion` errors detected by Go1.14. See https://golang.org/doc/go1.14#compiler . --- Makefile | 3 +++ app/vmagent/Makefile | 3 +++ app/vmbackup/Makefile | 3 +++ app/vmrestore/Makefile | 3 +++ app/vmselect/promql/arch.go | 5 ---- app/vmselect/promql/timeseries.go | 43 ++++++++++++++++++++++++++----- lib/fastnum/fastnum.go | 31 ++++++++++++++++------ 7 files changed, 72 insertions(+), 19 deletions(-) delete mode 100644 app/vmselect/promql/arch.go diff --git a/Makefile b/Makefile index 82b749c53d..0d1e7b71cf 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,9 @@ check-all: fmt vet lint errcheck golangci-lint test: GO111MODULE=on go test -mod=vendor ./lib/... ./app/... +test-race: + GO111MODULE=on go test -mod=vendor -race ./lib/... ./app/... + test-pure: GO111MODULE=on CGO_ENABLED=0 go test -mod=vendor ./lib/... ./app/... diff --git a/app/vmagent/Makefile b/app/vmagent/Makefile index e2fbfc2b38..e18755f64c 100644 --- a/app/vmagent/Makefile +++ b/app/vmagent/Makefile @@ -3,6 +3,9 @@ vmagent: APP_NAME=vmagent $(MAKE) app-local +vmagent-race: + APP_NAME=vmagent RACE=-race $(MAKE) app-local + vmagent-prod: APP_NAME=vmagent $(MAKE) app-via-docker diff --git a/app/vmbackup/Makefile b/app/vmbackup/Makefile index 842049767b..609e57c976 100644 --- a/app/vmbackup/Makefile +++ b/app/vmbackup/Makefile @@ -3,6 +3,9 @@ vmbackup: APP_NAME=vmbackup $(MAKE) app-local +vmbackup-race: + APP_NAME=vmbackup RACE=-race $(MAKE) app-local + vmbackup-prod: APP_NAME=vmbackup $(MAKE) app-via-docker diff --git a/app/vmrestore/Makefile b/app/vmrestore/Makefile index 305bee9f22..314a9a9386 100644 --- a/app/vmrestore/Makefile +++ b/app/vmrestore/Makefile @@ -3,6 +3,9 @@ vmrestore: APP_NAME=vmrestore $(MAKE) app-local +vmrestore-race: + APP_NAME=vmrestore RACE=-race $(MAKE) app-local + vmrestore-prod: APP_NAME=vmrestore $(MAKE) app-via-docker diff --git a/app/vmselect/promql/arch.go b/app/vmselect/promql/arch.go deleted file mode 100644 index 2e21b86747..0000000000 --- a/app/vmselect/promql/arch.go +++ /dev/null @@ -1,5 +0,0 @@ -package promql - -import "unsafe" - -const maxByteSliceLen = 1<<(31+9*(unsafe.Sizeof(int(0))/8)) - 1 diff --git a/app/vmselect/promql/timeseries.go b/app/vmselect/promql/timeseries.go index f40907cc13..728ecd516d 100644 --- a/app/vmselect/promql/timeseries.go +++ b/app/vmselect/promql/timeseries.go @@ -2,6 +2,7 @@ package promql import ( "fmt" + "reflect" "sort" "strconv" "sync" @@ -168,7 +169,7 @@ func (ts *timeseries) marshalFastNoTimestamps(dst []byte) []byte { // during marshalFastTimestamps. var valuesBuf []byte if len(ts.Values) > 0 { - valuesBuf = (*[maxByteSliceLen]byte)(unsafe.Pointer(&ts.Values[0]))[:len(ts.Values)*8] + valuesBuf = float64ToByteSlice(ts.Values) } dst = append(dst, valuesBuf...) return dst @@ -178,7 +179,7 @@ func marshalFastTimestamps(dst []byte, timestamps []int64) []byte { dst = encoding.MarshalUint32(dst, uint32(len(timestamps))) var timestampsBuf []byte if len(timestamps) > 0 { - timestampsBuf = (*[maxByteSliceLen]byte)(unsafe.Pointer(×tamps[0]))[:len(timestamps)*8] + timestampsBuf = int64ToByteSlice(timestamps) } dst = append(dst, timestampsBuf...) return dst @@ -199,8 +200,7 @@ func unmarshalFastTimestamps(src []byte) ([]byte, []int64, error) { if len(src) < bufSize { return src, nil, fmt.Errorf("cannot unmarshal timestamps; got %d bytes; want at least %d bytes", len(src), bufSize) } - timestamps := (*[maxByteSliceLen / 8]int64)(unsafe.Pointer(&src[0]))[:timestampsCount] - timestamps = timestamps[:len(timestamps):len(timestamps)] + timestamps := byteSliceToInt64(src[:bufSize]) src = src[bufSize:] return src, timestamps, nil @@ -229,12 +229,43 @@ func (ts *timeseries) unmarshalFastNoTimestamps(src []byte) ([]byte, error) { if len(src) < bufSize { return src, fmt.Errorf("cannot unmarshal values; got %d bytes; want at least %d bytes", len(src), bufSize) } - values := (*[maxByteSliceLen / 8]float64)(unsafe.Pointer(&src[0]))[:valuesCount] - ts.Values = values[:len(values):len(values)] + ts.Values = byteSliceToFloat64(src[:bufSize]) return src[bufSize:], nil } +func float64ToByteSlice(a []float64) (b []byte) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + sh.Data = uintptr(unsafe.Pointer(&a[0])) + sh.Len = len(a) * int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + +func int64ToByteSlice(a []int64) (b []byte) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + sh.Data = uintptr(unsafe.Pointer(&a[0])) + sh.Len = len(a) * int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + +func byteSliceToInt64(b []byte) (a []int64) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&a)) + sh.Data = uintptr(unsafe.Pointer(&b[0])) + sh.Len = len(b) / int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + +func byteSliceToFloat64(b []byte) (a []float64) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&a)) + sh.Data = uintptr(unsafe.Pointer(&b[0])) + sh.Len = len(b) / int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + // unmarshalMetricNameFast unmarshals mn from src, so mn members // hold references to src. // diff --git a/lib/fastnum/fastnum.go b/lib/fastnum/fastnum.go index f4b191a872..524098bb29 100644 --- a/lib/fastnum/fastnum.go +++ b/lib/fastnum/fastnum.go @@ -2,6 +2,7 @@ package fastnum import ( "bytes" + "reflect" "unsafe" ) @@ -84,7 +85,7 @@ func isInt64Data(a, data []int64) bool { if len(data) != 8*1024 { panic("len(data) must equal to 8*1024") } - b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0])) + b := int64ToByteSlice(data) for len(a) > 0 { n := len(data) if n > len(a) { @@ -92,9 +93,8 @@ func isInt64Data(a, data []int64) bool { } x := a[:n] a = a[n:] - xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0])) - xbLen := len(x) * 8 - if !bytes.Equal(xb[:xbLen], b[:xbLen]) { + xb := int64ToByteSlice(x) + if !bytes.Equal(xb, b[:len(xb)]) { return false } } @@ -108,7 +108,7 @@ func isFloat64Data(a, data []float64) bool { if len(data) != 8*1024 { panic("len(data) must equal to 8*1024") } - b := (*[64 * 1024]byte)(unsafe.Pointer(&data[0])) + b := float64ToByteSlice(data) for len(a) > 0 { n := len(data) if n > len(a) { @@ -116,15 +116,30 @@ func isFloat64Data(a, data []float64) bool { } x := a[:n] a = a[n:] - xb := (*[64 * 1024]byte)(unsafe.Pointer(&x[0])) - xbLen := len(x) * 8 - if !bytes.Equal(xb[:xbLen], b[:xbLen]) { + xb := float64ToByteSlice(x) + if !bytes.Equal(xb, b[:len(xb)]) { return false } } return true } +func int64ToByteSlice(a []int64) (b []byte) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + sh.Data = uintptr(unsafe.Pointer(&a[0])) + sh.Len = len(a) * int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + +func float64ToByteSlice(a []float64) (b []byte) { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + sh.Data = uintptr(unsafe.Pointer(&a[0])) + sh.Len = len(a) * int(unsafe.Sizeof(a[0])) + sh.Cap = sh.Len + return +} + var ( int64Zeros [8 * 1024]int64 int64Ones = func() (a [8 * 1024]int64) {