From 5f33fc8e46ae8dba1fa37420f20fae131523d9fe Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 22 Aug 2019 12:27:18 +0300 Subject: [PATCH] app/vminsert: add ability to ingest data via HTTP OpenTSDB `/api/put` requests This is manual merge of the https://github.com/VictoriaMetrics/VictoriaMetrics/pull/152 Thanks to nustinov@gmail.com for the initial pull request. --- README.md | 55 +- app/vminsert/common/gzip_reader.go | 30 + app/vminsert/graphite/parser.go | 3 - app/vminsert/influx/parser.go | 3 - app/vminsert/influx/request_handler.go | 24 +- app/vminsert/main.go | 14 +- app/vminsert/opentsdb/parser.go | 3 - app/vminsert/opentsdb/request_handler.go | 14 +- app/vminsert/opentsdbhttp/parser.go | 192 ++++ app/vminsert/opentsdbhttp/parser_test.go | 223 ++++ .../opentsdbhttp/parser_timing_test.go | 32 + app/vminsert/opentsdbhttp/request_handler.go | 153 +++ app/vminsert/opentsdbhttp/server.go | 70 ++ vendor/github.com/valyala/fastjson/.gitignore | 1 + .../github.com/valyala/fastjson/.travis.yml | 19 + vendor/github.com/valyala/fastjson/README.md | 212 ++++ vendor/github.com/valyala/fastjson/arena.go | 126 +++ vendor/github.com/valyala/fastjson/doc.go | 9 + vendor/github.com/valyala/fastjson/go.mod | 1 + vendor/github.com/valyala/fastjson/handy.go | 170 +++ vendor/github.com/valyala/fastjson/parser.go | 964 ++++++++++++++++++ vendor/github.com/valyala/fastjson/pool.go | 52 + vendor/github.com/valyala/fastjson/scanner.go | 94 ++ vendor/github.com/valyala/fastjson/update.go | 110 ++ vendor/github.com/valyala/fastjson/util.go | 30 + .../github.com/valyala/fastjson/validate.go | 308 ++++++ vendor/modules.txt | 1 + 27 files changed, 2872 insertions(+), 41 deletions(-) create mode 100644 app/vminsert/common/gzip_reader.go create mode 100644 app/vminsert/opentsdbhttp/parser.go create mode 100644 app/vminsert/opentsdbhttp/parser_test.go create mode 100644 app/vminsert/opentsdbhttp/parser_timing_test.go create mode 100644 app/vminsert/opentsdbhttp/request_handler.go create mode 100644 app/vminsert/opentsdbhttp/server.go create mode 100644 vendor/github.com/valyala/fastjson/.gitignore create mode 100644 vendor/github.com/valyala/fastjson/.travis.yml create mode 100644 vendor/github.com/valyala/fastjson/README.md create mode 100644 vendor/github.com/valyala/fastjson/arena.go create mode 100644 vendor/github.com/valyala/fastjson/doc.go create mode 100644 vendor/github.com/valyala/fastjson/go.mod create mode 100644 vendor/github.com/valyala/fastjson/handy.go create mode 100644 vendor/github.com/valyala/fastjson/parser.go create mode 100644 vendor/github.com/valyala/fastjson/pool.go create mode 100644 vendor/github.com/valyala/fastjson/scanner.go create mode 100644 vendor/github.com/valyala/fastjson/update.go create mode 100644 vendor/github.com/valyala/fastjson/util.go create mode 100644 vendor/github.com/valyala/fastjson/validate.go diff --git a/README.md b/README.md index 09b40326a..98acbfc97 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Cluster version is available [here](https://github.com/VictoriaMetrics/VictoriaM * [Graphite plaintext protocol](https://graphite.readthedocs.io/en/latest/feeding-carbon.html) with [tags](https://graphite.readthedocs.io/en/latest/tags.html#carbon) if `-graphiteListenAddr` is set. * [OpenTSDB put message](http://opentsdb.net/docs/build/html/api_telnet/put.html) if `-opentsdbListenAddr` is set. + * [HTTP OpenTSDB /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) if `-opentsdbHTTPListenAddr` is set. * Ideally works with big amounts of time series data from Kubernetes, IoT sensors, connected cars and industrial telemetry. * Has open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster). @@ -109,7 +110,8 @@ The following command-line flags are used the most: * `-retentionPeriod` - retention period in months for the data. Older data is automatically deleted. * `-httpListenAddr` - TCP address to listen to for http requests. By default, it listens port `8428` on all the network interfaces. * `-graphiteListenAddr` - TCP and UDP address to listen to for Graphite data. By default, it is disabled. -* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data. By default, it is disabled. +* `-opentsdbListenAddr` - TCP and UDP address to listen to for OpenTSDB data over telnet protocol. By default, it is disabled. +* `-opentsdbHTTPListenAddr` - TCP address to listen to for HTTP OpenTSDB data over `/api/put`. By default, it is disabled. Pass `-help` to see all the available flags with description and default values. @@ -237,7 +239,7 @@ An arbitrary number of lines delimited by '\n' may be sent in a single request. After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint: ``` -curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}' +curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}' ``` The `/api/v1/export` endpoint should return the following response: @@ -275,7 +277,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go. After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint: ``` -curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}' +curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}' ``` The `/api/v1/export` endpoint should return the following response: @@ -295,8 +297,13 @@ or via [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/mas ### How to send data from OpenTSDB-compatible agents? +VictoriaMetrics supports [telnet put protocol](http://opentsdb.net/docs/build/html/api_telnet/put.html) +and [HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html) for ingesting OpenTSDB data. + +#### Sending data via `telnet put` protocol + 1) Enable OpenTSDB receiver in VictoriaMetrics by setting `-opentsdbListenAddr` command line flag. For instance, -the following command will enable OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`: +the following command enables OpenTSDB receiver in VictoriaMetrics on TCP and UDP port `4242`: ``` /path/to/victoria-metrics-prod -opentsdbListenAddr=:4242 @@ -315,7 +322,7 @@ An arbitrary number of lines delimited by `\n` may be sent in one go. After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint: ``` -curl -G 'http://localhost:8428/api/v1/export' --data-urlencode 'match={__name__!=""}' +curl -G 'http://localhost:8428/api/v1/export' -d 'match={__name__!=""}' ``` The `/api/v1/export` endpoint should return the following response: @@ -325,6 +332,44 @@ The `/api/v1/export` endpoint should return the following response: ``` +#### Sending OpenTSDB data via HTTP `/api/put` requests + +1) Enable HTTP server for OpenTSDB `/api/put` requests by setting `-opentsdbHTTPListenAddr` command line flag. For instance, +the following command enables OpenTSDB HTTP server on port `4242`: + +``` +/path/to/victoria-metrics-prod -opentsdbHTTPListenAddr=:4242 +``` + +2) Send data to the given address from OpenTSDB-compatible agents. + +Example for writing a single data point: + +``` +curl -H 'Content-Type: application/json' -d '{"metric":"x.y.z","value":45.34,"tags":{"t1":"v1","t2":"v2"}}' http://localhost:4242/api/put +``` + +Example for writing multiple data points in a single request: + +``` +curl -H 'Content-Type: application/json' -d '[{"metric":"foo","value":45.34},{"metric":"bar","value":43}]' http://localhost:4242/api/put +``` + +After that the data may be read via [/api/v1/export](#how-to-export-time-series) endpoint: + +``` +curl -G 'http://localhost:8428/api/v1/export' -d 'match[]=x.y.z' -d 'match[]=foo' -d 'match[]=bar' +``` + +The `/api/v1/export` endpoint should return the following response: + +``` +{"metric":{"__name__":"foo"},"values":[45.34],"timestamps":[1566464846000]} +{"metric":{"__name__":"bar"},"values":[43],"timestamps":[1566464846000]} +{"metric":{"__name__":"x.y.z","t1":"v1","t2":"v2"},"values":[45.34],"timestamps":[1566464763000]} +``` + + ### How to build from sources We recommend using either [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases) or diff --git a/app/vminsert/common/gzip_reader.go b/app/vminsert/common/gzip_reader.go new file mode 100644 index 000000000..4cc9701e7 --- /dev/null +++ b/app/vminsert/common/gzip_reader.go @@ -0,0 +1,30 @@ +package common + +import ( + "compress/gzip" + "io" + "sync" +) + +// GetGzipReader returns new gzip reader from the pool. +// +// Return back the gzip reader when it no longer needed with PutGzipReader. +func GetGzipReader(r io.Reader) (*gzip.Reader, error) { + v := gzipReaderPool.Get() + if v == nil { + return gzip.NewReader(r) + } + zr := v.(*gzip.Reader) + if err := zr.Reset(r); err != nil { + return nil, err + } + return zr, nil +} + +// PutGzipReader returns back gzip reader obtained via GetGzipReader. +func PutGzipReader(zr *gzip.Reader) { + _ = zr.Close() + gzipReaderPool.Put(zr) +} + +var gzipReaderPool sync.Pool diff --git a/app/vminsert/graphite/parser.go b/app/vminsert/graphite/parser.go index a33ac2fa4..e604fcc55 100644 --- a/app/vminsert/graphite/parser.go +++ b/app/vminsert/graphite/parser.go @@ -37,9 +37,6 @@ func (rs *Rows) Reset() { func (rs *Rows) Unmarshal(s string) error { var err error rs.Rows, rs.tagsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0]) - if err != nil { - return err - } return err } diff --git a/app/vminsert/influx/parser.go b/app/vminsert/influx/parser.go index a8568726c..6e5066bcb 100644 --- a/app/vminsert/influx/parser.go +++ b/app/vminsert/influx/parser.go @@ -44,9 +44,6 @@ func (rs *Rows) Reset() { func (rs *Rows) Unmarshal(s string) error { var err error rs.Rows, rs.tagsPool, rs.fieldsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0], rs.fieldsPool[:0]) - if err != nil { - return err - } return err } diff --git a/app/vminsert/influx/request_handler.go b/app/vminsert/influx/request_handler.go index d121d7753..91c261860 100644 --- a/app/vminsert/influx/request_handler.go +++ b/app/vminsert/influx/request_handler.go @@ -1,7 +1,6 @@ package influx import ( - "compress/gzip" "flag" "fmt" "io" @@ -41,11 +40,11 @@ func insertHandlerInternal(req *http.Request) error { r := req.Body if req.Header.Get("Content-Encoding") == "gzip" { - zr, err := getGzipReader(r) + zr, err := common.GetGzipReader(r) if err != nil { return fmt.Errorf("cannot read gzipped influx line protocol data: %s", err) } - defer putGzipReader(zr) + defer common.PutGzipReader(zr) r = zr } @@ -120,25 +119,6 @@ func (ctx *pushCtx) InsertRows(db string) error { return ic.FlushBufs() } -func getGzipReader(r io.Reader) (*gzip.Reader, error) { - v := gzipReaderPool.Get() - if v == nil { - return gzip.NewReader(r) - } - zr := v.(*gzip.Reader) - if err := zr.Reset(r); err != nil { - return nil, err - } - return zr, nil -} - -func putGzipReader(zr *gzip.Reader) { - _ = zr.Close() - gzipReaderPool.Put(zr) -} - -var gzipReaderPool sync.Pool - func (ctx *pushCtx) Read(r io.Reader, tsMultiplier int64) bool { if ctx.err != nil { return false diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 263fb080e..62ff78001 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -10,15 +10,17 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/graphite" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/influx" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdb" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/opentsdbhttp" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/prometheus" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/metrics" ) var ( - graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty") - opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty") - maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes") + graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty") + opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB put messages. Usually :4242 must be set. Doesn't work if empty") + opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty") + maxInsertRequestSize = flag.Int("maxInsertRequestSize", 32*1024*1024, "The maximum size of a single insert request in bytes") ) // Init initializes vminsert. @@ -30,6 +32,9 @@ func Init() { if len(*opentsdbListenAddr) > 0 { go opentsdb.Serve(*opentsdbListenAddr) } + if len(*opentsdbHTTPListenAddr) > 0 { + go opentsdbhttp.Serve(*opentsdbHTTPListenAddr, int64(*maxInsertRequestSize)) + } } // Stop stops vminsert. @@ -40,6 +45,9 @@ func Stop() { if len(*opentsdbListenAddr) > 0 { opentsdb.Stop() } + if len(*opentsdbHTTPListenAddr) > 0 { + opentsdbhttp.Stop() + } } // RequestHandler is a handler for Prometheus remote storage write API diff --git a/app/vminsert/opentsdb/parser.go b/app/vminsert/opentsdb/parser.go index e68b90817..02b750d3b 100644 --- a/app/vminsert/opentsdb/parser.go +++ b/app/vminsert/opentsdb/parser.go @@ -37,9 +37,6 @@ func (rs *Rows) Reset() { func (rs *Rows) Unmarshal(s string) error { var err error rs.Rows, rs.tagsPool, err = unmarshalRows(rs.Rows[:0], s, rs.tagsPool[:0]) - if err != nil { - return err - } return err } diff --git a/app/vminsert/opentsdb/request_handler.go b/app/vminsert/opentsdb/request_handler.go index bc96c0a82..6ce881030 100644 --- a/app/vminsert/opentsdb/request_handler.go +++ b/app/vminsert/opentsdb/request_handler.go @@ -91,9 +91,19 @@ func (ctx *pushCtx) Read(r io.Reader) bool { return false } + // Fill in missing timestamps + currentTimestamp := time.Now().Unix() + rows := ctx.Rows.Rows + for i := range rows { + r := &rows[i] + if r.Timestamp == 0 { + r.Timestamp = currentTimestamp + } + } + // Convert timestamps from seconds to milliseconds - for i := range ctx.Rows.Rows { - ctx.Rows.Rows[i].Timestamp *= 1e3 + for i := range rows { + rows[i].Timestamp *= 1e3 } return true } diff --git a/app/vminsert/opentsdbhttp/parser.go b/app/vminsert/opentsdbhttp/parser.go new file mode 100644 index 000000000..c1cb32c43 --- /dev/null +++ b/app/vminsert/opentsdbhttp/parser.go @@ -0,0 +1,192 @@ +package opentsdbhttp + +import ( + "fmt" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/valyala/fastjson" + "github.com/valyala/fastjson/fastfloat" +) + +// Rows contains parsed OpenTSDB rows. +type Rows struct { + Rows []Row + + tagsPool []Tag +} + +// Reset resets rs. +func (rs *Rows) Reset() { + // Release references to objects, so they can be GC'ed. + for i := range rs.Rows { + rs.Rows[i].reset() + } + rs.Rows = rs.Rows[:0] + + for i := range rs.tagsPool { + rs.tagsPool[i].reset() + } + rs.tagsPool = rs.tagsPool[:0] +} + +// Unmarshal unmarshals OpenTSDB rows from av. +// +// See http://opentsdb.net/docs/build/html/api_http/put.html +// +// s must be unchanged until rs is in use. +func (rs *Rows) Unmarshal(av *fastjson.Value) error { + var err error + rs.Rows, rs.tagsPool, err = unmarshalRows(rs.Rows[:0], av, rs.tagsPool[:0]) + return err +} + +// Row is a single OpenTSDB row. +type Row struct { + Metric string + Tags []Tag + Value float64 + Timestamp int64 +} + +func (r *Row) reset() { + r.Metric = "" + r.Tags = nil + r.Value = 0 + r.Timestamp = 0 +} + +func (r *Row) unmarshal(o *fastjson.Value, tagsPool []Tag) ([]Tag, error) { + r.reset() + m := o.GetStringBytes("metric") + if m == nil { + return tagsPool, fmt.Errorf("missing `metric` in %s", o) + } + r.Metric = bytesutil.ToUnsafeString(m) + + rawTs := o.Get("timestamp") + if rawTs != nil { + ts, err := rawTs.Int64() + if err != nil { + return tagsPool, fmt.Errorf("invalid `timestamp` in %s: %s", o, err) + } + r.Timestamp = int64(ts) + } else { + // Allow missing timestamp. It is automatically populated + // with the current time in this case. + r.Timestamp = 0 + } + + rawV := o.Get("value") + if rawV == nil { + return tagsPool, fmt.Errorf("missing `value` in %s", o) + } + v, err := getValue(rawV) + if err != nil { + return tagsPool, fmt.Errorf("invalid `value` in %s: %s", o, err) + } + r.Value = v + + vt := o.Get("tags") + if vt == nil { + // Allow empty tags. + return tagsPool, nil + } + rawTags, err := vt.Object() + if err != nil { + return tagsPool, fmt.Errorf("invalid `tags` in %s: %s", o, err) + } + + tagsStart := len(tagsPool) + tagsPool, err = unmarshalTags(tagsPool, rawTags) + if err != nil { + return tagsPool, fmt.Errorf("cannot parse tags %s: %s", rawTags, err) + } + tags := tagsPool[tagsStart:] + r.Tags = tags[:len(tags):len(tags)] + return tagsPool, nil +} + +func getValue(v *fastjson.Value) (float64, error) { + switch v.Type() { + case fastjson.TypeNumber: + return v.Float64() + case fastjson.TypeString: + vStr, _ := v.StringBytes() + vFloat := fastfloat.ParseBestEffort(bytesutil.ToUnsafeString(vStr)) + if vFloat == 0 && string(vStr) != "0" && string(vStr) != "0.0" { + return 0, fmt.Errorf("invalid float64 value: %q", vStr) + } + return vFloat, nil + default: + return 0, fmt.Errorf("value doesn't contain float64; it contains %s", v.Type()) + } +} + +func unmarshalRows(dst []Row, av *fastjson.Value, tagsPool []Tag) ([]Row, []Tag, error) { + switch av.Type() { + case fastjson.TypeObject: + return unmarshalRow(dst, av, tagsPool) + case fastjson.TypeArray: + a, _ := av.Array() + for i, o := range a { + var err error + dst, tagsPool, err = unmarshalRow(dst, o, tagsPool) + if err != nil { + return dst, tagsPool, fmt.Errorf("cannot unmarshal %d object out of %d objects: %s", i, len(a), err) + } + } + return dst, tagsPool, nil + default: + return dst, tagsPool, fmt.Errorf("OpenTSDB body must be either object or array; got %s; body=%s", av.Type(), av) + } +} + +func unmarshalRow(dst []Row, o *fastjson.Value, tagsPool []Tag) ([]Row, []Tag, error) { + if cap(dst) > len(dst) { + dst = dst[:len(dst)+1] + } else { + dst = append(dst, Row{}) + } + r := &dst[len(dst)-1] + var err error + tagsPool, err = r.unmarshal(o, tagsPool) + if err != nil { + return dst, tagsPool, fmt.Errorf("cannot unmarshal OpenTSDB object %s: %s", o, err) + } + return dst, tagsPool, nil +} + +func unmarshalTags(dst []Tag, o *fastjson.Object) ([]Tag, error) { + var err error + o.Visit(func(k []byte, v *fastjson.Value) { + if v.Type() != fastjson.TypeString { + err = fmt.Errorf("tag value must be string; got %s; value=%s", v.Type(), v) + return + } + vStr, _ := v.StringBytes() + if len(vStr) == 0 { + // Skip empty tags + return + } + if cap(dst) > len(dst) { + dst = dst[:len(dst)+1] + } else { + dst = append(dst, Tag{}) + } + tag := &dst[len(dst)-1] + tag.Key = bytesutil.ToUnsafeString(k) + tag.Value = bytesutil.ToUnsafeString(vStr) + }) + return dst, err +} + +// Tag is an OpenTSDB tag. +type Tag struct { + Key string + Value string +} + +func (t *Tag) reset() { + t.Key = "" + t.Value = "" +} diff --git a/app/vminsert/opentsdbhttp/parser_test.go b/app/vminsert/opentsdbhttp/parser_test.go new file mode 100644 index 000000000..58375d9ba --- /dev/null +++ b/app/vminsert/opentsdbhttp/parser_test.go @@ -0,0 +1,223 @@ +package opentsdbhttp + +import ( + "reflect" + "testing" +) + +func TestRowsUnmarshalFailure(t *testing.T) { + f := func(s string) { + t.Helper() + var rows Rows + p := parserPool.Get() + defer parserPool.Put(p) + v, err := p.Parse(s) + if err != nil { + // Expected JSON parser error + return + } + // Verify OpenTSDB body parsing error + if err := rows.Unmarshal(v); err == nil { + t.Fatalf("expecting non-nil error when parsing %q", s) + } + // Try again + if err := rows.Unmarshal(v); err == nil { + t.Fatalf("expecting non-nil error when parsing %q", s) + } + } + + // invalid json + f("{g") + + // Invalid json type + f(`1`) + f(`"foo"`) + f(`[1,2]`) + f(`null`) + + // Incomplete object + f(`{}`) + f(`{"metric": "aaa"}`) + f(`{"metric": "aaa", "timestamp": 1122}`) + f(`{"metric": "aaa", "timestamp": "tststs"}`) + f(`{"timestamp": 1122, "value": 33}`) + f(`{"value": 33}`) + f(`{"value": 33, "tags": {"fooo":"bar"}}`) + + // Invalid value + f(`{"metric": "aaa", "timestamp": 1122, "value": "0.0.0"}`) + + // Invalid metric type + f(`{"metric": ["aaa"], "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`) + f(`{"metric": {"aaa":1}, "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`) + f(`{"metric": 1, "timestamp": 1122, "value": 0.45, "tags": {"foo": "bar"}}`) + + // Invalid timestamp type + f(`{"metric": "aaa", "timestamp": "foobar", "value": 0.45, "tags": {"foo": "bar"}}`) + f(`{"metric": "aaa", "timestamp": 123.456, "value": 0.45, "tags": {"foo": "bar"}}`) + f(`{"metric": "aaa", "timestamp": "123", "value": 0.45, "tags": {"foo": "bar"}}`) + + // Invalid value type + f(`{"metric": "aaa", "timestamp": 1122, "value": [0,1], "tags": {"foo":"bar"}}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": {"a":1}, "tags": {"foo":"bar"}}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": "foobar", "tags": {"foo":"bar"}}`) + + // Invalid tags type + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": 1}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": [1,2]}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": "foo"}`) + + // Invalid tag value type + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": ["bar"]}}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": {"bar":"baz"}}}`) + f(`{"metric": "aaa", "timestamp": 1122, "value": 0.45, "tags": {"foo": 1}}`) + + // Invalid multiline + f(`[{"metric": "aaa", "timestamp": 1122, "value": "trt", "tags":{"foo":"bar"}}, {"metric": "aaa", "timestamp": 1122, "value": 111}]`) +} + +func TestRowsUnmarshalSuccess(t *testing.T) { + f := func(s string, rowsExpected *Rows) { + t.Helper() + var rows Rows + + p := parserPool.Get() + defer parserPool.Put(p) + v, err := p.Parse(s) + if err != nil { + t.Fatalf("cannot parse json %s: %s", s, err) + } + if err := rows.Unmarshal(v); err != nil { + t.Fatalf("cannot unmarshal %s: %s", v, err) + } + if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) { + t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) + } + + // Try unmarshaling again + if err := rows.Unmarshal(v); err != nil { + t.Fatalf("cannot unmarshal %s: %s", v, err) + } + if !reflect.DeepEqual(rows.Rows, rowsExpected.Rows) { + t.Fatalf("unexpected rows;\ngot\n%+v;\nwant\n%+v", rows.Rows, rowsExpected.Rows) + } + + rows.Reset() + if len(rows.Rows) != 0 { + t.Fatalf("non-empty rows after reset: %+v", rows.Rows) + } + } + + // Normal line + f(`{"metric": "foobar", "timestamp": 789, "value": -123.456, "tags": {"a":"b"}}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -123.456, + Timestamp: 789, + Tags: []Tag{{ + Key: "a", + Value: "b", + }}, + }}, + }) + // Empty tags + f(`{"metric": "foobar", "timestamp": 789, "value": -123.456, "tags": {}}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -123.456, + Timestamp: 789, + Tags: nil, + }}, + }) + // Missing tags + f(`{"metric": "foobar", "timestamp": 789, "value": -123.456}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -123.456, + Timestamp: 789, + Tags: nil, + }}, + }) + // Empty tag value + f(`{"metric": "foobar", "timestamp": 123, "value": -123.456, "tags": {"a":"", "b":"c"}}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -123.456, + Timestamp: 123, + Tags: []Tag{ + { + Key: "b", + Value: "c", + }, + }, + }}, + }) + // Value as string + f(`{"metric": "foobar", "timestamp": 789, "value": "-12.456", "tags": {"a":"b"}}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -12.456, + Timestamp: 789, + Tags: []Tag{{ + Key: "a", + Value: "b", + }}, + }}, + }) + // Missing timestamp + f(`{"metric": "foobar", "value": "-12.456", "tags": {"a":"b"}}`, &Rows{ + Rows: []Row{{ + Metric: "foobar", + Value: -12.456, + Timestamp: 0, + Tags: []Tag{{ + Key: "a", + Value: "b", + }}, + }}, + }) + + // Multiple tags + f(`{"metric": "foo", "value": 1, "timestamp": 2, "tags": {"bar":"baz", "x": "y"}}`, &Rows{ + Rows: []Row{{ + Metric: "foo", + Tags: []Tag{ + { + Key: "bar", + Value: "baz", + }, + { + Key: "x", + Value: "y", + }, + }, + Value: 1, + Timestamp: 2, + }}, + }) + + // Multi lines + f(`[{"metric": "foo", "value": "0.3", "timestamp": 2, "tags": {"a":"b"}}, +{"metric": "bar.baz", "value": 0.34, "timestamp": 43, "tags": {"a":"b"}}]`, &Rows{ + Rows: []Row{ + { + Metric: "foo", + Value: 0.3, + Timestamp: 2, + Tags: []Tag{{ + Key: "a", + Value: "b", + }}, + }, + { + Metric: "bar.baz", + Value: 0.34, + Timestamp: 43, + Tags: []Tag{{ + Key: "a", + Value: "b", + }}, + }, + }, + }) +} diff --git a/app/vminsert/opentsdbhttp/parser_timing_test.go b/app/vminsert/opentsdbhttp/parser_timing_test.go new file mode 100644 index 000000000..844464600 --- /dev/null +++ b/app/vminsert/opentsdbhttp/parser_timing_test.go @@ -0,0 +1,32 @@ +package opentsdbhttp + +import ( + "fmt" + "testing" + + "github.com/valyala/fastjson" +) + +func BenchmarkRowsUnmarshal(b *testing.B) { + s := `[{"metric": "cpu.usage_user", "timestamp": 1234556768, "value": 1.23, "tags": {"a":"b", "x": "y"}}, +{"metric": "cpu.usage_system", "timestamp": 1234556768, "value": 23.344, "tags": {"a":"b"}}, +{"metric": "cpu.usage_iowait", "timestamp": 1234556769, "value":3.3443, "tags": {"a":"b"}}, +{"metric": "cpu.usage_irq", "timestamp": 1234556768, "value": 0.34432, "tags": {"a":"b"}} +] +` + b.SetBytes(int64(len(s))) + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + var rows Rows + var p fastjson.Parser + for pb.Next() { + v, err := p.Parse(s) + if err != nil { + panic(fmt.Errorf("cannot parse %q: %s", s, err)) + } + if err := rows.Unmarshal(v); err != nil { + panic(fmt.Errorf("cannot unmarshal %q: %s", s, err)) + } + } + }) +} diff --git a/app/vminsert/opentsdbhttp/request_handler.go b/app/vminsert/opentsdbhttp/request_handler.go new file mode 100644 index 000000000..cc449d9c0 --- /dev/null +++ b/app/vminsert/opentsdbhttp/request_handler.go @@ -0,0 +1,153 @@ +package opentsdbhttp + +import ( + "fmt" + "io" + "net/http" + "runtime" + "sync" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/concurrencylimiter" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/metrics" + "github.com/valyala/fastjson" +) + +var ( + rowsInserted = metrics.NewCounter(`vm_rows_inserted_total{type="opentsdb-http"}`) + rowsPerInsert = metrics.NewSummary(`vm_rows_per_insert{type="opentsdb-http"}`) + + opentsdbReadCalls = metrics.NewCounter(`vm_read_calls_total{name="opentsdb-http"}`) + opentsdbReadErrors = metrics.NewCounter(`vm_read_errors_total{name="opentsdb-http"}`) + opentsdbUnmarshalErrors = metrics.NewCounter(`vm_unmarshal_errors_total{name="opentsdb-http"}`) +) + +// insertHandler processes HTTP OpenTSDB put requests. +// See http://opentsdb.net/docs/build/html/api_http/put.html +func insertHandler(req *http.Request, maxSize int64) error { + return concurrencylimiter.Do(func() error { + return insertHandlerInternal(req, maxSize) + }) +} + +func insertHandlerInternal(req *http.Request, maxSize int64) error { + opentsdbReadCalls.Inc() + + r := req.Body + if req.Header.Get("Content-Encoding") == "gzip" { + zr, err := common.GetGzipReader(r) + if err != nil { + opentsdbReadErrors.Inc() + return fmt.Errorf("cannot read gzipped http protocol data: %s", err) + } + defer common.PutGzipReader(zr) + r = zr + } + + ctx := getPushCtx() + defer putPushCtx(ctx) + + // Read the request in ctx.reqBuf + lr := io.LimitReader(r, maxSize+1) + reqLen, err := ctx.reqBuf.ReadFrom(lr) + if err != nil { + opentsdbReadErrors.Inc() + return fmt.Errorf("cannot read HTTP OpenTSDB request: %s", err) + } + if reqLen > maxSize { + opentsdbReadErrors.Inc() + return fmt.Errorf("too big HTTP OpenTSDB request; mustn't exceed %d bytes", maxSize) + } + + // Unmarshal the request to ctx.Rows + p := parserPool.Get() + defer parserPool.Put(p) + v, err := p.ParseBytes(ctx.reqBuf.B) + if err != nil { + opentsdbUnmarshalErrors.Inc() + return fmt.Errorf("cannot parse HTTP OpenTSDB json: %s", err) + } + if err := ctx.Rows.Unmarshal(v); err != nil { + opentsdbUnmarshalErrors.Inc() + return fmt.Errorf("cannot unmarshal HTTP OpenTSDB json %s, %s", err, v) + } + + // Fill in missing timestamps + currentTimestamp := time.Now().Unix() + rows := ctx.Rows.Rows + for i := range rows { + r := &rows[i] + if r.Timestamp == 0 { + r.Timestamp = currentTimestamp + } + } + + // Convert timestamps in seconds to milliseconds if needed. + // See http://opentsdb.net/docs/javadoc/net/opentsdb/core/Const.html#SECOND_MASK + for i := range rows { + r := &rows[i] + if r.Timestamp&secondMask == 0 { + r.Timestamp *= 1e3 + } + } + + // Insert ctx.Rows to db. + ic := &ctx.Common + ic.Reset(len(rows)) + for i := range rows { + r := &rows[i] + ic.Labels = ic.Labels[:0] + ic.AddLabel("", r.Metric) + for j := range r.Tags { + tag := &r.Tags[j] + ic.AddLabel(tag.Key, tag.Value) + } + ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, r.Value) + } + rowsInserted.Add(len(rows)) + rowsPerInsert.Update(float64(len(rows))) + return ic.FlushBufs() +} + +const secondMask int64 = 0x7FFFFFFF00000000 + +var parserPool fastjson.ParserPool + +type pushCtx struct { + Rows Rows + Common common.InsertCtx + + reqBuf bytesutil.ByteBuffer +} + +func (ctx *pushCtx) reset() { + ctx.Rows.Reset() + ctx.Common.Reset(0) + ctx.reqBuf.Reset() +} + +func getPushCtx() *pushCtx { + select { + case ctx := <-pushCtxPoolCh: + return ctx + default: + if v := pushCtxPool.Get(); v != nil { + return v.(*pushCtx) + } + return &pushCtx{} + } +} + +func putPushCtx(ctx *pushCtx) { + ctx.reset() + select { + case pushCtxPoolCh <- ctx: + default: + pushCtxPool.Put(ctx) + } +} + +var pushCtxPool sync.Pool +var pushCtxPoolCh = make(chan *pushCtx, runtime.GOMAXPROCS(-1)) diff --git a/app/vminsert/opentsdbhttp/server.go b/app/vminsert/opentsdbhttp/server.go new file mode 100644 index 000000000..7a06f55da --- /dev/null +++ b/app/vminsert/opentsdbhttp/server.go @@ -0,0 +1,70 @@ +package opentsdbhttp + +import ( + "context" + "net/http" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/metrics" +) + +var ( + writeRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/put", protocol="opentsdb-http"}`) + writeErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/api/put", protocol="opentsdb-http"}`) +) + +var ( + httpServer *http.Server + httpAddr string + maxRequestSize int64 +) + +// Serve starts HTTP OpenTSDB server on the given addr. +func Serve(addr string, maxReqSize int64) { + logger.Infof("starting HTTP OpenTSDB server at %q", addr) + httpAddr = addr + maxRequestSize = maxReqSize + httpServer = &http.Server{ + Addr: addr, + Handler: http.HandlerFunc(requestHandler), + ReadTimeout: 30 * time.Second, + WriteTimeout: 10 * time.Second, + } + go func() { + err := httpServer.ListenAndServe() + if err == http.ErrServerClosed { + return + } + if err != nil { + logger.Fatalf("FATAL: error serving HTTP OpenTSDB: %s", err) + } + }() +} + +// requestHandler handles HTTP OpenTSDB insert request. +func requestHandler(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/put": + writeRequests.Inc() + if err := insertHandler(r, maxRequestSize); err != nil { + writeErrors.Inc() + httpserver.Errorf(w, "error in %q: %s", r.URL.Path, err) + return + } + w.WriteHeader(http.StatusNoContent) + default: + httpserver.Errorf(w, "unexpected path requested on HTTP OpenTSDB server: %q", r.URL.Path) + } +} + +// Stop stops HTTP OpenTSDB server. +func Stop() { + logger.Infof("stopping HTTP OpenTSDB server at %q...", httpAddr) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := httpServer.Shutdown(ctx); err != nil { + logger.Fatalf("FATAL: cannot close HTTP OpenTSDB server: %s", err) + } +} diff --git a/vendor/github.com/valyala/fastjson/.gitignore b/vendor/github.com/valyala/fastjson/.gitignore new file mode 100644 index 000000000..6e92f57d4 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/.gitignore @@ -0,0 +1 @@ +tags diff --git a/vendor/github.com/valyala/fastjson/.travis.yml b/vendor/github.com/valyala/fastjson/.travis.yml new file mode 100644 index 000000000..472a82190 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/.travis.yml @@ -0,0 +1,19 @@ +language: go + +go: + - 1.10.x + +script: + # build test for supported platforms + - GOOS=linux go build + - GOOS=darwin go build + - GOOS=freebsd go build + - GOOS=windows go build + + # run tests on a standard platform + - go test -v ./... -coverprofile=coverage.txt -covermode=atomic + - go test -v ./... -race + +after_success: + # Upload coverage results to codecov.io + - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/valyala/fastjson/README.md b/vendor/github.com/valyala/fastjson/README.md new file mode 100644 index 000000000..c04913a17 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/README.md @@ -0,0 +1,212 @@ +[![Build Status](https://travis-ci.org/valyala/fastjson.svg)](https://travis-ci.org/valyala/fastjson) +[![GoDoc](https://godoc.org/github.com/valyala/fastjson?status.svg)](http://godoc.org/github.com/valyala/fastjson) +[![Go Report](https://goreportcard.com/badge/github.com/valyala/fastjson)](https://goreportcard.com/report/github.com/valyala/fastjson) +[![codecov](https://codecov.io/gh/valyala/fastjson/branch/master/graph/badge.svg)](https://codecov.io/gh/valyala/fastjson) + +# fastjson - fast JSON parser and validator for Go + + +## Features + + * Fast. As usual, up to 15x faster than the standard [encoding/json](https://golang.org/pkg/encoding/json/). + See [benchmarks](#benchmarks). + * Parses arbitrary JSON without schema, reflection, struct magic and code generation + contrary to [easyjson](https://github.com/mailru/easyjson). + * Provides simple [API](http://godoc.org/github.com/valyala/fastjson). + * Outperforms [jsonparser](https://github.com/buger/jsonparser) and [gjson](https://github.com/tidwall/gjson) + when accessing multiple unrelated fields, since `fastjson` parses the input JSON only once. + * Validates the parsed JSON unlike [jsonparser](https://github.com/buger/jsonparser) + and [gjson](https://github.com/tidwall/gjson). + * May quickly extract a part of the original JSON with `Value.Get(...).MarshalTo` and modify it + with [Del](https://godoc.org/github.com/valyala/fastjson#Value.Del) + and [Set](https://godoc.org/github.com/valyala/fastjson#Value.Set) functions. + * May parse array containing values with distinct types (aka non-homogenous types). + For instance, `fastjson` easily parses the following JSON array `[123, "foo", [456], {"k": "v"}, null]`. + * `fastjson` preserves the original order of object items when calling + [Object.Visit](https://godoc.org/github.com/valyala/fastjson#Object.Visit). + + +## Known limitations + + * Requies extra care to work with - references to certain objects recursively + returned by [Parser](https://godoc.org/github.com/valyala/fastjson#Parser) + must be released before the next call to [Parse](https://godoc.org/github.com/valyala/fastjson#Parser.Parse). + Otherwise the program may work improperly. The same applies to objects returned by [Arena](https://godoc.org/github.com/valyala/fastjson#Arena). + Adhere recommendations from [docs](https://godoc.org/github.com/valyala/fastjson). + * Cannot parse JSON from `io.Reader`. There is [Scanner](https://godoc.org/github.com/valyala/fastjson#Scanner) + for parsing stream of JSON values from a string. + + +## Usage + +One-liner accessing a single field: +```go + s := []byte(`{"foo": [123, "bar"]}`) + fmt.Printf("foo.0=%d\n", fastjson.GetInt(s, "foo", "0")) + + // Output: + // foo.0=123 +``` + +Accessing multiple fields with error handling: +```go + var p fastjson.Parser + v, err := p.Parse(`{ + "str": "bar", + "int": 123, + "float": 1.23, + "bool": true, + "arr": [1, "foo", {}] + }`) + if err != nil { + log.Fatal(err) + } + fmt.Printf("foo=%s\n", v.GetStringBytes("str")) + fmt.Printf("int=%d\n", v.GetInt("int")) + fmt.Printf("float=%f\n", v.GetFloat64("float")) + fmt.Printf("bool=%v\n", v.GetBool("bool")) + fmt.Printf("arr.1=%s\n", v.GetStringBytes("arr", "1")) + + // Output: + // foo=bar + // int=123 + // float=1.230000 + // bool=true + // arr.1=foo +``` + +See also [examples](https://godoc.org/github.com/valyala/fastjson#pkg-examples). + + +## Security + + * `fastjson` shouldn't crash or panic when parsing input strings specially crafted + by an attacker. It must return error on invalid input JSON. + * `fastjson` requires up to `sizeof(Value) * len(inputJSON)` bytes of memory + for parsing `inputJSON` string. Limit the maximum size of the `inputJSON` + before parsing it in order to limit the maximum memory usage. + + +## Performance optimization tips + + * Re-use [Parser](https://godoc.org/github.com/valyala/fastjson#Parser) and [Scanner](https://godoc.org/github.com/valyala/fastjson#Scanner) + for parsing many JSONs. This reduces memory allocations overhead. + [ParserPool](https://godoc.org/github.com/valyala/fastjson#ParserPool) may be useful in this case. + * Prefer calling `Value.Get*` on the value returned from [Parser](https://godoc.org/github.com/valyala/fastjson#Parser) + instead of calling `Get*` one-liners when multiple fields + must be obtained from JSON, since each `Get*` one-liner re-parses + the input JSON again. + * Prefer calling once [Value.Get](https://godoc.org/github.com/valyala/fastjson#Value.Get) + for common prefix paths and then calling `Value.Get*` on the returned value + for distinct suffix paths. + * Prefer iterating over array returned from [Value.GetArray](https://godoc.org/github.com/valyala/fastjson#Object.Visit) + with a range loop instead of calling `Value.Get*` for each array item. + + +## Benchmarks + +Go 1.12 has been used for benchmarking. + +Legend: + + * `small` - parse [small.json](testdata/small.json) (190 bytes). + * `medium` - parse [medium.json](testdata/medium.json) (2.3KB). + * `large` - parse [large.json](testdata/large.json) (28KB). + * `canada` - parse [canada.json](testdata/canada.json) (2.2MB). + * `citm` - parse [citm_catalog.json](testdata/citm_catalog.json) (1.7MB). + * `twitter` - parse [twitter.json](testdata/twitter.json) (617KB). + + * `stdjson-map` - parse into a `map[string]interface{}` using `encoding/json`. + * `stdjson-struct` - parse into a struct containing + a subset of fields of the parsed JSON, using `encoding/json`. + * `stdjson-empty-struct` - parse into an empty struct using `encoding/json`. + This is the fastest possible solution for `encoding/json`, may be used + for json validation. See also benchmark results for json validation. + * `fastjson` - parse using `fastjson` without fields access. + * `fastjson-get` - parse using `fastjson` with fields access similar to `stdjson-struct`. + +``` +$ GOMAXPROCS=1 go test github.com/valyala/fastjson -bench='Parse$' +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastjson +BenchmarkParse/small/stdjson-map 200000 7305 ns/op 26.01 MB/s 960 B/op 51 allocs/op +BenchmarkParse/small/stdjson-struct 500000 3431 ns/op 55.37 MB/s 224 B/op 4 allocs/op +BenchmarkParse/small/stdjson-empty-struct 500000 2273 ns/op 83.58 MB/s 168 B/op 2 allocs/op +BenchmarkParse/small/fastjson 5000000 347 ns/op 547.53 MB/s 0 B/op 0 allocs/op +BenchmarkParse/small/fastjson-get 2000000 620 ns/op 306.39 MB/s 0 B/op 0 allocs/op +BenchmarkParse/medium/stdjson-map 30000 40672 ns/op 57.26 MB/s 10196 B/op 208 allocs/op +BenchmarkParse/medium/stdjson-struct 30000 47792 ns/op 48.73 MB/s 9174 B/op 258 allocs/op +BenchmarkParse/medium/stdjson-empty-struct 100000 22096 ns/op 105.40 MB/s 280 B/op 5 allocs/op +BenchmarkParse/medium/fastjson 500000 3025 ns/op 769.90 MB/s 0 B/op 0 allocs/op +BenchmarkParse/medium/fastjson-get 500000 3211 ns/op 725.20 MB/s 0 B/op 0 allocs/op +BenchmarkParse/large/stdjson-map 2000 614079 ns/op 45.79 MB/s 210734 B/op 2785 allocs/op +BenchmarkParse/large/stdjson-struct 5000 298554 ns/op 94.18 MB/s 15616 B/op 353 allocs/op +BenchmarkParse/large/stdjson-empty-struct 5000 268577 ns/op 104.69 MB/s 280 B/op 5 allocs/op +BenchmarkParse/large/fastjson 50000 35210 ns/op 798.56 MB/s 5 B/op 0 allocs/op +BenchmarkParse/large/fastjson-get 50000 35171 ns/op 799.46 MB/s 5 B/op 0 allocs/op +BenchmarkParse/canada/stdjson-map 20 68147307 ns/op 33.03 MB/s 12260502 B/op 392539 allocs/op +BenchmarkParse/canada/stdjson-struct 20 68044518 ns/op 33.08 MB/s 12260123 B/op 392534 allocs/op +BenchmarkParse/canada/stdjson-empty-struct 100 17709250 ns/op 127.11 MB/s 280 B/op 5 allocs/op +BenchmarkParse/canada/fastjson 300 4182404 ns/op 538.22 MB/s 254902 B/op 381 allocs/op +BenchmarkParse/canada/fastjson-get 300 4274744 ns/op 526.60 MB/s 254902 B/op 381 allocs/op +BenchmarkParse/citm/stdjson-map 50 27772612 ns/op 62.19 MB/s 5214163 B/op 95402 allocs/op +BenchmarkParse/citm/stdjson-struct 100 14936191 ns/op 115.64 MB/s 1989 B/op 75 allocs/op +BenchmarkParse/citm/stdjson-empty-struct 100 14946034 ns/op 115.56 MB/s 280 B/op 5 allocs/op +BenchmarkParse/citm/fastjson 1000 1879714 ns/op 918.87 MB/s 17628 B/op 30 allocs/op +BenchmarkParse/citm/fastjson-get 1000 1881598 ns/op 917.94 MB/s 17628 B/op 30 allocs/op +BenchmarkParse/twitter/stdjson-map 100 11289146 ns/op 55.94 MB/s 2187878 B/op 31266 allocs/op +BenchmarkParse/twitter/stdjson-struct 300 5779442 ns/op 109.27 MB/s 408 B/op 6 allocs/op +BenchmarkParse/twitter/stdjson-empty-struct 300 5738504 ns/op 110.05 MB/s 408 B/op 6 allocs/op +BenchmarkParse/twitter/fastjson 2000 774042 ns/op 815.86 MB/s 2541 B/op 2 allocs/op +BenchmarkParse/twitter/fastjson-get 2000 777833 ns/op 811.89 MB/s 2541 B/op 2 allocs/op +``` + +Benchmark results for json validation: + +``` +$ GOMAXPROCS=1 go test github.com/valyala/fastjson -bench='Validate$' +goos: linux +goarch: amd64 +pkg: github.com/valyala/fastjson +BenchmarkValidate/small/stdjson 2000000 955 ns/op 198.83 MB/s 72 B/op 2 allocs/op +BenchmarkValidate/small/fastjson 5000000 384 ns/op 493.60 MB/s 0 B/op 0 allocs/op +BenchmarkValidate/medium/stdjson 200000 10799 ns/op 215.66 MB/s 184 B/op 5 allocs/op +BenchmarkValidate/medium/fastjson 300000 3809 ns/op 611.30 MB/s 0 B/op 0 allocs/op +BenchmarkValidate/large/stdjson 10000 133064 ns/op 211.31 MB/s 184 B/op 5 allocs/op +BenchmarkValidate/large/fastjson 30000 45268 ns/op 621.14 MB/s 0 B/op 0 allocs/op +BenchmarkValidate/canada/stdjson 200 8470904 ns/op 265.74 MB/s 184 B/op 5 allocs/op +BenchmarkValidate/canada/fastjson 500 2973377 ns/op 757.07 MB/s 0 B/op 0 allocs/op +BenchmarkValidate/citm/stdjson 200 7273172 ns/op 237.48 MB/s 184 B/op 5 allocs/op +BenchmarkValidate/citm/fastjson 1000 1684430 ns/op 1025.39 MB/s 0 B/op 0 allocs/op +BenchmarkValidate/twitter/stdjson 500 2849439 ns/op 221.63 MB/s 312 B/op 6 allocs/op +BenchmarkValidate/twitter/fastjson 2000 1036796 ns/op 609.10 MB/s 0 B/op 0 allocs/op +``` + +## FAQ + + * Q: _There are a ton of other high-perf packages for JSON parsing in Go. Why creating yet another package?_ + A: Because other packages require either rigid JSON schema via struct magic + and code generation or perform poorly when multiple unrelated fields + must be obtained from the parsed JSON. + Additionally, `fastjson` provides nicer [API](http://godoc.org/github.com/valyala/fastjson). + + * Q: _What is the main purpose for `fastjson`?_ + A: High-perf JSON parsing for [RTB](https://www.iab.com/wp-content/uploads/2015/05/OpenRTB_API_Specification_Version_2_3_1.pdf) + and other [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) services. + + * Q: _Why fastjson doesn't provide fast marshaling (serialization)?_ + A: Actually it provides some sort of marshaling - see [Value.MarshalTo](https://godoc.org/github.com/valyala/fastjson#Value.MarshalTo). + But I'd recommend using [quicktemplate](https://github.com/valyala/quicktemplate#use-cases) + for high-performance JSON marshaling :) + + * Q: _`fastjson` crashes my program!_ + A: There is high probability of improper use. + * Make sure you don't hold references to objects recursively returned by `Parser` / `Scanner` + beyond the next `Parser.Parse` / `Scanner.Next` call + if such restriction is mentioned in [docs](https://github.com/valyala/fastjson/issues/new). + * Make sure you don't access `fastjson` objects from concurrently running goroutines + if such restriction is mentioned in [docs](https://github.com/valyala/fastjson/issues/new). + * Build and run your program with [-race](https://golang.org/doc/articles/race_detector.html) flag. + Make sure the race detector detects zero races. + * If your program continue crashing after fixing issues mentioned above, [file a bug](https://github.com/valyala/fastjson/issues/new). diff --git a/vendor/github.com/valyala/fastjson/arena.go b/vendor/github.com/valyala/fastjson/arena.go new file mode 100644 index 000000000..9fe21a48c --- /dev/null +++ b/vendor/github.com/valyala/fastjson/arena.go @@ -0,0 +1,126 @@ +package fastjson + +import ( + "strconv" +) + +// Arena may be used for fast creation and re-use of Values. +// +// Typical Arena lifecycle: +// +// 1) Construct Values via the Arena and Value.Set* calls. +// 2) Marshal the constructed Values with Value.MarshalTo call. +// 3) Reset all the constructed Values at once by Arena.Reset call. +// 4) Go to 1 and re-use the Arena. +// +// It is unsafe calling Arena methods from concurrent goroutines. +// Use per-goroutine Arenas or ArenaPool instead. +type Arena struct { + b []byte + c cache +} + +// Reset resets all the Values allocated by a. +// +// Values previously allocated by a cannot be used after the Reset call. +func (a *Arena) Reset() { + a.b = a.b[:0] + a.c.reset() +} + +// NewObject returns new empty object value. +// +// New entries may be added to the returned object via Set call. +// +// The returned object is valid until Reset is called on a. +func (a *Arena) NewObject() *Value { + v := a.c.getValue() + v.t = TypeObject + v.o.reset() + return v +} + +// NewArray returns new empty array value. +// +// New entries may be added to the returned array via Set* calls. +// +// The returned array is valid until Reset is called on a. +func (a *Arena) NewArray() *Value { + v := a.c.getValue() + v.t = TypeArray + v.a = v.a[:0] + return v +} + +// NewString returns new string value containing s. +// +// The returned string is valid until Reset is called on a. +func (a *Arena) NewString(s string) *Value { + v := a.c.getValue() + v.t = typeRawString + bLen := len(a.b) + a.b = escapeString(a.b, s) + v.s = b2s(a.b[bLen+1 : len(a.b)-1]) + return v +} + +// NewStringBytes returns new string value containing b. +// +// The returned string is valid until Reset is called on a. +func (a *Arena) NewStringBytes(b []byte) *Value { + v := a.c.getValue() + v.t = typeRawString + bLen := len(a.b) + a.b = escapeString(a.b, b2s(b)) + v.s = b2s(a.b[bLen+1 : len(a.b)-1]) + return v +} + +// NewNumberFloat64 returns new number value containing f. +// +// The returned number is valid until Reset is called on a. +func (a *Arena) NewNumberFloat64(f float64) *Value { + v := a.c.getValue() + v.t = TypeNumber + bLen := len(a.b) + a.b = strconv.AppendFloat(a.b, f, 'g', -1, 64) + v.s = b2s(a.b[bLen:]) + return v +} + +// NewNumberInt returns new number value containing n. +// +// The returned number is valid until Reset is called on a. +func (a *Arena) NewNumberInt(n int) *Value { + v := a.c.getValue() + v.t = TypeNumber + bLen := len(a.b) + a.b = strconv.AppendInt(a.b, int64(n), 10) + v.s = b2s(a.b[bLen:]) + return v +} + +// NewNumberString returns new number value containing s. +// +// The returned number is valid until Reset is called on a. +func (a *Arena) NewNumberString(s string) *Value { + v := a.c.getValue() + v.t = TypeNumber + v.s = s + return v +} + +// NewNull returns null value. +func (a *Arena) NewNull() *Value { + return valueNull +} + +// NewTrue returns true value. +func (a *Arena) NewTrue() *Value { + return valueTrue +} + +// NewFalse return false value. +func (a *Arena) NewFalse() *Value { + return valueFalse +} diff --git a/vendor/github.com/valyala/fastjson/doc.go b/vendor/github.com/valyala/fastjson/doc.go new file mode 100644 index 000000000..8076189cf --- /dev/null +++ b/vendor/github.com/valyala/fastjson/doc.go @@ -0,0 +1,9 @@ +/* +Package fastjson provides fast JSON parsing. + +Arbitrary JSON may be parsed by fastjson without the need for creating structs +or for generating go code. Just parse JSON and get the required fields with +Get* functions. + +*/ +package fastjson diff --git a/vendor/github.com/valyala/fastjson/go.mod b/vendor/github.com/valyala/fastjson/go.mod new file mode 100644 index 000000000..c0a990c77 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/go.mod @@ -0,0 +1 @@ +module github.com/valyala/fastjson diff --git a/vendor/github.com/valyala/fastjson/handy.go b/vendor/github.com/valyala/fastjson/handy.go new file mode 100644 index 000000000..e3380ce25 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/handy.go @@ -0,0 +1,170 @@ +package fastjson + +var handyPool ParserPool + +// GetString returns string value for the field identified by keys path +// in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// An empty string is returned on error. Use Parser for proper error handling. +// +// Parser is faster for obtaining multiple fields from JSON. +func GetString(data []byte, keys ...string) string { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return "" + } + sb := v.GetStringBytes(keys...) + str := string(sb) + handyPool.Put(p) + return str +} + +// GetBytes returns string value for the field identified by keys path +// in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// nil is returned on error. Use Parser for proper error handling. +// +// Parser is faster for obtaining multiple fields from JSON. +func GetBytes(data []byte, keys ...string) []byte { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return nil + } + sb := v.GetStringBytes(keys...) + + // Make a copy of sb, since sb belongs to p. + var b []byte + if sb != nil { + b = append(b, sb...) + } + + handyPool.Put(p) + return b +} + +// GetInt returns int value for the field identified by keys path +// in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned on error. Use Parser for proper error handling. +// +// Parser is faster for obtaining multiple fields from JSON. +func GetInt(data []byte, keys ...string) int { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return 0 + } + n := v.GetInt(keys...) + handyPool.Put(p) + return n +} + +// GetFloat64 returns float64 value for the field identified by keys path +// in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned on error. Use Parser for proper error handling. +// +// Parser is faster for obtaining multiple fields from JSON. +func GetFloat64(data []byte, keys ...string) float64 { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return 0 + } + f := v.GetFloat64(keys...) + handyPool.Put(p) + return f +} + +// GetBool returns boolean value for the field identified by keys path +// in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// False is returned on error. Use Parser for proper error handling. +// +// Parser is faster for obtaining multiple fields from JSON. +func GetBool(data []byte, keys ...string) bool { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return false + } + b := v.GetBool(keys...) + handyPool.Put(p) + return b +} + +// Exists returns true if the field identified by keys path exists in JSON data. +// +// Array indexes may be represented as decimal numbers in keys. +// +// False is returned on error. Use Parser for proper error handling. +// +// Parser is faster when multiple fields must be checked in the JSON. +func Exists(data []byte, keys ...string) bool { + p := handyPool.Get() + v, err := p.ParseBytes(data) + if err != nil { + handyPool.Put(p) + return false + } + ok := v.Exists(keys...) + handyPool.Put(p) + return ok +} + +// Parse parses json string s. +// +// The function is slower than the Parser.Parse for re-used Parser. +func Parse(s string) (*Value, error) { + var p Parser + return p.Parse(s) +} + +// MustParse parses json string s. +// +// The function panics if s cannot be parsed. +// The function is slower than the Parser.Parse for re-used Parser. +func MustParse(s string) *Value { + v, err := Parse(s) + if err != nil { + panic(err) + } + return v +} + +// ParseBytes parses b containing json. +// +// The function is slower than the Parser.ParseBytes for re-used Parser. +func ParseBytes(b []byte) (*Value, error) { + var p Parser + return p.ParseBytes(b) +} + +// MustParseBytes parses b containing json. +// +// The function banics if b cannot be parsed. +// The function is slower than the Parser.ParseBytes for re-used Parser. +func MustParseBytes(b []byte) *Value { + v, err := ParseBytes(b) + if err != nil { + panic(err) + } + return v +} diff --git a/vendor/github.com/valyala/fastjson/parser.go b/vendor/github.com/valyala/fastjson/parser.go new file mode 100644 index 000000000..2c1e9f994 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/parser.go @@ -0,0 +1,964 @@ +package fastjson + +import ( + "fmt" + "github.com/valyala/fastjson/fastfloat" + "strconv" + "strings" + "unicode/utf16" +) + +// Parser parses JSON. +// +// Parser may be re-used for subsequent parsing. +// +// Parser cannot be used from concurrent goroutines. +// Use per-goroutine parsers or ParserPool instead. +type Parser struct { + // b contains working copy of the string to be parsed. + b []byte + + // c is a cache for json values. + c cache +} + +// Parse parses s containing JSON. +// +// The returned value is valid until the next call to Parse*. +// +// Use Scanner if a stream of JSON values must be parsed. +func (p *Parser) Parse(s string) (*Value, error) { + s = skipWS(s) + p.b = append(p.b[:0], s...) + p.c.reset() + + v, tail, err := parseValue(b2s(p.b), &p.c) + if err != nil { + return nil, fmt.Errorf("cannot parse JSON: %s; unparsed tail: %q", err, startEndString(tail)) + } + tail = skipWS(tail) + if len(tail) > 0 { + return nil, fmt.Errorf("unexpected tail: %q", startEndString(tail)) + } + return v, nil +} + +// ParseBytes parses b containing JSON. +// +// The returned Value is valid until the next call to Parse*. +// +// Use Scanner if a stream of JSON values must be parsed. +func (p *Parser) ParseBytes(b []byte) (*Value, error) { + return p.Parse(b2s(b)) +} + +type cache struct { + vs []Value +} + +func (c *cache) reset() { + c.vs = c.vs[:0] +} + +func (c *cache) getValue() *Value { + if cap(c.vs) > len(c.vs) { + c.vs = c.vs[:len(c.vs)+1] + } else { + c.vs = append(c.vs, Value{}) + } + // Do not reset the value, since the caller must properly init it. + return &c.vs[len(c.vs)-1] +} + +func skipWS(s string) string { + if len(s) == 0 || s[0] > 0x20 { + // Fast path. + return s + } + return skipWSSlow(s) +} + +func skipWSSlow(s string) string { + if len(s) == 0 || s[0] != 0x20 && s[0] != 0x0A && s[0] != 0x09 && s[0] != 0x0D { + return s + } + for i := 1; i < len(s); i++ { + if s[i] != 0x20 && s[i] != 0x0A && s[i] != 0x09 && s[i] != 0x0D { + return s[i:] + } + } + return "" +} + +type kv struct { + k string + v *Value +} + +func parseValue(s string, c *cache) (*Value, string, error) { + if len(s) == 0 { + return nil, s, fmt.Errorf("cannot parse empty string") + } + + if s[0] == '{' { + v, tail, err := parseObject(s[1:], c) + if err != nil { + return nil, tail, fmt.Errorf("cannot parse object: %s", err) + } + return v, tail, nil + } + if s[0] == '[' { + v, tail, err := parseArray(s[1:], c) + if err != nil { + return nil, tail, fmt.Errorf("cannot parse array: %s", err) + } + return v, tail, nil + } + if s[0] == '"' { + ss, tail, err := parseRawString(s[1:]) + if err != nil { + return nil, tail, fmt.Errorf("cannot parse string: %s", err) + } + v := c.getValue() + v.t = typeRawString + v.s = ss + return v, tail, nil + } + if s[0] == 't' { + if len(s) < len("true") || s[:len("true")] != "true" { + return nil, s, fmt.Errorf("unexpected value found: %q", s) + } + return valueTrue, s[len("true"):], nil + } + if s[0] == 'f' { + if len(s) < len("false") || s[:len("false")] != "false" { + return nil, s, fmt.Errorf("unexpected value found: %q", s) + } + return valueFalse, s[len("false"):], nil + } + if s[0] == 'n' { + if len(s) < len("null") || s[:len("null")] != "null" { + return nil, s, fmt.Errorf("unexpected value found: %q", s) + } + return valueNull, s[len("null"):], nil + } + + ns, tail, err := parseRawNumber(s) + if err != nil { + return nil, tail, fmt.Errorf("cannot parse number: %s", err) + } + v := c.getValue() + v.t = TypeNumber + v.s = ns + return v, tail, nil +} + +func parseArray(s string, c *cache) (*Value, string, error) { + s = skipWS(s) + if len(s) == 0 { + return nil, s, fmt.Errorf("missing ']'") + } + + if s[0] == ']' { + v := c.getValue() + v.t = TypeArray + v.a = v.a[:0] + return v, s[1:], nil + } + + a := c.getValue() + a.t = TypeArray + a.a = a.a[:0] + for { + var v *Value + var err error + + s = skipWS(s) + v, s, err = parseValue(s, c) + if err != nil { + return nil, s, fmt.Errorf("cannot parse array value: %s", err) + } + a.a = append(a.a, v) + + s = skipWS(s) + if len(s) == 0 { + return nil, s, fmt.Errorf("unexpected end of array") + } + if s[0] == ',' { + s = s[1:] + continue + } + if s[0] == ']' { + s = s[1:] + return a, s, nil + } + return nil, s, fmt.Errorf("missing ',' after array value") + } +} + +func parseObject(s string, c *cache) (*Value, string, error) { + s = skipWS(s) + if len(s) == 0 { + return nil, s, fmt.Errorf("missing '}'") + } + + if s[0] == '}' { + v := c.getValue() + v.t = TypeObject + v.o.reset() + return v, s[1:], nil + } + + o := c.getValue() + o.t = TypeObject + o.o.reset() + for { + var err error + kv := o.o.getKV() + + // Parse key. + s = skipWS(s) + if len(s) == 0 || s[0] != '"' { + return nil, s, fmt.Errorf(`cannot find opening '"" for object key`) + } + kv.k, s, err = parseRawKey(s[1:]) + if err != nil { + return nil, s, fmt.Errorf("cannot parse object key: %s", err) + } + s = skipWS(s) + if len(s) == 0 || s[0] != ':' { + return nil, s, fmt.Errorf("missing ':' after object key") + } + s = s[1:] + + // Parse value + s = skipWS(s) + kv.v, s, err = parseValue(s, c) + if err != nil { + return nil, s, fmt.Errorf("cannot parse object value: %s", err) + } + s = skipWS(s) + if len(s) == 0 { + return nil, s, fmt.Errorf("unexpected end of object") + } + if s[0] == ',' { + s = s[1:] + continue + } + if s[0] == '}' { + return o, s[1:], nil + } + return nil, s, fmt.Errorf("missing ',' after object value") + } +} + +func escapeString(dst []byte, s string) []byte { + if !hasSpecialChars(s) { + // Fast path - nothing to escape. + dst = append(dst, '"') + dst = append(dst, s...) + dst = append(dst, '"') + return dst + } + + // Slow path. + return strconv.AppendQuote(dst, s) +} + +func hasSpecialChars(s string) bool { + if strings.IndexByte(s, '"') >= 0 || strings.IndexByte(s, '\\') >= 0 { + return true + } + for i := 0; i < len(s); i++ { + if s[i] < 0x20 { + return true + } + } + return false +} + +func unescapeStringBestEffort(s string) string { + n := strings.IndexByte(s, '\\') + if n < 0 { + // Fast path - nothing to unescape. + return s + } + + // Slow path - unescape string. + b := s2b(s) // It is safe to do, since s points to a byte slice in Parser.b. + b = b[:n] + s = s[n+1:] + for len(s) > 0 { + ch := s[0] + s = s[1:] + switch ch { + case '"': + b = append(b, '"') + case '\\': + b = append(b, '\\') + case '/': + b = append(b, '/') + case 'b': + b = append(b, '\b') + case 'f': + b = append(b, '\f') + case 'n': + b = append(b, '\n') + case 'r': + b = append(b, '\r') + case 't': + b = append(b, '\t') + case 'u': + if len(s) < 4 { + // Too short escape sequence. Just store it unchanged. + b = append(b, "\\u"...) + break + } + xs := s[:4] + x, err := strconv.ParseUint(xs, 16, 16) + if err != nil { + // Invalid escape sequence. Just store it unchanged. + b = append(b, "\\u"...) + break + } + s = s[4:] + if !utf16.IsSurrogate(rune(x)) { + b = append(b, string(rune(x))...) + break + } + + // Surrogate. + // See https://en.wikipedia.org/wiki/Universal_Character_Set_characters#Surrogates + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + b = append(b, "\\u"...) + b = append(b, xs...) + break + } + x1, err := strconv.ParseUint(s[2:6], 16, 16) + if err != nil { + b = append(b, "\\u"...) + b = append(b, xs...) + break + } + r := utf16.DecodeRune(rune(x), rune(x1)) + b = append(b, string(r)...) + s = s[6:] + default: + // Unknown escape sequence. Just store it unchanged. + b = append(b, '\\', ch) + } + n = strings.IndexByte(s, '\\') + if n < 0 { + b = append(b, s...) + break + } + b = append(b, s[:n]...) + s = s[n+1:] + } + return b2s(b) +} + +// parseRawKey is similar to parseRawString, but is optimized +// for small-sized keys without escape sequences. +func parseRawKey(s string) (string, string, error) { + for i := 0; i < len(s); i++ { + if s[i] == '"' { + // Fast path. + return s[:i], s[i+1:], nil + } + if s[i] == '\\' { + // Slow path. + return parseRawString(s) + } + } + return s, "", fmt.Errorf(`missing closing '"'`) +} + +func parseRawString(s string) (string, string, error) { + n := strings.IndexByte(s, '"') + if n < 0 { + return s, "", fmt.Errorf(`missing closing '"'`) + } + if n == 0 || s[n-1] != '\\' { + // Fast path. No escaped ". + return s[:n], s[n+1:], nil + } + + // Slow path - possible escaped " found. + ss := s + for { + i := n - 1 + for i > 0 && s[i-1] == '\\' { + i-- + } + if uint(n-i)%2 == 0 { + return ss[:len(ss)-len(s)+n], s[n+1:], nil + } + s = s[n+1:] + + n = strings.IndexByte(s, '"') + if n < 0 { + return ss, "", fmt.Errorf(`missing closing '"'`) + } + if n == 0 || s[n-1] != '\\' { + return ss[:len(ss)-len(s)+n], s[n+1:], nil + } + } +} + +func parseRawNumber(s string) (string, string, error) { + // The caller must ensure len(s) > 0 + + // Find the end of the number. + for i := 0; i < len(s); i++ { + ch := s[i] + if (ch >= '0' && ch <= '9') || ch == '.' || ch == '-' || ch == 'e' || ch == 'E' || ch == '+' { + continue + } + if i == 0 { + return "", s, fmt.Errorf("unexpected char: %q", s[:1]) + } + ns := s[:i] + s = s[i:] + return ns, s, nil + } + return s, "", nil +} + +// Object represents JSON object. +// +// Object cannot be used from concurrent goroutines. +// Use per-goroutine parsers or ParserPool instead. +type Object struct { + kvs []kv + keysUnescaped bool +} + +func (o *Object) reset() { + o.kvs = o.kvs[:0] + o.keysUnescaped = false +} + +// MarshalTo appends marshaled o to dst and returns the result. +func (o *Object) MarshalTo(dst []byte) []byte { + dst = append(dst, '{') + for i, kv := range o.kvs { + if o.keysUnescaped { + dst = escapeString(dst, kv.k) + } else { + dst = append(dst, '"') + dst = append(dst, kv.k...) + dst = append(dst, '"') + } + dst = append(dst, ':') + dst = kv.v.MarshalTo(dst) + if i != len(o.kvs)-1 { + dst = append(dst, ',') + } + } + dst = append(dst, '}') + return dst +} + +// String returns string representation for the o. +// +// This function is for debugging purposes only. It isn't optimized for speed. +// See MarshalTo instead. +func (o *Object) String() string { + b := o.MarshalTo(nil) + // It is safe converting b to string without allocation, since b is no longer + // reachable after this line. + return b2s(b) +} + +func (o *Object) getKV() *kv { + if cap(o.kvs) > len(o.kvs) { + o.kvs = o.kvs[:len(o.kvs)+1] + } else { + o.kvs = append(o.kvs, kv{}) + } + return &o.kvs[len(o.kvs)-1] +} + +func (o *Object) unescapeKeys() { + if o.keysUnescaped { + return + } + for i := range o.kvs { + kv := &o.kvs[i] + kv.k = unescapeStringBestEffort(kv.k) + } + o.keysUnescaped = true +} + +// Len returns the number of items in the o. +func (o *Object) Len() int { + return len(o.kvs) +} + +// Get returns the value for the given key in the o. +// +// Returns nil if the value for the given key isn't found. +// +// The returned value is valid until Parse is called on the Parser returned o. +func (o *Object) Get(key string) *Value { + if !o.keysUnescaped && strings.IndexByte(key, '\\') < 0 { + // Fast path - try searching for the key without object keys unescaping. + for _, kv := range o.kvs { + if kv.k == key { + return kv.v + } + } + } + + // Slow path - unescape object keys. + o.unescapeKeys() + + for _, kv := range o.kvs { + if kv.k == key { + return kv.v + } + } + return nil +} + +// Visit calls f for each item in the o in the original order +// of the parsed JSON. +// +// f cannot hold key and/or v after returning. +func (o *Object) Visit(f func(key []byte, v *Value)) { + if o == nil { + return + } + + o.unescapeKeys() + + for _, kv := range o.kvs { + f(s2b(kv.k), kv.v) + } +} + +// Value represents any JSON value. +// +// Call Type in order to determine the actual type of the JSON value. +// +// Value cannot be used from concurrent goroutines. +// Use per-goroutine parsers or ParserPool instead. +type Value struct { + o Object + a []*Value + s string + t Type +} + +// MarshalTo appends marshaled v to dst and returns the result. +func (v *Value) MarshalTo(dst []byte) []byte { + switch v.t { + case typeRawString: + dst = append(dst, '"') + dst = append(dst, v.s...) + dst = append(dst, '"') + return dst + case TypeObject: + return v.o.MarshalTo(dst) + case TypeArray: + dst = append(dst, '[') + for i, vv := range v.a { + dst = vv.MarshalTo(dst) + if i != len(v.a)-1 { + dst = append(dst, ',') + } + } + dst = append(dst, ']') + return dst + case TypeString: + return escapeString(dst, v.s) + case TypeNumber: + return append(dst, v.s...) + case TypeTrue: + return append(dst, "true"...) + case TypeFalse: + return append(dst, "false"...) + case TypeNull: + return append(dst, "null"...) + default: + panic(fmt.Errorf("BUG: unexpected Value type: %d", v.t)) + } +} + +// String returns string representation of the v. +// +// The function is for debugging purposes only. It isn't optimized for speed. +// See MarshalTo instead. +// +// Don't confuse this function with StringBytes, which must be called +// for obtaining the underlying JSON string for the v. +func (v *Value) String() string { + b := v.MarshalTo(nil) + // It is safe converting b to string without allocation, since b is no longer + // reachable after this line. + return b2s(b) +} + +// Type represents JSON type. +type Type int + +const ( + // TypeNull is JSON null. + TypeNull Type = 0 + + // TypeObject is JSON object type. + TypeObject Type = 1 + + // TypeArray is JSON array type. + TypeArray Type = 2 + + // TypeString is JSON string type. + TypeString Type = 3 + + // TypeNumber is JSON number type. + TypeNumber Type = 4 + + // TypeTrue is JSON true. + TypeTrue Type = 5 + + // TypeFalse is JSON false. + TypeFalse Type = 6 + + typeRawString Type = 7 +) + +// String returns string representation of t. +func (t Type) String() string { + switch t { + case TypeObject: + return "object" + case TypeArray: + return "array" + case TypeString: + return "string" + case TypeNumber: + return "number" + case TypeTrue: + return "true" + case TypeFalse: + return "false" + case TypeNull: + return "null" + + // typeRawString is skipped intentionally, + // since it shouldn't be visible to user. + default: + panic(fmt.Errorf("BUG: unknown Value type: %d", t)) + } +} + +// Type returns the type of the v. +func (v *Value) Type() Type { + if v.t == typeRawString { + v.s = unescapeStringBestEffort(v.s) + v.t = TypeString + } + return v.t +} + +// Exists returns true if the field exists for the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +func (v *Value) Exists(keys ...string) bool { + v = v.Get(keys...) + return v != nil +} + +// Get returns value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// nil is returned for non-existing keys path. +// +// The returned value is valid until Parse is called on the Parser returned v. +func (v *Value) Get(keys ...string) *Value { + if v == nil { + return nil + } + for _, key := range keys { + if v.t == TypeObject { + v = v.o.Get(key) + if v == nil { + return nil + } + } else if v.t == TypeArray { + n, err := strconv.Atoi(key) + if err != nil || n < 0 || n >= len(v.a) { + return nil + } + v = v.a[n] + } else { + return nil + } + } + return v +} + +// GetObject returns object value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// nil is returned for non-existing keys path or for invalid value type. +// +// The returned object is valid until Parse is called on the Parser returned v. +func (v *Value) GetObject(keys ...string) *Object { + v = v.Get(keys...) + if v == nil || v.t != TypeObject { + return nil + } + return &v.o +} + +// GetArray returns array value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// nil is returned for non-existing keys path or for invalid value type. +// +// The returned array is valid until Parse is called on the Parser returned v. +func (v *Value) GetArray(keys ...string) []*Value { + v = v.Get(keys...) + if v == nil || v.t != TypeArray { + return nil + } + return v.a +} + +// GetFloat64 returns float64 value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned for non-existing keys path or for invalid value type. +func (v *Value) GetFloat64(keys ...string) float64 { + v = v.Get(keys...) + if v == nil || v.Type() != TypeNumber { + return 0 + } + return fastfloat.ParseBestEffort(v.s) +} + +// GetInt returns int value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned for non-existing keys path or for invalid value type. +func (v *Value) GetInt(keys ...string) int { + v = v.Get(keys...) + if v == nil || v.Type() != TypeNumber { + return 0 + } + n := fastfloat.ParseInt64BestEffort(v.s) + nn := int(n) + if int64(nn) != n { + return 0 + } + return nn +} + +// GetUint returns uint value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned for non-existing keys path or for invalid value type. +func (v *Value) GetUint(keys ...string) uint { + v = v.Get(keys...) + if v == nil || v.Type() != TypeNumber { + return 0 + } + n := fastfloat.ParseUint64BestEffort(v.s) + nn := uint(n) + if uint64(nn) != n { + return 0 + } + return nn +} + +// GetInt64 returns int64 value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned for non-existing keys path or for invalid value type. +func (v *Value) GetInt64(keys ...string) int64 { + v = v.Get(keys...) + if v == nil || v.Type() != TypeNumber { + return 0 + } + return fastfloat.ParseInt64BestEffort(v.s) +} + +// GetUint64 returns uint64 value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// 0 is returned for non-existing keys path or for invalid value type. +func (v *Value) GetUint64(keys ...string) uint64 { + v = v.Get(keys...) + if v == nil || v.Type() != TypeNumber { + return 0 + } + return fastfloat.ParseUint64BestEffort(v.s) +} + +// GetStringBytes returns string value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// nil is returned for non-existing keys path or for invalid value type. +// +// The returned string is valid until Parse is called on the Parser returned v. +func (v *Value) GetStringBytes(keys ...string) []byte { + v = v.Get(keys...) + if v == nil || v.Type() != TypeString { + return nil + } + return s2b(v.s) +} + +// GetBool returns bool value by the given keys path. +// +// Array indexes may be represented as decimal numbers in keys. +// +// false is returned for non-existing keys path or for invalid value type. +func (v *Value) GetBool(keys ...string) bool { + v = v.Get(keys...) + if v != nil && v.t == TypeTrue { + return true + } + return false +} + +// Object returns the underlying JSON object for the v. +// +// The returned object is valid until Parse is called on the Parser returned v. +// +// Use GetObject if you don't need error handling. +func (v *Value) Object() (*Object, error) { + if v.t != TypeObject { + return nil, fmt.Errorf("value doesn't contain object; it contains %s", v.Type()) + } + return &v.o, nil +} + +// Array returns the underlying JSON array for the v. +// +// The returned array is valid until Parse is called on the Parser returned v. +// +// Use GetArray if you don't need error handling. +func (v *Value) Array() ([]*Value, error) { + if v.t != TypeArray { + return nil, fmt.Errorf("value doesn't contain array; it contains %s", v.Type()) + } + return v.a, nil +} + +// StringBytes returns the underlying JSON string for the v. +// +// The returned string is valid until Parse is called on the Parser returned v. +// +// Use GetStringBytes if you don't need error handling. +func (v *Value) StringBytes() ([]byte, error) { + if v.Type() != TypeString { + return nil, fmt.Errorf("value doesn't contain string; it contains %s", v.Type()) + } + return s2b(v.s), nil +} + +// Float64 returns the underlying JSON number for the v. +// +// Use GetFloat64 if you don't need error handling. +func (v *Value) Float64() (float64, error) { + if v.Type() != TypeNumber { + return 0, fmt.Errorf("value doesn't contain number; it contains %s", v.Type()) + } + f := fastfloat.ParseBestEffort(v.s) + return f, nil +} + +// Int returns the underlying JSON int for the v. +// +// Use GetInt if you don't need error handling. +func (v *Value) Int() (int, error) { + if v.Type() != TypeNumber { + return 0, fmt.Errorf("value doesn't contain number; it contains %s", v.Type()) + } + n := fastfloat.ParseInt64BestEffort(v.s) + if n == 0 && v.s != "0" { + return 0, fmt.Errorf("cannot parse int %q", v.s) + } + nn := int(n) + if int64(nn) != n { + return 0, fmt.Errorf("number %q doesn't fit int", v.s) + } + return nn, nil +} + +// Uint returns the underlying JSON uint for the v. +// +// Use GetInt if you don't need error handling. +func (v *Value) Uint() (uint, error) { + if v.Type() != TypeNumber { + return 0, fmt.Errorf("value doesn't contain number; it contains %s", v.Type()) + } + n := fastfloat.ParseUint64BestEffort(v.s) + if n == 0 && v.s != "0" { + return 0, fmt.Errorf("cannot parse uint %q", v.s) + } + nn := uint(n) + if uint64(nn) != n { + return 0, fmt.Errorf("number %q doesn't fit uint", v.s) + } + return nn, nil +} + +// Int64 returns the underlying JSON int64 for the v. +// +// Use GetInt64 if you don't need error handling. +func (v *Value) Int64() (int64, error) { + if v.Type() != TypeNumber { + return 0, fmt.Errorf("value doesn't contain number; it contains %s", v.Type()) + } + n := fastfloat.ParseInt64BestEffort(v.s) + if n == 0 && v.s != "0" { + return 0, fmt.Errorf("cannot parse int64 %q", v.s) + } + return n, nil +} + +// Uint64 returns the underlying JSON uint64 for the v. +// +// Use GetInt64 if you don't need error handling. +func (v *Value) Uint64() (uint64, error) { + if v.Type() != TypeNumber { + return 0, fmt.Errorf("value doesn't contain number; it contains %s", v.Type()) + } + n := fastfloat.ParseUint64BestEffort(v.s) + if n == 0 && v.s != "0" { + return 0, fmt.Errorf("cannot parse uint64 %q", v.s) + } + return n, nil +} + +// Bool returns the underlying JSON bool for the v. +// +// Use GetBool if you don't need error handling. +func (v *Value) Bool() (bool, error) { + if v.t == TypeTrue { + return true, nil + } + if v.t == TypeFalse { + return false, nil + } + return false, fmt.Errorf("value doesn't contain bool; it contains %s", v.Type()) +} + +var ( + valueTrue = &Value{t: TypeTrue} + valueFalse = &Value{t: TypeFalse} + valueNull = &Value{t: TypeNull} +) diff --git a/vendor/github.com/valyala/fastjson/pool.go b/vendor/github.com/valyala/fastjson/pool.go new file mode 100644 index 000000000..00cfb42fa --- /dev/null +++ b/vendor/github.com/valyala/fastjson/pool.go @@ -0,0 +1,52 @@ +package fastjson + +import ( + "sync" +) + +// ParserPool may be used for pooling Parsers for similarly typed JSONs. +type ParserPool struct { + pool sync.Pool +} + +// Get returns a Parser from pp. +// +// The Parser must be Put to pp after use. +func (pp *ParserPool) Get() *Parser { + v := pp.pool.Get() + if v == nil { + return &Parser{} + } + return v.(*Parser) +} + +// Put returns p to pp. +// +// p and objects recursively returned from p cannot be used after p +// is put into pp. +func (pp *ParserPool) Put(p *Parser) { + pp.pool.Put(p) +} + +// ArenaPool may be used for pooling Arenas for similarly typed JSONs. +type ArenaPool struct { + pool sync.Pool +} + +// Get returns an Arena from ap. +// +// The Arena must be Put to ap after use. +func (ap *ArenaPool) Get() *Arena { + v := ap.pool.Get() + if v == nil { + return &Arena{} + } + return v.(*Arena) +} + +// Put returns a to ap. +// +// a and objects created by a cannot be used after a is put into ap. +func (ap *ArenaPool) Put(a *Arena) { + ap.pool.Put(a) +} diff --git a/vendor/github.com/valyala/fastjson/scanner.go b/vendor/github.com/valyala/fastjson/scanner.go new file mode 100644 index 000000000..b9ed24264 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/scanner.go @@ -0,0 +1,94 @@ +package fastjson + +import ( + "errors" +) + +// Scanner scans a series of JSON values. Values may be delimited by whitespace. +// +// Scanner may parse JSON lines ( http://jsonlines.org/ ). +// +// Scanner may be re-used for subsequent parsing. +// +// Scanner cannot be used from concurrent goroutines. +// +// Use Parser for parsing only a single JSON value. +type Scanner struct { + // b contains a working copy of json value passed to Init. + b []byte + + // s points to the next JSON value to parse. + s string + + // err contains the last error. + err error + + // v contains the last parsed JSON value. + v *Value + + // c is used for caching JSON values. + c cache +} + +// Init initializes sc with the given s. +// +// s may contain multiple JSON values, which may be delimited by whitespace. +func (sc *Scanner) Init(s string) { + sc.b = append(sc.b[:0], s...) + sc.s = b2s(sc.b) + sc.err = nil + sc.v = nil +} + +// InitBytes initializes sc with the given b. +// +// b may contain multiple JSON values, which may be delimited by whitespace. +func (sc *Scanner) InitBytes(b []byte) { + sc.Init(b2s(b)) +} + +// Next parses the next JSON value from s passed to Init. +// +// Returns true on success. The parsed value is available via Value call. +// +// Returns false either on error or on the end of s. +// Call Error in order to determine the cause of the returned false. +func (sc *Scanner) Next() bool { + if sc.err != nil { + return false + } + + sc.s = skipWS(sc.s) + if len(sc.s) == 0 { + sc.err = errEOF + return false + } + + sc.c.reset() + v, tail, err := parseValue(sc.s, &sc.c) + if err != nil { + sc.err = err + return false + } + + sc.s = tail + sc.v = v + return true +} + +// Error returns the last error. +func (sc *Scanner) Error() error { + if sc.err == errEOF { + return nil + } + return sc.err +} + +// Value returns the last parsed value. +// +// The value is valid until the Next call. +func (sc *Scanner) Value() *Value { + return sc.v +} + +var errEOF = errors.New("end of s") diff --git a/vendor/github.com/valyala/fastjson/update.go b/vendor/github.com/valyala/fastjson/update.go new file mode 100644 index 000000000..f8099bdbb --- /dev/null +++ b/vendor/github.com/valyala/fastjson/update.go @@ -0,0 +1,110 @@ +package fastjson + +import ( + "strconv" + "strings" +) + +// Del deletes the entry with the given key from o. +func (o *Object) Del(key string) { + if o == nil { + return + } + if !o.keysUnescaped && strings.IndexByte(key, '\\') < 0 { + // Fast path - try searching for the key without object keys unescaping. + for i, kv := range o.kvs { + if kv.k == key { + o.kvs = append(o.kvs[:i], o.kvs[i+1:]...) + return + } + } + } + + // Slow path - unescape object keys before item search. + o.unescapeKeys() + + for i, kv := range o.kvs { + if kv.k == key { + o.kvs = append(o.kvs[:i], o.kvs[i+1:]...) + return + } + } +} + +// Del deletes the entry with the given key from array or object v. +func (v *Value) Del(key string) { + if v == nil { + return + } + if v.t == TypeObject { + v.o.Del(key) + return + } + if v.t == TypeArray { + n, err := strconv.Atoi(key) + if err != nil || n < 0 || n >= len(v.a) { + return + } + v.a = append(v.a[:n], v.a[n+1:]...) + } +} + +// Set sets (key, value) entry in the o. +// +// The value must be unchanged during o lifetime. +func (o *Object) Set(key string, value *Value) { + if o == nil { + return + } + if value == nil { + value = valueNull + } + o.unescapeKeys() + + // Try substituting already existing entry with the given key. + for i := range o.kvs { + kv := &o.kvs[i] + if kv.k == key { + kv.v = value + return + } + } + + // Add new entry. + kv := o.getKV() + kv.k = key + kv.v = value +} + +// Set sets (key, value) entry in the array or object v. +// +// The value must be unchanged during v lifetime. +func (v *Value) Set(key string, value *Value) { + if v == nil { + return + } + if v.t == TypeObject { + v.o.Set(key, value) + return + } + if v.t == TypeArray { + idx, err := strconv.Atoi(key) + if err != nil || idx < 0 { + return + } + v.SetArrayItem(idx, value) + } +} + +// SetArrayItem sets the value in the array v at idx position. +// +// The value must be unchanged during v lifetime. +func (v *Value) SetArrayItem(idx int, value *Value) { + if v == nil || v.t != TypeArray { + return + } + for idx >= len(v.a) { + v.a = append(v.a, valueNull) + } + v.a[idx] = value +} diff --git a/vendor/github.com/valyala/fastjson/util.go b/vendor/github.com/valyala/fastjson/util.go new file mode 100644 index 000000000..4e4a18266 --- /dev/null +++ b/vendor/github.com/valyala/fastjson/util.go @@ -0,0 +1,30 @@ +package fastjson + +import ( + "reflect" + "unsafe" +) + +func b2s(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func s2b(s string) []byte { + strh := (*reflect.StringHeader)(unsafe.Pointer(&s)) + var sh reflect.SliceHeader + sh.Data = strh.Data + sh.Len = strh.Len + sh.Cap = strh.Len + return *(*[]byte)(unsafe.Pointer(&sh)) +} + +const maxStartEndStringLen = 80 + +func startEndString(s string) string { + if len(s) <= maxStartEndStringLen { + return s + } + start := s[:40] + end := s[len(s)-40:] + return start + "..." + end +} diff --git a/vendor/github.com/valyala/fastjson/validate.go b/vendor/github.com/valyala/fastjson/validate.go new file mode 100644 index 000000000..196f1c3dc --- /dev/null +++ b/vendor/github.com/valyala/fastjson/validate.go @@ -0,0 +1,308 @@ +package fastjson + +import ( + "fmt" + "strconv" + "strings" +) + +// Validate validates JSON s. +func Validate(s string) error { + s = skipWS(s) + + tail, err := validateValue(s) + if err != nil { + return fmt.Errorf("cannot parse JSON: %s; unparsed tail: %q", err, startEndString(tail)) + } + tail = skipWS(tail) + if len(tail) > 0 { + return fmt.Errorf("unexpected tail: %q", startEndString(tail)) + } + return nil +} + +// ValidateBytes validates JSON b. +func ValidateBytes(b []byte) error { + return Validate(b2s(b)) +} + +func validateValue(s string) (string, error) { + if len(s) == 0 { + return s, fmt.Errorf("cannot parse empty string") + } + + if s[0] == '{' { + tail, err := validateObject(s[1:]) + if err != nil { + return tail, fmt.Errorf("cannot parse object: %s", err) + } + return tail, nil + } + if s[0] == '[' { + tail, err := validateArray(s[1:]) + if err != nil { + return tail, fmt.Errorf("cannot parse array: %s", err) + } + return tail, nil + } + if s[0] == '"' { + sv, tail, err := validateString(s[1:]) + if err != nil { + return tail, fmt.Errorf("cannot parse string: %s", err) + } + // Scan the string for control chars. + for i := 0; i < len(sv); i++ { + if sv[i] < 0x20 { + return tail, fmt.Errorf("string cannot contain control char 0x%02X", sv[i]) + } + } + return tail, nil + } + if s[0] == 't' { + if len(s) < len("true") || s[:len("true")] != "true" { + return s, fmt.Errorf("unexpected value found: %q", s) + } + return s[len("true"):], nil + } + if s[0] == 'f' { + if len(s) < len("false") || s[:len("false")] != "false" { + return s, fmt.Errorf("unexpected value found: %q", s) + } + return s[len("false"):], nil + } + if s[0] == 'n' { + if len(s) < len("null") || s[:len("null")] != "null" { + return s, fmt.Errorf("unexpected value found: %q", s) + } + return s[len("null"):], nil + } + + tail, err := validateNumber(s) + if err != nil { + return tail, fmt.Errorf("cannot parse number: %s", err) + } + return tail, nil +} + +func validateArray(s string) (string, error) { + s = skipWS(s) + if len(s) == 0 { + return s, fmt.Errorf("missing ']'") + } + if s[0] == ']' { + return s[1:], nil + } + + for { + var err error + + s = skipWS(s) + s, err = validateValue(s) + if err != nil { + return s, fmt.Errorf("cannot parse array value: %s", err) + } + + s = skipWS(s) + if len(s) == 0 { + return s, fmt.Errorf("unexpected end of array") + } + if s[0] == ',' { + s = s[1:] + continue + } + if s[0] == ']' { + s = s[1:] + return s, nil + } + return s, fmt.Errorf("missing ',' after array value") + } +} + +func validateObject(s string) (string, error) { + s = skipWS(s) + if len(s) == 0 { + return s, fmt.Errorf("missing '}'") + } + if s[0] == '}' { + return s[1:], nil + } + + for { + var err error + + // Parse key. + s = skipWS(s) + if len(s) == 0 || s[0] != '"' { + return s, fmt.Errorf(`cannot find opening '"" for object key`) + } + + var key string + key, s, err = validateKey(s[1:]) + if err != nil { + return s, fmt.Errorf("cannot parse object key: %s", err) + } + // Scan the key for control chars. + for i := 0; i < len(key); i++ { + if key[i] < 0x20 { + return s, fmt.Errorf("object key cannot contain control char 0x%02X", key[i]) + } + } + s = skipWS(s) + if len(s) == 0 || s[0] != ':' { + return s, fmt.Errorf("missing ':' after object key") + } + s = s[1:] + + // Parse value + s = skipWS(s) + s, err = validateValue(s) + if err != nil { + return s, fmt.Errorf("cannot parse object value: %s", err) + } + s = skipWS(s) + if len(s) == 0 { + return s, fmt.Errorf("unexpected end of object") + } + if s[0] == ',' { + s = s[1:] + continue + } + if s[0] == '}' { + return s[1:], nil + } + return s, fmt.Errorf("missing ',' after object value") + } +} + +// validateKey is similar to validateString, but is optimized +// for typical object keys, which are quite small and have no escape sequences. +func validateKey(s string) (string, string, error) { + for i := 0; i < len(s); i++ { + if s[i] == '"' { + // Fast path - the key doesn't contain escape sequences. + return s[:i], s[i+1:], nil + } + if s[i] == '\\' { + // Slow path - the key contains escape sequences. + return validateString(s) + } + } + return "", s, fmt.Errorf(`missing closing '"'`) +} + +func validateString(s string) (string, string, error) { + // Try fast path - a string without escape sequences. + if n := strings.IndexByte(s, '"'); n >= 0 && strings.IndexByte(s[:n], '\\') < 0 { + return s[:n], s[n+1:], nil + } + + // Slow path - escape sequences are present. + rs, tail, err := parseRawString(s) + if err != nil { + return rs, tail, err + } + for { + n := strings.IndexByte(rs, '\\') + if n < 0 { + return rs, tail, nil + } + n++ + if n >= len(rs) { + return rs, tail, fmt.Errorf("BUG: parseRawString returned invalid string with trailing backslash: %q", rs) + } + ch := rs[n] + rs = rs[n+1:] + switch ch { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + // Valid escape sequences - see http://json.org/ + break + case 'u': + if len(rs) < 4 { + return rs, tail, fmt.Errorf(`too short escape sequence: \u%s`, rs) + } + xs := rs[:4] + _, err := strconv.ParseUint(xs, 16, 16) + if err != nil { + return rs, tail, fmt.Errorf(`invalid escape sequence \u%s: %s`, xs, err) + } + rs = rs[4:] + default: + return rs, tail, fmt.Errorf(`unknown escape sequence \%c`, ch) + } + } +} + +func validateNumber(s string) (string, error) { + if len(s) == 0 { + return s, fmt.Errorf("zero-length number") + } + if s[0] == '-' { + s = s[1:] + if len(s) == 0 { + return s, fmt.Errorf("missing number after minus") + } + } + i := 0 + for i < len(s) { + if s[i] < '0' || s[i] > '9' { + break + } + i++ + } + if i <= 0 { + return s, fmt.Errorf("expecting 0..9 digit, got %c", s[0]) + } + if s[0] == '0' && i != 1 { + return s, fmt.Errorf("unexpected number starting from 0") + } + if i >= len(s) { + return "", nil + } + if s[i] == '.' { + // Validate fractional part + s = s[i+1:] + if len(s) == 0 { + return s, fmt.Errorf("missing fractional part") + } + i = 0 + for i < len(s) { + if s[i] < '0' || s[i] > '9' { + break + } + i++ + } + if i == 0 { + return s, fmt.Errorf("expecting 0..9 digit in fractional part, got %c", s[0]) + } + if i >= len(s) { + return "", nil + } + } + if s[i] == 'e' || s[i] == 'E' { + // Validate exponent part + s = s[i+1:] + if len(s) == 0 { + return s, fmt.Errorf("missing exponent part") + } + if s[0] == '-' || s[0] == '+' { + s = s[1:] + if len(s) == 0 { + return s, fmt.Errorf("missing exponent part") + } + } + i = 0 + for i < len(s) { + if s[i] < '0' || s[i] > '9' { + break + } + i++ + } + if i == 0 { + return s, fmt.Errorf("expecting 0..9 digit in exponent part, got %c", s[0]) + } + if i >= len(s) { + return "", nil + } + } + return s[i:], nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index cc904d848..cfcc7a2bb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -15,6 +15,7 @@ github.com/klauspost/compress/zstd/internal/xxhash # github.com/valyala/bytebufferpool v1.0.0 github.com/valyala/bytebufferpool # github.com/valyala/fastjson v1.4.1 +github.com/valyala/fastjson github.com/valyala/fastjson/fastfloat # github.com/valyala/fastrand v1.0.0 github.com/valyala/fastrand