diff --git a/NOTICE.txt b/NOTICE.txt index b24ef53e05c5..29ab87e7d375 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4662,7 +4662,7 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/mailru/easyjson -Revision: d5b7844b561a +Version: v0.7.1 License type (autodetected): MIT ./vendor/github.com/mailru/easyjson/LICENSE: -------------------------------------------------------------------- diff --git a/go.mod b/go.mod index f1bab9b5122d..f25f89db9c69 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 github.com/magefile/mage v1.9.0 + github.com/mailru/easyjson v0.7.1 // indirect github.com/mattn/go-colorable v0.0.8 github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe // indirect github.com/mattn/go-isatty v0.0.2 // indirect diff --git a/go.sum b/go.sum index fc67c6af39f5..0b026a7eff90 100644 --- a/go.sum +++ b/go.sum @@ -448,6 +448,8 @@ github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a h1:TpvdAwDAt1K4ANVOfcihouRdvP+MgAfDWwBuct4l6ZY= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11 h1:YFh+sjyJTMQSYjKwM4dFKhJPJC/wfo98tPUc17HdoYw= github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI= github.com/mattn/go-colorable v0.0.8 h1:KatiXbcoFpoKmM5pL0yhug+tx/POfZO+0aVsuGhUhgo= diff --git a/vendor/github.com/mailru/easyjson/.gitignore b/vendor/github.com/mailru/easyjson/.gitignore index db8c66edf8b6..fbfaf7a3f444 100644 --- a/vendor/github.com/mailru/easyjson/.gitignore +++ b/vendor/github.com/mailru/easyjson/.gitignore @@ -2,3 +2,5 @@ *_easyjson.go *.iml .idea +*.swp +bin/* diff --git a/vendor/github.com/mailru/easyjson/.travis.yml b/vendor/github.com/mailru/easyjson/.travis.yml index 3e5ac1320baf..0ececa0718fc 100644 --- a/vendor/github.com/mailru/easyjson/.travis.yml +++ b/vendor/github.com/mailru/easyjson/.travis.yml @@ -1,8 +1,12 @@ language: go go: - - tip + - tip + - stable + +matrix: + allow_failures: + - go: tip + install: - - go get github.com/ugorji/go/codec - - go get github.com/pquerna/ffjson/fflib/v1 - - go get github.com/golang/lint/golint + - go get golang.org/x/lint/golint diff --git a/vendor/github.com/mailru/easyjson/Makefile b/vendor/github.com/mailru/easyjson/Makefile index 5a951047db4f..80449f0d2749 100644 --- a/vendor/github.com/mailru/easyjson/Makefile +++ b/vendor/github.com/mailru/easyjson/Makefile @@ -1,50 +1,56 @@ -PKG=github.com/mailru/easyjson -GOPATH:=$(PWD)/.root:$(GOPATH) -export GOPATH - all: test -.root/src/$(PKG): - mkdir -p $@ - for i in $$PWD/* ; do ln -s $$i $@/`basename $$i` ; done - -root: .root/src/$(PKG) - clean: - rm -rf .root + rm -rf bin + rm -rf tests/*_easyjson.go + rm -rf benchmark/*_easyjson.go build: - go build -i -o .root/bin/easyjson $(PKG)/easyjson - -generate: root build - .root/bin/easyjson -stubs \ - .root/src/$(PKG)/tests/snake.go \ - .root/src/$(PKG)/tests/data.go \ - .root/src/$(PKG)/tests/omitempty.go \ - .root/src/$(PKG)/tests/nothing.go - - .root/bin/easyjson -all .root/src/$(PKG)/tests/data.go - .root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go - .root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go - .root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go - .root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go - -test: generate root + go build -i -o ./bin/easyjson ./easyjson + +generate: build + bin/easyjson -stubs \ + ./tests/snake.go \ + ./tests/data.go \ + ./tests/omitempty.go \ + ./tests/nothing.go \ + ./tests/named_type.go \ + ./tests/custom_map_key_type.go \ + ./tests/embedded_type.go \ + ./tests/reference_to_pointer.go \ + ./tests/html.go \ + ./tests/unknown_fields.go \ + + bin/easyjson -all ./tests/data.go + bin/easyjson -all ./tests/nothing.go + bin/easyjson -all ./tests/errors.go + bin/easyjson -all ./tests/html.go + bin/easyjson -snake_case ./tests/snake.go + bin/easyjson -omit_empty ./tests/omitempty.go + bin/easyjson -build_tags=use_easyjson ./benchmark/data.go + bin/easyjson ./tests/nested_easy.go + bin/easyjson ./tests/named_type.go + bin/easyjson ./tests/custom_map_key_type.go + bin/easyjson ./tests/embedded_type.go + bin/easyjson ./tests/reference_to_pointer.go + bin/easyjson ./tests/key_marshaler_map.go + bin/easyjson -disallow_unknown_fields ./tests/disallow_unknown.go + bin/easyjson ./tests/unknown_fields.go + +test: generate go test \ - $(PKG)/tests \ - $(PKG)/jlexer \ - $(PKG)/gen \ - $(PKG)/buffer - go test -benchmem -tags use_easyjson -bench . $(PKG)/benchmark - golint -set_exit_status .root/src/$(PKG)/tests/*_easyjson.go - -bench-other: generate root - @go test -benchmem -bench . $(PKG)/benchmark - @go test -benchmem -tags use_ffjson -bench . $(PKG)/benchmark - @go test -benchmem -tags use_codec -bench . $(PKG)/benchmark + ./tests \ + ./jlexer \ + ./gen \ + ./buffer + cd benchmark && go test -benchmem -tags use_easyjson -bench . + golint -set_exit_status ./tests/*_easyjson.go + +bench-other: generate + cd benchmark && make bench-python: benchmark/ujson.sh -.PHONY: root clean generate test build +.PHONY: clean generate test build diff --git a/vendor/github.com/mailru/easyjson/README.md b/vendor/github.com/mailru/easyjson/README.md index e990111fc2c9..3bdcf2d06c23 100644 --- a/vendor/github.com/mailru/easyjson/README.md +++ b/vendor/github.com/mailru/easyjson/README.md @@ -1,193 +1,336 @@ -# easyjson [![Build Status](https://travis-ci.org/mailru/easyjson.svg?branch=master)](https://travis-ci.org/mailru/easyjson) +# easyjson [![Build Status](https://travis-ci.org/mailru/easyjson.svg?branch=master)](https://travis-ci.org/mailru/easyjson) [![Go Report Card](https://goreportcard.com/badge/github.com/mailru/easyjson)](https://goreportcard.com/report/github.com/mailru/easyjson) -easyjson allows to (un-)marshal JSON golang structs without the use of reflection by generating marshaller code. +Package easyjson provides a fast and easy way to marshal/unmarshal Go structs +to/from JSON without the use of reflection. In performance tests, easyjson +outperforms the standard `encoding/json` package by a factor of 4-5x, and other +JSON encoding packages by a factor of 2-3x. -One of the aims of the library is to keep generated code simple enough so that it can be easily optimized or fixed. Another goal is to provide users with ability to customize the generated code not available in 'encoding/json', such as generating snake_case names or enabling 'omitempty' behavior by default. +easyjson aims to keep generated Go code simple enough so that it can be easily +optimized or fixed. Another goal is to provide users with the ability to +customize the generated code by providing options not available with the +standard `encoding/json` package, such as generating "snake_case" names or +enabling `omitempty` behavior by default. -## usage -``` -go get github.com/mailru/easyjson/... +## Usage +```sh +# install +go get -u github.com/mailru/easyjson/... + +# run easyjson -all .go ``` -This will generate `_easyjson.go` with marshaller/unmarshaller methods for structs. `GOPATH` variable needs to be set up correctly, since the generation invokes a `go run` on a temporary file (this is a really convenient approach to code generation borrowed from https://github.com/pquerna/ffjson). +The above will generate `_easyjson.go` containing the appropriate marshaler and +unmarshaler funcs for all structs contained in `.go`. -## options -``` -Usage of .root/bin/easyjson: +Please note that easyjson requires a full Go build environment and the `GOPATH` +environment variable to be set. This is because easyjson code generation +invokes `go run` on a temporary file (an approach to code generation borrowed +from [ffjson](https://github.com/pquerna/ffjson)). + +## Options +```txt +Usage of easyjson: -all - generate un-/marshallers for all structs in a file + generate marshaler/unmarshalers for all structs in a file -build_tags string - build tags to add to generated file + build tags to add to generated file -leave_temps - do not delete temporary files + do not delete temporary files -no_std_marshalers - don't generate MarshalJSON/UnmarshalJSON methods + don't generate MarshalJSON/UnmarshalJSON funcs -noformat - do not run 'gofmt -w' on output file + do not run 'gofmt -w' on output file -omit_empty - omit empty fields by default + omit empty fields by default + -output_filename string + specify the filename of the output + -pkg + process the whole package instead of just the given file -snake_case - use snake_case names instead of CamelCase by default + use snake_case names instead of CamelCase by default + -lower_camel_case + use lowerCamelCase instead of CamelCase by default -stubs - only generate stubs for marshallers/unmarshallers methods + only generate stubs for marshaler/unmarshaler funcs + -disallow_unknown_fields + return error if some unknown field in json appeared ``` -Using `-all` will generate (un-)marshallers for all structs in the file. By default, structs need to have a line beginning with `easyjson:json` in their docstring, e.g.: -``` +Using `-all` will generate marshalers/unmarshalers for all Go structs in the +file. If `-all` is not provided, then only those structs whose preceding +comment starts with `easyjson:json` will have marshalers/unmarshalers +generated. For example: + +```go //easyjson:json -struct A{} +type A struct {} ``` -`-snake_case` tells easyjson to generate snake\_case field names by default (unless explicitly overriden by a field tag). The CamelCase to snake\_case conversion algorithm should work in most cases (e.g. HTTPVersion will be converted to http_version). There can be names like JSONHTTPRPC where the conversion will return an unexpected result (jsonhttprpc without underscores), but such names require a dictionary to do the conversion and may be ambiguous. - -`-build_tags` will add corresponding build tag line for the generated file. -## marshaller/unmarshaller interfaces - -easyjson generates MarshalJSON/UnmarshalJSON methods that are compatible with interfaces from 'encoding/json'. They are usable with 'json.Marshal' and 'json.Unmarshal' functions, however actually using those will result in significantly worse performance compared to custom interfaces. - -`MarshalEasyJSON` / `UnmarshalEasyJSON` methods are generated for faster parsing using custom Lexer/Writer structs (`jlexer.Lexer` and `jwriter.Writer`). The method signature is defined in `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces. These interfaces allow to avoid using any unnecessary reflection or type assertions during parsing. Functions can be used manually or with `easyjson.Marshal<...>` and `easyjson.Unmarshal<...>` helper methods. - -`jwriter.Writer` struct in addition to function for returning the data as a single slice also has methods to return the size and to send the data to an `io.Writer`. This is aimed at a typical HTTP use-case, when you want to know the `Content-Length` before actually starting to send the data. - -There are helpers in the top-level package for marhsaling/unmarshaling the data using custom interfaces to and from writers, including a helper for `http.ResponseWriter`. - -## custom types -If `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces are implemented by a type involved in JSON parsing, the type will be marshaled/unmarshaled using these methods. `easyjson.Optional` interface allows for a custom type to integrate with 'omitempty' logic. +Additional option notes: + +* `-snake_case` tells easyjson to generate snake\_case field names by default + (unless overridden by a field tag). The CamelCase to snake\_case conversion + algorithm should work in most cases (ie, HTTPVersion will be converted to + "http_version"). + +* `-build_tags` will add the specified build tags to generated Go sources. + +## Generated Marshaler/Unmarshaler Funcs + +For Go struct types, easyjson generates the funcs `MarshalEasyJSON` / +`UnmarshalEasyJSON` for marshaling/unmarshaling JSON. In turn, these satisify +the `easyjson.Marshaler` and `easyjson.Unmarshaler` interfaces and when used in +conjunction with `easyjson.Marshal` / `easyjson.Unmarshal` avoid unnecessary +reflection / type assertions during marshaling/unmarshaling to/from JSON for Go +structs. + +easyjson also generates `MarshalJSON` and `UnmarshalJSON` funcs for Go struct +types compatible with the standard `json.Marshaler` and `json.Unmarshaler` +interfaces. Please be aware that using the standard `json.Marshal` / +`json.Unmarshal` for marshaling/unmarshaling will incur a significant +performance penalty when compared to using `easyjson.Marshal` / +`easyjson.Unmarshal`. + +Additionally, easyjson exposes utility funcs that use the `MarshalEasyJSON` and +`UnmarshalEasyJSON` for marshaling/unmarshaling to and from standard readers +and writers. For example, easyjson provides `easyjson.MarshalToHTTPResponseWriter` +which marshals to the standard `http.ResponseWriter`. Please see the [GoDoc +listing](https://godoc.org/github.com/mailru/easyjson) for the full listing of +utility funcs that are available. + +## Controlling easyjson Marshaling and Unmarshaling Behavior + +Go types can provide their own `MarshalEasyJSON` and `UnmarshalEasyJSON` funcs +that satisify the `easyjson.Marshaler` / `easyjson.Unmarshaler` interfaces. +These will be used by `easyjson.Marshal` and `easyjson.Unmarshal` when defined +for a Go type. + +Go types can also satisify the `easyjson.Optional` interface, which allows the +type to define its own `omitempty` logic. + +## Type Wrappers + +easyjson provides additional type wrappers defined in the `easyjson/opt` +package. These wrap the standard Go primitives and in turn satisify the +easyjson interfaces. + +The `easyjson/opt` type wrappers are useful when needing to distinguish between +a missing value and/or when needing to specifying a default value. Type +wrappers allow easyjson to avoid additional pointers and heap allocations and +can significantly increase performance when used properly. + +## Memory Pooling + +easyjson uses a buffer pool that allocates data in increasing chunks from 128 +to 32768 bytes. Chunks of 512 bytes and larger will be reused with the help of +`sync.Pool`. The maximum size of a chunk is bounded to reduce redundant memory +allocation and to allow larger reusable buffers. + +easyjson's custom allocation buffer pool is defined in the `easyjson/buffer` +package, and the default behavior pool behavior can be modified (if necessary) +through a call to `buffer.Init()` prior to any marshaling or unmarshaling. +Please see the [GoDoc listing](https://godoc.org/github.com/mailru/easyjson/buffer) +for more information. + +## Issues, Notes, and Limitations + +* easyjson is still early in its development. As such, there are likely to be + bugs and missing features when compared to `encoding/json`. In the case of a + missing feature or bug, please create a GitHub issue. Pull requests are + welcome! + +* Unlike `encoding/json`, object keys are case-sensitive. Case-insensitive + matching is not currently provided due to the significant performance hit + when doing case-insensitive key matching. In the future, case-insensitive + object key matching may be provided via an option to the generator. + +* easyjson makes use of `unsafe`, which simplifies the code and + provides significant performance benefits by allowing no-copy + conversion from `[]byte` to `string`. That said, `unsafe` is used + only when unmarshaling and parsing JSON, and any `unsafe` operations + / memory allocations done will be safely deallocated by + easyjson. Set the build tag `easyjson_nounsafe` to compile it + without `unsafe`. + +* easyjson is compatible with Google App Engine. The `appengine` build + tag (set by App Engine's environment) will automatically disable the + use of `unsafe`, which is not allowed in App Engine's Standard + Environment. Note that the use with App Engine is still experimental. + +* Floats are formatted using the default precision from Go's `strconv` package. + As such, easyjson will not correctly handle high precision floats when + marshaling/unmarshaling JSON. Note, however, that there are very few/limited + uses where this behavior is not sufficient for general use. That said, a + different package may be needed if precise marshaling/unmarshaling of high + precision floats to/from JSON is required. + +* While unmarshaling, the JSON parser does the minimal amount of work needed to + skip over unmatching parens, and as such full validation is not done for the + entire JSON value being unmarshaled/parsed. + +* Currently there is no true streaming support for encoding/decoding as + typically for many uses/protocols the final, marshaled length of the JSON + needs to be known prior to sending the data. Currently this is not possible + with easyjson's architecture. + +* easyjson parser and codegen based on reflection, so it wont works on `package main` + files, because they cant be imported by parser. + +## Benchmarks + +Most benchmarks were done using the example +[13kB example JSON](https://dev.twitter.com/rest/reference/get/search/tweets) +(9k after eliminating whitespace). This example is similar to real-world data, +is well-structured, and contains a healthy variety of different types, making +it ideal for JSON serialization benchmarks. + +Note: + +* For small request benchmarks, an 80 byte portion of the above example was + used. + +* For large request marshaling benchmarks, a struct containing 50 regular + samples was used, making a ~500kB output JSON. + +* Benchmarks are showing the results of easyjson's default behaviour, + which makes use of `unsafe`. -As an example, easyjson includes an `easyjson.RawMessage` analogous to `json.RawMessage`. - -Also, there are 'optional' wrappers for primitive types in `easyjson/opt` package. These are useful in the case when it is necessary to distinguish between missing and default value for the type. Wrappers allow to avoid pointers and extra heap allocations in such cases. - -## memory pooling - -The library uses a custom buffer which allocates data in increasing chunks (128-32768 bytes). Chunks of 512 bytes and larger are reused with the help of `sync.Pool`. The maximum size of a chunk is bounded to reduce redundancy in memory allocation and to make the chunks more reusable in the case of large buffer sizes. - -The buffer code is in `easyjson/buffer` package the exact values can be tweaked by a `buffer.Init()` call before the first serialization. - -## limitations -* The library is at an early stage, there are likely to be some bugs and some features of 'encoding/json' may not be supported. Please report such cases, so that they may be fixed sooner. -* Object keys are case-sensitive (unlike encodin/json). Case-insentive behavior will be implemented as an option (case-insensitive matching is slower). -* Unsafe package is used by the code. While a non-unsafe version of easyjson can be made in the future, using unsafe package simplifies a lot of code by allowing no-copy []byte to string conversion within the library. This is used only during parsing and all the returned values are allocated properly. -* Floats are currently formatted with default precision for 'strconv' package. It is obvious that it is not always the correct way to handle it, but there aren't enough use-cases for floats at hand to do anything better. -* During parsing, parts of JSON that are skipped over are not syntactically validated more than required to skip matching parentheses. -* No true streaming support for encoding/decoding. For many use-cases and protocols, data length is typically known on input and needs to be known before sending the data. - -## benchmarks -Most benchmarks were done using a sample 13kB JSON (9k if serialized back trimming the whitespace) from https://dev.twitter.com/rest/reference/get/search/tweets. The sample is very close to real-world data, quite structured and contains a variety of different types. - -For small request benchmarks, an 80-byte portion of the regular sample was used. - -For large request marshalling benchmarks, a struct containing 50 regular samples was used, making a ~500kB output JSON. - -Benchmarks are available in the repository and are run on 'make'. +Benchmarks are available in the repository and can be run by invoking `make`. ### easyjson vs. encoding/json -easyjson seems to be 5-6 times faster than the default json serialization for unmarshalling, 3-4 times faster for non-concurrent marshalling. Concurrent marshalling is 6-7x faster if marshalling to a writer. +easyjson is roughly 5-6 times faster than the standard `encoding/json` for +unmarshaling, and 3-4 times faster for non-concurrent marshaling. Concurrent +marshaling is 6-7x faster if marshaling to a writer. ### easyjson vs. ffjson -easyjson uses the same approach for code generation as ffjson, but a significantly different approach to lexing and generated code. This allows easyjson to be 2-3x faster for unmarshalling and 1.5-2x faster for non-concurrent unmarshalling. +easyjson uses the same approach for JSON marshaling as +[ffjson](https://github.com/pquerna/ffjson), but takes a significantly +different approach to lexing and parsing JSON during unmarshaling. This means +easyjson is roughly 2-3x faster for unmarshaling and 1.5-2x faster for +non-concurrent unmarshaling. -ffjson seems to behave weird if used concurrently: for large request pooling hurts performance instead of boosting it, it also does not quite scale well. These issues are likely to be fixable and until that comparisons might vary from version to version a lot. +As of this writing, `ffjson` seems to have issues when used concurrently: +specifically, large request pooling hurts `ffjson`'s performance and causes +scalability issues. These issues with `ffjson` can likely be fixed, but as of +writing remain outstanding/known issues with `ffjson`. -easyjson is similar in performance for small requests and 2-5x times faster for large ones if used with a writer. +easyjson and `ffjson` have similar performance for small requests, however +easyjson outperforms `ffjson` by roughly 2-5x times for large requests when +used with a writer. ### easyjson vs. go/codec -github.com/ugorji/go/codec library provides compile-time helpers for JSON generation. In this case, helpers are not exactly marshallers as they are encoding-independent. +[go/codec](https://github.com/ugorji/go) provides +compile-time helpers for JSON generation. In this case, helpers do not work +like marshalers as they are encoding-independent. -easyjson is generally ~2x faster for non-concurrent benchmarks and about 3x faster for concurrent encoding (without marshalling to a writer). Unsafe option for generated helpers was used. +easyjson is generally 2x faster than `go/codec` for non-concurrent benchmarks +and about 3x faster for concurrent encoding (without marshaling to a writer). -As an attempt to measure marshalling performance of 'go/codec' (as opposed to allocations/memcpy/writer interface invocations), a benchmark was done with resetting lenght of a byte slice rather than resetting the whole slice to nil. However, the optimization in this exact form may not be applicable in practice, since the memory is not freed between marshalling operations. +In an attempt to measure marshaling performance of `go/codec` (as opposed to +allocations/memcpy/writer interface invocations), a benchmark was done with +resetting length of a byte slice rather than resetting the whole slice to nil. +However, the optimization in this exact form may not be applicable in practice, +since the memory is not freed between marshaling operations. ### easyjson vs 'ujson' python module -ujson is using C code for parsing, so it is interesting to see how plain golang compares to that. It is imporant to note that the resulting object for python is slower to access, since the library parses JSON object into dictionaries. - -easyjson seems to be slightly faster for unmarshalling (finally!) and 2-3x faster for marshalling. - -### benchmark figures -The data was measured on 4 February, 2016 using current ffjson and golang 1.6. Data for go/codec was added on 4 March 2016, benchmarked on the same machine. - -#### Unmarshalling -| lib | json size | MB/s | allocs/op | B/op -|--------|-----------|------|-----------|------- -|standard| regular | 22 | 218 | 10229 -|standard| small | 9.7 | 14 | 720 -|--------|-----------|------|-----------|------- -|easyjson| regular | 125 | 128 | 9794 -|easyjson| small | 67 | 3 | 128 -|--------|-----------|------|-----------|------- -|ffjson | regular | 66 | 141 | 9985 -|ffjson | small | 17.6 | 10 | 488 -|--------|-----------|------|-----------|------- -|codec | regular | 55 | 434 | 19299 -|codec | small | 29 | 7 | 336 -|--------|-----------|------|-----------|------- -|ujson | regular | 103 | N/A | N/A - -#### Marshalling, one goroutine. -| lib | json size | MB/s | allocs/op | B/op -|----------|-----------|------|-----------|------- -|standard | regular | 75 | 9 | 23256 -|standard | small | 32 | 3 | 328 -|standard | large | 80 | 17 | 1.2M -|----------|-----------|------|-----------|------- -|easyjson | regular | 213 | 9 | 10260 -|easyjson* | regular | 263 | 8 | 742 -|easyjson | small | 125 | 1 | 128 -|easyjson | large | 212 | 33 | 490k -|easyjson* | large | 262 | 25 | 2879 -|----------|-----------|------|-----------|------- -|ffjson | regular | 122 | 153 | 21340 -|ffjson** | regular | 146 | 152 | 4897 -|ffjson | small | 36 | 5 | 384 -|ffjson** | small | 64 | 4 | 128 -|ffjson | large | 134 | 7317 | 818k -|ffjson** | large | 125 | 7320 | 827k -|----------|-----------|------|-----------|------- -|codec | regular | 80 | 17 | 33601 -|codec*** | regular | 108 | 9 | 1153 -|codec | small | 42 | 3 | 304 -|codec*** | small | 56 | 1 | 48 -|codec | large | 73 | 483 | 2.5M -|codec*** | large | 103 | 451 | 66007 -|----------|-----------|------|-----------|------- -|ujson | regular | 92 | N/A | N/A -\* marshalling to a writer, + +[ujson](https://github.com/esnme/ultrajson) is using C code for parsing, so it +is interesting to see how plain golang compares to that. It is imporant to note +that the resulting object for python is slower to access, since the library +parses JSON object into dictionaries. + +easyjson is slightly faster for unmarshaling and 2-3x faster than `ujson` for +marshaling. + +### Benchmark Results + +`ffjson` results are from February 4th, 2016, using the latest `ffjson` and go1.6. +`go/codec` results are from March 4th, 2016, using the latest `go/codec` and go1.6. + +#### Unmarshaling + +| lib | json size | MB/s | allocs/op | B/op | +|:---------|:----------|-----:|----------:|------:| +| standard | regular | 22 | 218 | 10229 | +| standard | small | 9.7 | 14 | 720 | +| | | | | | +| easyjson | regular | 125 | 128 | 9794 | +| easyjson | small | 67 | 3 | 128 | +| | | | | | +| ffjson | regular | 66 | 141 | 9985 | +| ffjson | small | 17.6 | 10 | 488 | +| | | | | | +| codec | regular | 55 | 434 | 19299 | +| codec | small | 29 | 7 | 336 | +| | | | | | +| ujson | regular | 103 | N/A | N/A | + +#### Marshaling, one goroutine. + +| lib | json size | MB/s | allocs/op | B/op | +|:----------|:----------|-----:|----------:|------:| +| standard | regular | 75 | 9 | 23256 | +| standard | small | 32 | 3 | 328 | +| standard | large | 80 | 17 | 1.2M | +| | | | | | +| easyjson | regular | 213 | 9 | 10260 | +| easyjson* | regular | 263 | 8 | 742 | +| easyjson | small | 125 | 1 | 128 | +| easyjson | large | 212 | 33 | 490k | +| easyjson* | large | 262 | 25 | 2879 | +| | | | | | +| ffjson | regular | 122 | 153 | 21340 | +| ffjson** | regular | 146 | 152 | 4897 | +| ffjson | small | 36 | 5 | 384 | +| ffjson** | small | 64 | 4 | 128 | +| ffjson | large | 134 | 7317 | 818k | +| ffjson** | large | 125 | 7320 | 827k | +| | | | | | +| codec | regular | 80 | 17 | 33601 | +| codec*** | regular | 108 | 9 | 1153 | +| codec | small | 42 | 3 | 304 | +| codec*** | small | 56 | 1 | 48 | +| codec | large | 73 | 483 | 2.5M | +| codec*** | large | 103 | 451 | 66007 | +| | | | | | +| ujson | regular | 92 | N/A | N/A | + +\* marshaling to a writer, \*\* using `ffjson.Pool()`, \*\*\* reusing output slice instead of resetting it to nil -#### Marshalling, concurrent. -| lib | json size | MB/s | allocs/op | B/op -|----------|-----------|-------|-----------|------- -|standard | regular | 252 | 9 | 23257 -|standard | small | 124 | 3 | 328 -|standard | large | 289 | 17 | 1.2M -|----------|-----------|-------|-----------|------- -|easyjson | regular | 792 | 9 | 10597 -|easyjson* | regular | 1748 | 8 | 779 -|easyjson | small | 333 | 1 | 128 -|easyjson | large | 718 | 36 | 548k -|easyjson* | large | 2134 | 25 | 4957 -|----------|-----------|------|-----------|------- -|ffjson | regular | 301 | 153 | 21629 -|ffjson** | regular | 707 | 152 | 5148 -|ffjson | small | 62 | 5 | 384 -|ffjson** | small | 282 | 4 | 128 -|ffjson | large | 438 | 7330 | 1.0M -|ffjson** | large | 131 | 7319 | 820k -|----------|-----------|------|-----------|------- -|codec | regular | 183 | 17 | 33603 -|codec*** | regular | 671 | 9 | 1157 -|codec | small | 147 | 3 | 304 -|codec*** | small | 299 | 1 | 48 -|codec | large | 190 | 483 | 2.5M -|codec*** | large | 752 | 451 | 77574 -\* marshalling to a writer, +#### Marshaling, concurrent. + +| lib | json size | MB/s | allocs/op | B/op | +|:----------|:----------|-----:|----------:|------:| +| standard | regular | 252 | 9 | 23257 | +| standard | small | 124 | 3 | 328 | +| standard | large | 289 | 17 | 1.2M | +| | | | | | +| easyjson | regular | 792 | 9 | 10597 | +| easyjson* | regular | 1748 | 8 | 779 | +| easyjson | small | 333 | 1 | 128 | +| easyjson | large | 718 | 36 | 548k | +| easyjson* | large | 2134 | 25 | 4957 | +| | | | | | +| ffjson | regular | 301 | 153 | 21629 | +| ffjson** | regular | 707 | 152 | 5148 | +| ffjson | small | 62 | 5 | 384 | +| ffjson** | small | 282 | 4 | 128 | +| ffjson | large | 438 | 7330 | 1.0M | +| ffjson** | large | 131 | 7319 | 820k | +| | | | | | +| codec | regular | 183 | 17 | 33603 | +| codec*** | regular | 671 | 9 | 1157 | +| codec | small | 147 | 3 | 304 | +| codec*** | small | 299 | 1 | 48 | +| codec | large | 190 | 483 | 2.5M | +| codec*** | large | 752 | 451 | 77574 | + +\* marshaling to a writer, \*\* using `ffjson.Pool()`, \*\*\* reusing output slice instead of resetting it to nil - - - diff --git a/vendor/github.com/mailru/easyjson/buffer/pool.go b/vendor/github.com/mailru/easyjson/buffer/pool.go index 4de4a51db562..07fb4bc1f7bf 100644 --- a/vendor/github.com/mailru/easyjson/buffer/pool.go +++ b/vendor/github.com/mailru/easyjson/buffer/pool.go @@ -179,18 +179,25 @@ func (b *Buffer) DumpTo(w io.Writer) (written int, err error) { } // BuildBytes creates a single byte slice with all the contents of the buffer. Data is -// copied if it does not fit in a single chunk. -func (b *Buffer) BuildBytes() []byte { +// copied if it does not fit in a single chunk. You can optionally provide one byte +// slice as argument that it will try to reuse. +func (b *Buffer) BuildBytes(reuse ...[]byte) []byte { if len(b.bufs) == 0 { - ret := b.Buf b.toPool = nil b.Buf = nil - return ret } - ret := make([]byte, 0, b.Size()) + var ret []byte + size := b.Size() + + // If we got a buffer as argument and it is big enought, reuse it. + if len(reuse) == 1 && cap(reuse[0]) >= size { + ret = reuse[0][:0] + } else { + ret = make([]byte, 0, size) + } for _, buf := range b.bufs { ret = append(ret, buf...) putBuf(buf) @@ -205,3 +212,59 @@ func (b *Buffer) BuildBytes() []byte { return ret } + +type readCloser struct { + offset int + bufs [][]byte +} + +func (r *readCloser) Read(p []byte) (n int, err error) { + for _, buf := range r.bufs { + // Copy as much as we can. + x := copy(p[n:], buf[r.offset:]) + n += x // Increment how much we filled. + + // Did we empty the whole buffer? + if r.offset+x == len(buf) { + // On to the next buffer. + r.offset = 0 + r.bufs = r.bufs[1:] + + // We can release this buffer. + putBuf(buf) + } else { + r.offset += x + } + + if n == len(p) { + break + } + } + // No buffers left or nothing read? + if len(r.bufs) == 0 { + err = io.EOF + } + return +} + +func (r *readCloser) Close() error { + // Release all remaining buffers. + for _, buf := range r.bufs { + putBuf(buf) + } + // In case Close gets called multiple times. + r.bufs = nil + + return nil +} + +// ReadCloser creates an io.ReadCloser with all the contents of the buffer. +func (b *Buffer) ReadCloser() io.ReadCloser { + ret := &readCloser{0, append(b.bufs, b.Buf)} + + b.bufs = nil + b.toPool = nil + b.Buf = nil + + return ret +} diff --git a/vendor/github.com/mailru/easyjson/go.mod b/vendor/github.com/mailru/easyjson/go.mod new file mode 100644 index 000000000000..7bc4a6584425 --- /dev/null +++ b/vendor/github.com/mailru/easyjson/go.mod @@ -0,0 +1,3 @@ +module github.com/mailru/easyjson + +go 1.12 diff --git a/vendor/github.com/mailru/easyjson/helpers.go b/vendor/github.com/mailru/easyjson/helpers.go index b86b87d228a1..04ac63562870 100644 --- a/vendor/github.com/mailru/easyjson/helpers.go +++ b/vendor/github.com/mailru/easyjson/helpers.go @@ -26,6 +26,16 @@ type Optional interface { IsDefined() bool } +// UnknownsUnmarshaler provides a method to unmarshal unknown struct fileds and save them as you want +type UnknownsUnmarshaler interface { + UnmarshalUnknown(in *jlexer.Lexer, key string) +} + +// UnknownsMarshaler provides a method to write additional struct fields +type UnknownsMarshaler interface { + MarshalUnknowns(w *jwriter.Writer, first bool) +} + // Marshal returns data as a single byte slice. Method is suboptimal as the data is likely to be copied // from a chain of smaller chunks. func Marshal(v Marshaler) ([]byte, error) { diff --git a/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go b/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go new file mode 100644 index 000000000000..ff7b27c5b203 --- /dev/null +++ b/vendor/github.com/mailru/easyjson/jlexer/bytestostr.go @@ -0,0 +1,24 @@ +// This file will only be included to the build if neither +// easyjson_nounsafe nor appengine build tag is set. See README notes +// for more details. + +//+build !easyjson_nounsafe +//+build !appengine + +package jlexer + +import ( + "reflect" + "unsafe" +) + +// bytesToStr creates a string pointing at the slice to avoid copying. +// +// Warning: the string returned by the function should be used with care, as the whole input data +// chunk may be either blocked from being freed by GC because of a single string or the buffer.Data +// may be garbage-collected even when the string exists. +func bytesToStr(data []byte) string { + h := (*reflect.SliceHeader)(unsafe.Pointer(&data)) + shdr := reflect.StringHeader{Data: h.Data, Len: h.Len} + return *(*string)(unsafe.Pointer(&shdr)) +} diff --git a/vendor/github.com/mailru/easyjson/jlexer/bytestostr_nounsafe.go b/vendor/github.com/mailru/easyjson/jlexer/bytestostr_nounsafe.go new file mode 100644 index 000000000000..864d1be67638 --- /dev/null +++ b/vendor/github.com/mailru/easyjson/jlexer/bytestostr_nounsafe.go @@ -0,0 +1,13 @@ +// This file is included to the build if any of the buildtags below +// are defined. Refer to README notes for more details. + +//+build easyjson_nounsafe appengine + +package jlexer + +// bytesToStr creates a string normally from []byte +// +// Note that this method is roughly 1.5x slower than using the 'unsafe' method. +func bytesToStr(data []byte) string { + return string(data) +} diff --git a/vendor/github.com/mailru/easyjson/jlexer/lexer.go b/vendor/github.com/mailru/easyjson/jlexer/lexer.go index d700c0a32fb5..ddd376b844cb 100644 --- a/vendor/github.com/mailru/easyjson/jlexer/lexer.go +++ b/vendor/github.com/mailru/easyjson/jlexer/lexer.go @@ -5,12 +5,15 @@ package jlexer import ( + "encoding/base64" + "encoding/json" + "errors" "fmt" "io" - "reflect" "strconv" + "unicode" + "unicode/utf16" "unicode/utf8" - "unsafe" ) // tokenKind determines type of a token. @@ -45,11 +48,13 @@ type Lexer struct { firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. - err error // Error encountered during lexing, if any. + UseMultipleErrors bool // If we want to use multiple errors. + fatalError error // Fatal error occurred during lexing. It is usually a syntax error. + multipleErrors []*LexerError // Semantic errors occurred during lexing. Marshalling will be continued after finding this errors. } -// fetchToken scans the input for the next token. -func (r *Lexer) fetchToken() { +// FetchToken scans the input for the next token. +func (r *Lexer) FetchToken() { r.token.kind = tokenUndef r.start = r.pos @@ -147,7 +152,7 @@ func (r *Lexer) fetchToken() { return } } - r.err = io.EOF + r.fatalError = io.EOF return } @@ -199,17 +204,6 @@ func (r *Lexer) fetchFalse() { } } -// bytesToStr creates a string pointing at the slice to avoid copying. -// -// Warning: the string returned by the function should be used with care, as the whole input data -// chunk may be either blocked from being freed by GC because of a single string or the buffer.Data -// may be garbage-collected even when the string exists. -func bytesToStr(data []byte) string { - h := (*reflect.SliceHeader)(unsafe.Pointer(&data)) - shdr := reflect.StringHeader{h.Data, h.Len} - return *(*string)(unsafe.Pointer(&shdr)) -} - // fetchNumber scans a number literal token. func (r *Lexer) fetchNumber() { hasE := false @@ -246,7 +240,7 @@ func (r *Lexer) fetchNumber() { // findStringLen tries to scan into the string literal for ending quote char to determine required size. // The size will be exact if no escapes are present and may be inexact if there are escaped chars. -func findStringLen(data []byte) (hasEscapes bool, length int) { +func findStringLen(data []byte) (isValid, hasEscapes bool, length int) { delta := 0 for i := 0; i < len(data); i++ { @@ -258,11 +252,38 @@ func findStringLen(data []byte) (hasEscapes bool, length int) { delta++ } case '"': - return (delta > 0), (i - delta) + return true, (delta > 0), (i - delta) } } - return false, len(data) + return false, false, len(data) +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + var val rune + for i := 2; i < len(s) && i < 6; i++ { + var v byte + c := s[i] + switch c { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + v = c - '0' + case 'a', 'b', 'c', 'd', 'e', 'f': + v = c - 'a' + 10 + case 'A', 'B', 'C', 'D', 'E', 'F': + v = c - 'A' + 10 + default: + return -1 + } + + val <<= 4 + val |= rune(v) + } + return val } // processEscape processes a single escape sequence and returns number of bytes processed. @@ -292,39 +313,28 @@ func (r *Lexer) processEscape(data []byte) (int, error) { r.token.byteValue = append(r.token.byteValue, '\t') return 2, nil case 'u': - default: - return 0, fmt.Errorf("syntax error") - } - - var val rune - - for i := 2; i < len(data) && i < 6; i++ { - var v byte - c = data[i] - switch c { - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - v = c - '0' - case 'a', 'b', 'c', 'd', 'e', 'f': - v = c - 'a' + 10 - case 'A', 'B', 'C', 'D', 'E', 'F': - v = c - 'A' + 10 - default: - return 0, fmt.Errorf("syntax error") + rr := getu4(data) + if rr < 0 { + return 0, errors.New("syntax error") } - val <<= 4 - val |= rune(v) - } - - l := utf8.RuneLen(val) - if l == -1 { - return 0, fmt.Errorf("invalid unicode escape") + read := 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(data[read:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + read += 6 + rr = dec + } else { + rr = unicode.ReplacementChar + } + } + var d [4]byte + s := utf8.EncodeRune(d[:], rr) + r.token.byteValue = append(r.token.byteValue, d[:s]...) + return read, nil } - var d [4]byte - utf8.EncodeRune(d[:], val) - r.token.byteValue = append(r.token.byteValue, d[:l]...) - return 6, nil + return 0, errors.New("syntax error") } // fetchString scans a string literal token. @@ -332,7 +342,12 @@ func (r *Lexer) fetchString() { r.pos++ data := r.Data[r.pos:] - hasEscapes, length := findStringLen(data) + isValid, hasEscapes, length := findStringLen(data) + if !isValid { + r.pos += length + r.errParse("unterminated string literal") + return + } if !hasEscapes { r.token.byteValue = data[:length] r.pos += length + 1 @@ -368,11 +383,11 @@ func (r *Lexer) fetchString() { // scanToken scans the next token if no token is currently available in the lexer. func (r *Lexer) scanToken() { - if r.token.kind != tokenUndef || r.err != nil { + if r.token.kind != tokenUndef || r.fatalError != nil { return } - r.fetchToken() + r.FetchToken() } // consume resets the current token to allow scanning the next one. @@ -383,20 +398,20 @@ func (r *Lexer) consume() { // Ok returns true if no error (including io.EOF) was encountered during scanning. func (r *Lexer) Ok() bool { - return r.err == nil + return r.fatalError == nil } const maxErrorContextLen = 13 func (r *Lexer) errParse(what string) { - if r.err == nil { + if r.fatalError == nil { var str string if len(r.Data)-r.pos <= maxErrorContextLen { str = string(r.Data) } else { str = string(r.Data[r.pos:r.pos+maxErrorContextLen-3]) + "..." } - r.err = &LexerError{ + r.fatalError = &LexerError{ Reason: what, Offset: r.pos, Data: str, @@ -409,36 +424,64 @@ func (r *Lexer) errSyntax() { } func (r *Lexer) errInvalidToken(expected string) { - if r.err == nil { - var str string - if len(r.token.byteValue) <= maxErrorContextLen { - str = string(r.token.byteValue) - } else { - str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." + if r.fatalError != nil { + return + } + if r.UseMultipleErrors { + r.pos = r.start + r.consume() + r.SkipRecursive() + switch expected { + case "[": + r.token.delimValue = ']' + r.token.kind = tokenDelim + case "{": + r.token.delimValue = '}' + r.token.kind = tokenDelim } - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), - Offset: r.pos, - Data: str, - } + Offset: r.start, + Data: string(r.Data[r.start:r.pos]), + }) + return + } + + var str string + if len(r.token.byteValue) <= maxErrorContextLen { + str = string(r.token.byteValue) + } else { + str = string(r.token.byteValue[:maxErrorContextLen-3]) + "..." } + r.fatalError = &LexerError{ + Reason: fmt.Sprintf("expected %s", expected), + Offset: r.pos, + Data: str, + } +} + +func (r *Lexer) GetPos() int { + return r.pos } // Delim consumes a token and verifies that it is the given delimiter. func (r *Lexer) Delim(c byte) { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } + if !r.Ok() || r.token.delimValue != c { + r.consume() // errInvalidToken can change token if UseMultipleErrors is enabled. r.errInvalidToken(string([]byte{c})) + } else { + r.consume() } - r.consume() } // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } return !r.Ok() || r.token.delimValue == c } @@ -446,7 +489,7 @@ func (r *Lexer) IsDelim(c byte) bool { // Null verifies that the next token is null and consumes it. func (r *Lexer) Null() { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() || r.token.kind != tokenNull { r.errInvalidToken("null") @@ -457,7 +500,7 @@ func (r *Lexer) Null() { // IsNull returns true if the next token is a null keyword. func (r *Lexer) IsNull() bool { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } return r.Ok() && r.token.kind == tokenNull } @@ -465,7 +508,7 @@ func (r *Lexer) IsNull() bool { // Skip skips a single token. func (r *Lexer) Skip() { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } r.consume() } @@ -476,14 +519,14 @@ func (r *Lexer) Skip() { // Note: no syntax validation is performed on the skipped data. func (r *Lexer) SkipRecursive() { r.scanToken() - var start, end byte - if r.token.delimValue == '{' { + switch r.token.delimValue { + case '{': start, end = '{', '}' - } else if r.token.delimValue == '[' { + case '[': start, end = '[', ']' - } else { + default: r.consume() return } @@ -505,7 +548,7 @@ func (r *Lexer) SkipRecursive() { return } case c == '\\' && inQuotes: - wasEscape = true + wasEscape = !wasEscape continue case c == '"' && inQuotes: inQuotes = wasEscape @@ -515,7 +558,11 @@ func (r *Lexer) SkipRecursive() { wasEscape = false } r.pos = len(r.Data) - r.err = io.EOF + r.fatalError = &LexerError{ + Reason: "EOF reached while skipping array/object or token", + Offset: r.pos, + Data: string(r.Data[r.pos:]), + } } // Raw fetches the next item recursively as a data slice @@ -527,48 +574,107 @@ func (r *Lexer) Raw() []byte { return r.Data[r.start:r.pos] } -// UnsafeString returns the string value if the token is a string literal. -// -// Warning: returned string may point to the input buffer, so the string should not outlive -// the input buffer. Intended pattern of usage is as an argument to a switch statement. -func (r *Lexer) UnsafeString() string { +// IsStart returns whether the lexer is positioned at the start +// of an input string. +func (r *Lexer) IsStart() bool { + return r.pos == 0 +} + +// Consumed reads all remaining bytes from the input, publishing an error if +// there is anything but whitespace remaining. +func (r *Lexer) Consumed() { + if r.pos > len(r.Data) || !r.Ok() { + return + } + + for _, c := range r.Data[r.pos:] { + if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + r.AddError(&LexerError{ + Reason: "invalid character '" + string(c) + "' after top-level value", + Offset: r.pos, + Data: string(r.Data[r.pos:]), + }) + return + } + + r.pos++ + r.start++ + } +} + +func (r *Lexer) unsafeString() (string, []byte) { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() || r.token.kind != tokenString { r.errInvalidToken("string") - return "" + return "", nil } - + bytes := r.token.byteValue ret := bytesToStr(r.token.byteValue) r.consume() + return ret, bytes +} + +// UnsafeString returns the string value if the token is a string literal. +// +// Warning: returned string may point to the input buffer, so the string should not outlive +// the input buffer. Intended pattern of usage is as an argument to a switch statement. +func (r *Lexer) UnsafeString() string { + ret, _ := r.unsafeString() + return ret +} + +// UnsafeBytes returns the byte slice if the token is a string literal. +func (r *Lexer) UnsafeBytes() []byte { + _, ret := r.unsafeString() return ret } // String reads a string literal. func (r *Lexer) String() string { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() || r.token.kind != tokenString { r.errInvalidToken("string") return "" - } ret := string(r.token.byteValue) r.consume() return ret } +// Bytes reads a string literal and base64 decodes it into a byte slice. +func (r *Lexer) Bytes() []byte { + if r.token.kind == tokenUndef && r.Ok() { + r.FetchToken() + } + if !r.Ok() || r.token.kind != tokenString { + r.errInvalidToken("string") + return nil + } + ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue))) + n, err := base64.StdEncoding.Decode(ret, r.token.byteValue) + if err != nil { + r.fatalError = &LexerError{ + Reason: err.Error(), + } + return nil + } + + r.consume() + return ret[:n] +} + // Bool reads a true or false boolean keyword. func (r *Lexer) Bool() bool { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() || r.token.kind != tokenBool { r.errInvalidToken("bool") return false - } ret := r.token.boolValue r.consume() @@ -577,12 +683,11 @@ func (r *Lexer) Bool() bool { func (r *Lexer) number() string { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() || r.token.kind != tokenNumber { r.errInvalidToken("number") return "" - } ret := bytesToStr(r.token.byteValue) r.consume() @@ -597,9 +702,11 @@ func (r *Lexer) Uint8() uint8 { n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return uint8(n) } @@ -612,9 +719,11 @@ func (r *Lexer) Uint16() uint16 { n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return uint16(n) } @@ -627,9 +736,11 @@ func (r *Lexer) Uint32() uint32 { n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return uint32(n) } @@ -642,9 +753,11 @@ func (r *Lexer) Uint64() uint64 { n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return n } @@ -661,9 +774,11 @@ func (r *Lexer) Int8() int8 { n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return int8(n) } @@ -676,9 +791,11 @@ func (r *Lexer) Int16() int16 { n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return int16(n) } @@ -691,9 +808,11 @@ func (r *Lexer) Int32() int32 { n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return int32(n) } @@ -706,9 +825,11 @@ func (r *Lexer) Int64() int64 { n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) } return n } @@ -718,61 +839,69 @@ func (r *Lexer) Int() int { } func (r *Lexer) Uint8Str() uint8 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return uint8(n) } func (r *Lexer) Uint16Str() uint16 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return uint16(n) } func (r *Lexer) Uint32Str() uint32 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return uint32(n) } func (r *Lexer) Uint64Str() uint64 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseUint(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return n } @@ -781,62 +910,74 @@ func (r *Lexer) UintStr() uint { return uint(r.Uint64Str()) } +func (r *Lexer) UintptrStr() uintptr { + return uintptr(r.Uint64Str()) +} + func (r *Lexer) Int8Str() int8 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 8) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return int8(n) } func (r *Lexer) Int16Str() int16 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 16) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return int16(n) } func (r *Lexer) Int32Str() int32 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return int32(n) } func (r *Lexer) Int64Str() int64 { - s := r.UnsafeString() + s, b := r.unsafeString() if !r.Ok() { return 0 } n, err := strconv.ParseInt(s, 10, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: string(b), + }) } return n } @@ -853,9 +994,27 @@ func (r *Lexer) Float32() float32 { n, err := strconv.ParseFloat(s, 32) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) + } + return float32(n) +} + +func (r *Lexer) Float32Str() float32 { + s, b := r.unsafeString() + if !r.Ok() { + return 0 + } + n, err := strconv.ParseFloat(s, 32) + if err != nil { + r.addNonfatalError(&LexerError{ + Offset: r.start, + Reason: err.Error(), + Data: string(b), + }) } return float32(n) } @@ -868,27 +1027,94 @@ func (r *Lexer) Float64() float64 { n, err := strconv.ParseFloat(s, 64) if err != nil { - r.err = &LexerError{ + r.addNonfatalError(&LexerError{ + Offset: r.start, Reason: err.Error(), - } + Data: s, + }) + } + return n +} + +func (r *Lexer) Float64Str() float64 { + s, b := r.unsafeString() + if !r.Ok() { + return 0 + } + n, err := strconv.ParseFloat(s, 64) + if err != nil { + r.addNonfatalError(&LexerError{ + Offset: r.start, + Reason: err.Error(), + Data: string(b), + }) } return n } func (r *Lexer) Error() error { - return r.err + return r.fatalError } func (r *Lexer) AddError(e error) { - if r.err == nil { - r.err = e + if r.fatalError == nil { + r.fatalError = e + } +} + +func (r *Lexer) AddNonFatalError(e error) { + r.addNonfatalError(&LexerError{ + Offset: r.start, + Data: string(r.Data[r.start:r.pos]), + Reason: e.Error(), + }) +} + +func (r *Lexer) addNonfatalError(err *LexerError) { + if r.UseMultipleErrors { + // We don't want to add errors with the same offset. + if len(r.multipleErrors) != 0 && r.multipleErrors[len(r.multipleErrors)-1].Offset == err.Offset { + return + } + r.multipleErrors = append(r.multipleErrors, err) + return + } + r.fatalError = err +} + +func (r *Lexer) GetNonFatalErrors() []*LexerError { + return r.multipleErrors +} + +// JsonNumber fetches and json.Number from 'encoding/json' package. +// Both int, float or string, contains them are valid values +func (r *Lexer) JsonNumber() json.Number { + if r.token.kind == tokenUndef && r.Ok() { + r.FetchToken() + } + if !r.Ok() { + r.errInvalidToken("json.Number") + return json.Number("") + } + + switch r.token.kind { + case tokenString: + return json.Number(r.String()) + case tokenNumber: + return json.Number(r.Raw()) + case tokenNull: + r.Null() + return json.Number("") + default: + r.errSyntax() + return json.Number("") } } // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { if r.token.kind == tokenUndef && r.Ok() { - r.fetchToken() + r.FetchToken() } if !r.Ok() { @@ -926,7 +1152,7 @@ func (r *Lexer) Interface() interface{} { } else if r.token.delimValue == '[' { r.consume() - var ret []interface{} + ret := []interface{}{} for !r.IsDelim(']') { ret = append(ret, r.Interface()) r.WantComma() diff --git a/vendor/github.com/mailru/easyjson/jwriter/writer.go b/vendor/github.com/mailru/easyjson/jwriter/writer.go index 907675f9ca51..eb8547ccc27a 100644 --- a/vendor/github.com/mailru/easyjson/jwriter/writer.go +++ b/vendor/github.com/mailru/easyjson/jwriter/writer.go @@ -9,10 +9,22 @@ import ( "github.com/mailru/easyjson/buffer" ) +// Flags describe various encoding options. The behavior may be actually implemented in the encoder, but +// Flags field in Writer is used to set and pass them around. +type Flags int + +const ( + NilMapAsEmpty Flags = 1 << iota // Encode nil map as '{}' rather than 'null'. + NilSliceAsEmpty // Encode nil slice as '[]' rather than 'null'. +) + // Writer is a JSON writer. type Writer struct { - Error error - Buffer buffer.Buffer + Flags Flags + + Error error + Buffer buffer.Buffer + NoEscapeHTML bool } // Size returns the size of the data that was written out. @@ -25,13 +37,24 @@ func (w *Writer) DumpTo(out io.Writer) (written int, err error) { return w.Buffer.DumpTo(out) } -// BuildBytes returns writer data as a single byte slice. -func (w *Writer) BuildBytes() ([]byte, error) { +// BuildBytes returns writer data as a single byte slice. You can optionally provide one byte slice +// as argument that it will try to reuse. +func (w *Writer) BuildBytes(reuse ...[]byte) ([]byte, error) { if w.Error != nil { return nil, w.Error } - return w.Buffer.BuildBytes(), nil + return w.Buffer.BuildBytes(reuse...), nil +} + +// ReadCloser returns an io.ReadCloser that can be used to read the data. +// ReadCloser also resets the buffer. +func (w *Writer) ReadCloser() (io.ReadCloser, error) { + if w.Error != nil { + return nil, w.Error + } + + return w.Buffer.ReadCloser(), nil } // RawByte appends raw binary data to the buffer. @@ -44,7 +67,7 @@ func (w *Writer) RawString(s string) { w.Buffer.AppendString(s) } -// RawByte appends raw binary data to the buffer or sets the error if it is given. Useful for +// Raw appends raw binary data to the buffer or sets the error if it is given. Useful for // calling with results of MarshalJSON-like functions. func (w *Writer) Raw(data []byte, err error) { switch { @@ -59,6 +82,32 @@ func (w *Writer) Raw(data []byte, err error) { } } +// RawText encloses raw binary data in quotes and appends in to the buffer. +// Useful for calling with results of MarshalText-like functions. +func (w *Writer) RawText(data []byte, err error) { + switch { + case w.Error != nil: + return + case err != nil: + w.Error = err + case len(data) > 0: + w.String(string(data)) + default: + w.RawString("null") + } +} + +// Base64Bytes appends data to the buffer after base64 encoding it +func (w *Writer) Base64Bytes(data []byte) { + if data == nil { + w.Buffer.AppendString("null") + return + } + w.Buffer.AppendByte('"') + w.base64(data) + w.Buffer.AppendByte('"') +} + func (w *Writer) Uint8(n uint8) { w.Buffer.EnsureSpace(3) w.Buffer.Buf = strconv.AppendUint(w.Buffer.Buf, uint64(n), 10) @@ -144,6 +193,13 @@ func (w *Writer) Uint64Str(n uint64) { w.Buffer.Buf = append(w.Buffer.Buf, '"') } +func (w *Writer) UintptrStr(n uintptr) { + w.Buffer.EnsureSpace(20) + w.Buffer.Buf = append(w.Buffer.Buf, '"') + w.Buffer.Buf = strconv.AppendUint(w.Buffer.Buf, uint64(n), 10) + w.Buffer.Buf = append(w.Buffer.Buf, '"') +} + func (w *Writer) Int8Str(n int8) { w.Buffer.EnsureSpace(4) w.Buffer.Buf = append(w.Buffer.Buf, '"') @@ -184,11 +240,25 @@ func (w *Writer) Float32(n float32) { w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 32) } +func (w *Writer) Float32Str(n float32) { + w.Buffer.EnsureSpace(20) + w.Buffer.Buf = append(w.Buffer.Buf, '"') + w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 32) + w.Buffer.Buf = append(w.Buffer.Buf, '"') +} + func (w *Writer) Float64(n float64) { w.Buffer.EnsureSpace(20) w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, n, 'g', -1, 64) } +func (w *Writer) Float64Str(n float64) { + w.Buffer.EnsureSpace(20) + w.Buffer.Buf = append(w.Buffer.Buf, '"') + w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 64) + w.Buffer.Buf = append(w.Buffer.Buf, '"') +} + func (w *Writer) Bool(v bool) { w.Buffer.EnsureSpace(5) if v { @@ -200,6 +270,25 @@ func (w *Writer) Bool(v bool) { const chars = "0123456789abcdef" +func getTable(falseValues ...int) [128]bool { + table := [128]bool{} + + for i := 0; i < 128; i++ { + table[i] = true + } + + for _, v := range falseValues { + table[v] = false + } + + return table +} + +var ( + htmlEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '&', '<', '>', '\\') + htmlNoEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '\\') +) + func (w *Writer) String(s string) { w.Buffer.AppendByte('"') @@ -208,40 +297,41 @@ func (w *Writer) String(s string) { p := 0 // last non-escape symbol + var escapeTable [128]bool + if w.NoEscapeHTML { + escapeTable = htmlNoEscapeTable + } else { + escapeTable = htmlEscapeTable + } + for i := 0; i < len(s); { - // single-with character - if c := s[i]; c < utf8.RuneSelf { - var escape byte + c := s[i] + + if c < utf8.RuneSelf { + if escapeTable[c] { + // single-width character, no escaping is required + i++ + continue + } + + w.Buffer.AppendString(s[p:i]) switch c { case '\t': - escape = 't' + w.Buffer.AppendString(`\t`) case '\r': - escape = 'r' + w.Buffer.AppendString(`\r`) case '\n': - escape = 'n' + w.Buffer.AppendString(`\n`) case '\\': - escape = '\\' + w.Buffer.AppendString(`\\`) case '"': - escape = '"' - case '<', '>': - // do nothing + w.Buffer.AppendString(`\"`) default: - if c >= 0x20 { - // no escaping is required - i++ - continue - } - } - if escape != 0 { - w.Buffer.AppendString(s[p:i]) - w.Buffer.AppendByte('\\') - w.Buffer.AppendByte(escape) - } else { - w.Buffer.AppendString(s[p:i]) w.Buffer.AppendString(`\u00`) w.Buffer.AppendByte(chars[c>>4]) w.Buffer.AppendByte(chars[c&0xf]) } + i++ p = i continue @@ -271,3 +361,47 @@ func (w *Writer) String(s string) { w.Buffer.AppendString(s[p:]) w.Buffer.AppendByte('"') } + +const encode = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" +const padChar = '=' + +func (w *Writer) base64(in []byte) { + + if len(in) == 0 { + return + } + + w.Buffer.EnsureSpace(((len(in)-1)/3 + 1) * 4) + + si := 0 + n := (len(in) / 3) * 3 + + for si < n { + // Convert 3x 8bit source bytes into 4 bytes + val := uint(in[si+0])<<16 | uint(in[si+1])<<8 | uint(in[si+2]) + + w.Buffer.Buf = append(w.Buffer.Buf, encode[val>>18&0x3F], encode[val>>12&0x3F], encode[val>>6&0x3F], encode[val&0x3F]) + + si += 3 + } + + remain := len(in) - si + if remain == 0 { + return + } + + // Add the remaining small block + val := uint(in[si+0]) << 16 + if remain == 2 { + val |= uint(in[si+1]) << 8 + } + + w.Buffer.Buf = append(w.Buffer.Buf, encode[val>>18&0x3F], encode[val>>12&0x3F]) + + switch remain { + case 2: + w.Buffer.Buf = append(w.Buffer.Buf, encode[val>>6&0x3F], byte(padChar)) + case 1: + w.Buffer.Buf = append(w.Buffer.Buf, byte(padChar), byte(padChar)) + } +} diff --git a/vendor/github.com/mailru/easyjson/raw.go b/vendor/github.com/mailru/easyjson/raw.go index 737c73a88c6f..81bd002e19f8 100644 --- a/vendor/github.com/mailru/easyjson/raw.go +++ b/vendor/github.com/mailru/easyjson/raw.go @@ -5,8 +5,8 @@ import ( "github.com/mailru/easyjson/jwriter" ) -// RawMessage is a raw piece of JSON (number, string, bool, object, array or null) that is extracted -// without parsing and output as is during marshalling. +// RawMessage is a raw piece of JSON (number, string, bool, object, array or +// null) that is extracted without parsing and output as is during marshaling. type RawMessage []byte // MarshalEasyJSON does JSON marshaling using easyjson interface. diff --git a/vendor/github.com/mailru/easyjson/unknown_fields.go b/vendor/github.com/mailru/easyjson/unknown_fields.go new file mode 100644 index 000000000000..6cfdf8300ba0 --- /dev/null +++ b/vendor/github.com/mailru/easyjson/unknown_fields.go @@ -0,0 +1,34 @@ +package easyjson + +import ( + json "encoding/json" + + jlexer "github.com/mailru/easyjson/jlexer" + "github.com/mailru/easyjson/jwriter" +) + +// UnknownFieldsProxy implemets UnknownsUnmarshaler and UnknownsMarshaler +// use it as embedded field in your structure to parse and then serialize unknown struct fields +type UnknownFieldsProxy struct { + unknownFields map[string]interface{} +} + +func (s *UnknownFieldsProxy) UnmarshalUnknown(in *jlexer.Lexer, key string) { + if s.unknownFields == nil { + s.unknownFields = make(map[string]interface{}, 1) + } + s.unknownFields[key] = in.Interface() +} + +func (s UnknownFieldsProxy) MarshalUnknowns(out *jwriter.Writer, first bool) { + for key, val := range s.unknownFields { + if first { + first = false + } else { + out.RawByte(',') + } + out.String(string(key)) + out.RawByte(':') + out.Raw(json.Marshal(val)) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 754cb972caae..5167340ec7be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -623,7 +623,7 @@ github.com/magefile/mage/mg github.com/magefile/mage/parse github.com/magefile/mage/sh github.com/magefile/mage/target -# github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a +# github.com/mailru/easyjson v0.7.1 github.com/mailru/easyjson github.com/mailru/easyjson/buffer github.com/mailru/easyjson/jlexer diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index fac92de2f347..4bfe1be46591 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -52,8 +52,6 @@ type stackdriverMetricsRequester struct { func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out []*monitoringpb.TimeSeries) { out = make([]*monitoringpb.TimeSeries, 0) - filter := r.getFilterForMetric(m) - req := &monitoringpb.ListTimeSeriesRequest{ Name: "projects/" + r.config.ProjectID, Interval: r.interval,