From a9b7a05f20e6a1e900319fba4e6f037ba956a9e8 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Wed, 12 Jan 2022 13:39:01 +0300 Subject: [PATCH 1/4] feat: improve null decoding --- dec.go | 16 ++++++++++++++++ dec_skip.go | 27 +++++++++++++++++++++++++-- null_test.go | 22 ++++++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/dec.go b/dec.go index 4388fde..591bf6e 100644 --- a/dec.go +++ b/dec.go @@ -246,6 +246,22 @@ func (d *Decoder) read() error { return nil } +func (d *Decoder) readAtLeast(min int) error { + if d.reader == nil { + d.head = d.tail + return io.ErrUnexpectedEOF + } + + n, err := io.ReadAtLeast(d.reader, d.buf, min) + if err != nil { + return err + } + + d.head = 0 + d.tail = n + return nil +} + func (d *Decoder) unread() { d.head-- } // limit maximum depth of nesting, as allowed by https://tools.ietf.org/html/rfc7159#section-9 diff --git a/dec_skip.go b/dec_skip.go index 6ba4686..b113a3c 100644 --- a/dec_skip.go +++ b/dec_skip.go @@ -2,6 +2,7 @@ package jx import ( "io" + "math/bits" "github.com/go-faster/errors" ) @@ -9,10 +10,32 @@ import ( // Null reads a json object as null and // returns whether it's a null or not. func (d *Decoder) Null() error { - if err := d.consume('n'); err != nil { + const encodedNull = 'n' | 'u'<<8 | 'l'<<16 | 'l'<<24 + + if buf := d.buf[d.head:d.tail]; len(buf) >= 4 { + c := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + if mask := c ^ encodedNull; mask != 0 { + idx := bits.TrailingZeros32(mask) / 8 + return badToken(buf[idx]) + } + d.head += 4 + return nil + } + + var buf [4]byte + n := copy(buf[:], d.buf[d.head:d.tail]) + if err := d.readAtLeast(4 - n); err != nil { return err } - return d.skipThreeBytes('u', 'l', 'l') // null + copy(buf[n:], d.buf[d.head:d.tail]) + + c := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + if mask := c ^ encodedNull; mask != 0 { + idx := bits.TrailingZeros32(mask) / 8 + return badToken(buf[idx]) + } + d.head += 4 + return nil } // Bool reads a json object as Bool diff --git a/null_test.go b/null_test.go index 641bde5..73bbf0a 100644 --- a/null_test.go +++ b/null_test.go @@ -44,3 +44,25 @@ func Test_decode_null_skip(t *testing.T) { t.FailNow() } } + +func TestNullError(t *testing.T) { + a := require.New(t) + var ( + b = [4]byte{'n', 'u', 'l', 'l'} + valid = b + ) + for i := range b { + // Reset buffer. + b = valid + for c := byte(0); c < 255; c++ { + // Skip expected value. + if valid[i] == c { + continue + } + b[i] = c + var token badTokenErr + a.ErrorAs(DecodeBytes(b[:]).Null(), &token) + a.Equalf(c, token.Token, "%c != %c (%q)", c, token.Token, b) + } + } +} From 1082630e5cfe49db70340d2f9f0417c389b54414 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Fri, 14 Jan 2022 13:08:31 +0300 Subject: [PATCH 2/4] fix: return ErrUnexpectedEOF if ReadAtLeast return EOF --- dec.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dec.go b/dec.go index 591bf6e..0dc606a 100644 --- a/dec.go +++ b/dec.go @@ -254,6 +254,9 @@ func (d *Decoder) readAtLeast(min int) error { n, err := io.ReadAtLeast(d.reader, d.buf, min) if err != nil { + if err == io.EOF && n == 0 { + return io.ErrUnexpectedEOF + } return err } From 971965f3992078607698feab9310f4dfd7c62e91 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Fri, 14 Jan 2022 13:37:43 +0300 Subject: [PATCH 3/4] feat: improve true/false reading --- dec.go | 3 ++ dec_skip.go | 109 +++++++++++++++++++++++----------------------------- 2 files changed, 52 insertions(+), 60 deletions(-) diff --git a/dec.go b/dec.go index 0dc606a..857e01e 100644 --- a/dec.go +++ b/dec.go @@ -252,6 +252,9 @@ func (d *Decoder) readAtLeast(min int) error { return io.ErrUnexpectedEOF } + if need := min - len(d.buf); need > 0 { + d.buf = append(d.buf, make([]byte, need)...) + } n, err := io.ReadAtLeast(d.reader, d.buf, min) if err != nil { if err == io.EOF && n == 0 { diff --git a/dec_skip.go b/dec_skip.go index b113a3c..89c9518 100644 --- a/dec_skip.go +++ b/dec_skip.go @@ -7,53 +7,67 @@ import ( "github.com/go-faster/errors" ) -// Null reads a json object as null and -// returns whether it's a null or not. -func (d *Decoder) Null() error { - const encodedNull = 'n' | 'u'<<8 | 'l'<<16 | 'l'<<24 - - if buf := d.buf[d.head:d.tail]; len(buf) >= 4 { - c := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - if mask := c ^ encodedNull; mask != 0 { - idx := bits.TrailingZeros32(mask) / 8 - return badToken(buf[idx]) - } - d.head += 4 +func (d *Decoder) readExact4(b *[4]byte) error { + if buf := d.buf[d.head:d.tail]; len(buf) >= len(b) { + d.head += copy(b[:], buf[:4]) return nil } - var buf [4]byte - n := copy(buf[:], d.buf[d.head:d.tail]) - if err := d.readAtLeast(4 - n); err != nil { + n := copy(b[:], d.buf[d.head:d.tail]) + if err := d.readAtLeast(len(b) - n); err != nil { return err } - copy(buf[n:], d.buf[d.head:d.tail]) + d.head += copy(b[n:], d.buf[d.head:d.tail]) + return nil +} +func findInvalidToken4(buf [4]byte, mask uint32) error { c := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - if mask := c ^ encodedNull; mask != 0 { - idx := bits.TrailingZeros32(mask) / 8 - return badToken(buf[idx]) + idx := bits.TrailingZeros32(c^mask) / 8 + return badToken(buf[idx]) +} + +// Null reads a json object as null and +// returns whether it's a null or not. +func (d *Decoder) Null() error { + var buf [4]byte + if err := d.readExact4(&buf); err != nil { + return err + } + + if string(buf[:]) != "null" { + const encodedNull = 'n' | 'u'<<8 | 'l'<<16 | 'l'<<24 + return findInvalidToken4(buf, encodedNull) } - d.head += 4 return nil } // Bool reads a json object as Bool func (d *Decoder) Bool() (bool, error) { - c, err := d.next() - if err != nil { + var buf [4]byte + if err := d.readExact4(&buf); err != nil { return false, err } - switch c { - case 't': - if err := d.skipThreeBytes('r', 'u', 'e'); err != nil { + + switch string(buf[:]) { + case "true": + return true, nil + case "fals": + if err := d.consume('e'); err != nil { return false, err } - return true, nil - case 'f': - return false, d.skipFourBytes('a', 'l', 's', 'e') + return false, nil default: - return false, badToken(c) + switch c := buf[0]; c { + case 't': + const encodedTrue = 't' | 'r'<<8 | 'u'<<16 | 'e'<<24 + return false, findInvalidToken4(buf, encodedTrue) + case 'f': + const encodedAlse = 'a' | 'l'<<8 | 's'<<16 | 'e'<<24 + return false, findInvalidToken4(buf, encodedAlse) + default: + return false, badToken(c) + } } } @@ -70,11 +84,12 @@ func (d *Decoder) Skip() error { } return nil case 'n': - return d.skipThreeBytes('u', 'l', 'l') // null - case 't': - return d.skipThreeBytes('r', 'u', 'e') // true - case 'f': - return d.skipFourBytes('a', 'l', 's', 'e') // false + d.unread() + return d.Null() + case 't', 'f': + d.unread() + _, err := d.Bool() + return err case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': d.unread() return d.skipNumber() @@ -93,32 +108,6 @@ func (d *Decoder) Skip() error { } } -func (d *Decoder) skipFourBytes(b1, b2, b3, b4 byte) error { - for _, b := range [...]byte{b1, b2, b3, b4} { - c, err := d.byte() - if err != nil { - return err - } - if c != b { - return badToken(c) - } - } - return nil -} - -func (d *Decoder) skipThreeBytes(b1, b2, b3 byte) error { - for _, b := range [...]byte{b1, b2, b3} { - c, err := d.byte() - if err != nil { - return err - } - if c != b { - return badToken(c) - } - } - return nil -} - var ( skipNumberSet = [256]byte{ '0': 1, From cbdf6d5c5a504761ae7bb887e6b4a68b77d54572 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Fri, 14 Jan 2022 13:50:22 +0300 Subject: [PATCH 4/4] chore: add bools.json and nulls.json --- testdata/bools.json | 103 ++++++++++++++++++++++++++++++++++++++++++++ testdata/nulls.json | 103 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 testdata/bools.json create mode 100644 testdata/nulls.json diff --git a/testdata/bools.json b/testdata/bools.json new file mode 100644 index 0000000..67158fe --- /dev/null +++ b/testdata/bools.json @@ -0,0 +1,103 @@ +[ + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false, + true, + false +] + diff --git a/testdata/nulls.json b/testdata/nulls.json new file mode 100644 index 0000000..df506aa --- /dev/null +++ b/testdata/nulls.json @@ -0,0 +1,103 @@ +[ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null +] +