From ae8ccf5823b022dec70a0eed0bfa446d04d96712 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 21 Oct 2020 09:29:02 +0200 Subject: [PATCH 1/4] core/types, rlp: optimize derivesha --- core/types/derive_sha.go | 24 ++++++++++++------------ rlp/encode.go | 13 +++++++++++++ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go index 51b8506bce43..7fdaa00cf6d2 100644 --- a/core/types/derive_sha.go +++ b/core/types/derive_sha.go @@ -17,8 +17,6 @@ package types import ( - "bytes" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) @@ -37,26 +35,28 @@ type Hasher interface { func DeriveSha(list DerivableList, hasher Hasher) common.Hash { hasher.Reset() - keybuf := new(bytes.Buffer) // StackTrie requires values to be inserted in increasing // hash order, which is not the order that `list` provides // hashes in. This insertion sequence ensures that the // order is correct. + + buf := make([]byte, 9) // 9 bytes is the max an rlp-encoded int will ever use + for i := 1; i < list.Len() && i <= 0x7f; i++ { - keybuf.Reset() - rlp.Encode(keybuf, uint(i)) - hasher.Update(keybuf.Bytes(), list.GetRlp(i)) + off := rlp.PutInt(buf, i) + hasher.Update(buf[:off], list.GetRlp(i)) + } if list.Len() > 0 { - keybuf.Reset() - rlp.Encode(keybuf, uint(0)) - hasher.Update(keybuf.Bytes(), list.GetRlp(0)) + off := rlp.PutInt(buf, 0) + hasher.Update(buf[:off], list.GetRlp(0)) + } for i := 0x80; i < list.Len(); i++ { - keybuf.Reset() - rlp.Encode(keybuf, uint(i)) - hasher.Update(keybuf.Bytes(), list.GetRlp(i)) + off := rlp.PutInt(buf, i) + hasher.Update(buf[:off], list.GetRlp(i)) + } return hasher.Hash() } diff --git a/rlp/encode.go b/rlp/encode.go index 77b591045d2f..f245995fd15e 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -604,6 +604,19 @@ func makeEncoderWriter(typ reflect.Type) writer { return w } +func PutInt(b []byte, i int) (size int) { + if i == 0 { + b[0] = 0x80 + return 1 + } else if i < 128 { + b[0] = byte(i) + return 1 + } + s := putint(b[1:], uint64(i)) + b[0] = 0x80 + byte(s) + return s + 1 +} + // putint writes i to the beginning of b in big endian byte // order, using the least number of bytes needed to represent i. func putint(b []byte, i uint64) (size int) { From 77fd6def3c1b3612cb0d1c28ad9872a3a1da2765 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 30 Oct 2020 09:29:18 +0100 Subject: [PATCH 2/4] core/types, rlp: improve RLP PutInt handling --- core/types/derive_sha.go | 16 ++++----- rlp/encode.go | 75 +++++++++++++++++++++++++++++++++++----- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go index 7fdaa00cf6d2..2588392d531b 100644 --- a/core/types/derive_sha.go +++ b/core/types/derive_sha.go @@ -41,22 +41,20 @@ func DeriveSha(list DerivableList, hasher Hasher) common.Hash { // hashes in. This insertion sequence ensures that the // order is correct. - buf := make([]byte, 9) // 9 bytes is the max an rlp-encoded int will ever use - + var buf []byte for i := 1; i < list.Len() && i <= 0x7f; i++ { - off := rlp.PutInt(buf, i) - hasher.Update(buf[:off], list.GetRlp(i)) + buf = rlp.AppendInt(buf[:0], uint64(i)) + hasher.Update(buf, list.GetRlp(i)) } if list.Len() > 0 { - off := rlp.PutInt(buf, 0) - hasher.Update(buf[:off], list.GetRlp(0)) + buf = rlp.AppendInt(buf[:0], 0) + hasher.Update(buf, list.GetRlp(0)) } for i := 0x80; i < list.Len(); i++ { - off := rlp.PutInt(buf, i) - hasher.Update(buf[:off], list.GetRlp(i)) - + buf = rlp.AppendInt(buf[:0], uint64(i)) + hasher.Update(buf, list.GetRlp(i)) } return hasher.Hash() } diff --git a/rlp/encode.go b/rlp/encode.go index f245995fd15e..f03d0129845f 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -604,17 +604,76 @@ func makeEncoderWriter(typ reflect.Type) writer { return w } -func PutInt(b []byte, i int) (size int) { +// AppendInt appends the rlp-encoding of i to b, and returns the +// resulting slice +func AppendInt(b []byte, i uint64) []byte { if i == 0 { - b[0] = 0x80 - return 1 + return append(b, 0x80) } else if i < 128 { - b[0] = byte(i) - return 1 + return append(b, byte(i)) + } + switch { + case i < (1 << 8): + return append(b, 0x81, byte(i)) + case i < (1 << 16): + return append(b, 0x82, + byte(i>>8), + byte(i), + ) + case i < (1 << 24): + return append(b, 0x83, + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 32): + return append(b, 0x84, + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 40): + return append(b, 0x85, + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + case i < (1 << 48): + return append(b, 0x86, + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 56): + return append(b, 0x87, + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + default: + return append(b, 0x88, + byte(i>>56), + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) } - s := putint(b[1:], uint64(i)) - b[0] = 0x80 + byte(s) - return s + 1 } // putint writes i to the beginning of b in big endian byte From e4bacf96341b6f5687e1fa0b688a336414c9df8e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 4 Nov 2020 17:14:29 +0100 Subject: [PATCH 3/4] rlp: rename AppendInt -> AppendUint64 and add tests --- rlp/encode.go | 72 ------------------------------------------------- rlp/raw.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ rlp/raw_test.go | 38 ++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 72 deletions(-) diff --git a/rlp/encode.go b/rlp/encode.go index f03d0129845f..77b591045d2f 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -604,78 +604,6 @@ func makeEncoderWriter(typ reflect.Type) writer { return w } -// AppendInt appends the rlp-encoding of i to b, and returns the -// resulting slice -func AppendInt(b []byte, i uint64) []byte { - if i == 0 { - return append(b, 0x80) - } else if i < 128 { - return append(b, byte(i)) - } - switch { - case i < (1 << 8): - return append(b, 0x81, byte(i)) - case i < (1 << 16): - return append(b, 0x82, - byte(i>>8), - byte(i), - ) - case i < (1 << 24): - return append(b, 0x83, - byte(i>>16), - byte(i>>8), - byte(i), - ) - case i < (1 << 32): - return append(b, 0x84, - byte(i>>24), - byte(i>>16), - byte(i>>8), - byte(i), - ) - case i < (1 << 40): - return append(b, 0x85, - byte(i>>32), - byte(i>>24), - byte(i>>16), - byte(i>>8), - byte(i), - ) - - case i < (1 << 48): - return append(b, 0x86, - byte(i>>40), - byte(i>>32), - byte(i>>24), - byte(i>>16), - byte(i>>8), - byte(i), - ) - case i < (1 << 56): - return append(b, 0x87, - byte(i>>48), - byte(i>>40), - byte(i>>32), - byte(i>>24), - byte(i>>16), - byte(i>>8), - byte(i), - ) - - default: - return append(b, 0x88, - byte(i>>56), - byte(i>>48), - byte(i>>40), - byte(i>>32), - byte(i>>24), - byte(i>>16), - byte(i>>8), - byte(i), - ) - } -} - // putint writes i to the beginning of b in big endian byte // order, using the least number of bytes needed to represent i. func putint(b []byte, i uint64) (size int) { diff --git a/rlp/raw.go b/rlp/raw.go index c2a8517f62f1..3071e99cab1b 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -180,3 +180,74 @@ func readSize(b []byte, slen byte) (uint64, error) { } return s, nil } + +// AppendUint64 appends the RLP encoding of i to b, and returns the resulting slice. +func AppendUint64(b []byte, i uint64) []byte { + if i == 0 { + return append(b, 0x80) + } else if i < 128 { + return append(b, byte(i)) + } + switch { + case i < (1 << 8): + return append(b, 0x81, byte(i)) + case i < (1 << 16): + return append(b, 0x82, + byte(i>>8), + byte(i), + ) + case i < (1 << 24): + return append(b, 0x83, + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 32): + return append(b, 0x84, + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 40): + return append(b, 0x85, + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + case i < (1 << 48): + return append(b, 0x86, + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + case i < (1 << 56): + return append(b, 0x87, + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + + default: + return append(b, 0x88, + byte(i>>56), + byte(i>>48), + byte(i>>40), + byte(i>>32), + byte(i>>24), + byte(i>>16), + byte(i>>8), + byte(i), + ) + } +} diff --git a/rlp/raw_test.go b/rlp/raw_test.go index cdae4ff08859..c976c4f73429 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -21,6 +21,7 @@ import ( "io" "reflect" "testing" + "testing/quick" ) func TestCountValues(t *testing.T) { @@ -239,3 +240,40 @@ func TestReadSize(t *testing.T) { } } } + +func TestAppendUint64(t *testing.T) { + tests := []struct { + input uint64 + slice []byte + output string + }{ + {0, nil, "80"}, + {1, nil, "01"}, + {2, nil, "02"}, + {127, nil, "7F"}, + {128, nil, "8180"}, + {129, nil, "8181"}, + {0xFFFFFF, nil, "83FFFFFF"}, + {127, []byte{1, 2, 3}, "0102037F"}, + {0xFFFFFF, []byte{1, 2, 3}, "01020383FFFFFF"}, + } + + for _, test := range tests { + x := AppendUint64(test.slice, test.input) + if !bytes.Equal(x, unhex(test.output)) { + t.Errorf("AppendUint64(%v, %d): got %x, want %s", test.slice, test.input, x, test.output) + } + } +} + +func TestAppendUint64Random(t *testing.T) { + fn := func(i uint64) bool { + enc, _ := EncodeToBytes(i) + encAppend := AppendUint64(nil, i) + return bytes.Equal(enc, encAppend) + } + config := quick.Config{MaxCountScale: 50} + if err := quick.Check(fn, &config); err != nil { + t.Fatal(err) + } +} From 61e442efc16c987ae0932301493da0042e2ee8e2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 4 Nov 2020 17:15:48 +0100 Subject: [PATCH 4/4] core/types: update function name --- core/types/derive_sha.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/types/derive_sha.go b/core/types/derive_sha.go index 2588392d531b..51a10f3f3da7 100644 --- a/core/types/derive_sha.go +++ b/core/types/derive_sha.go @@ -43,17 +43,15 @@ func DeriveSha(list DerivableList, hasher Hasher) common.Hash { var buf []byte for i := 1; i < list.Len() && i <= 0x7f; i++ { - buf = rlp.AppendInt(buf[:0], uint64(i)) + buf = rlp.AppendUint64(buf[:0], uint64(i)) hasher.Update(buf, list.GetRlp(i)) - } if list.Len() > 0 { - buf = rlp.AppendInt(buf[:0], 0) + buf = rlp.AppendUint64(buf[:0], 0) hasher.Update(buf, list.GetRlp(0)) - } for i := 0x80; i < list.Len(); i++ { - buf = rlp.AppendInt(buf[:0], uint64(i)) + buf = rlp.AppendUint64(buf[:0], uint64(i)) hasher.Update(buf, list.GetRlp(i)) } return hasher.Hash()