Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fast Cache - Downgrade - reupgrade protection and other improvements #12

Merged
merged 27 commits into from
Feb 7, 2022
Merged
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
6350dc8
add leaf hash to fast node and unit test
p0mvn Jan 26, 2022
5f579c5
refactor get with index and get by index, fix migration in load versi…
p0mvn Jan 26, 2022
f6ebabb
use Get in GetVersioned of mutable tree
p0mvn Jan 27, 2022
89633f7
refactor non membership proof to use fast storage if available
p0mvn Jan 27, 2022
75d9f81
bench non-membership proof
p0mvn Jan 27, 2022
19ee7d3
fix bench tests to work with the new changes
p0mvn Jan 28, 2022
cbf2f41
add downgrade-reupgrade protection and unit test
p0mvn Jan 28, 2022
96208dc
remove leaf hash from fast node
p0mvn Jan 28, 2022
5217e71
resolve multithreading bug related to iterators not being closed
p0mvn Jan 28, 2022
ea072a6
clean up
p0mvn Jan 28, 2022
806b1a0
use correct tree in bench tests
p0mvn Jan 29, 2022
3f24e71
add cache to tree used to bench non membership proofs
p0mvn Jan 29, 2022
0cb480e
add benc tests for GetWithIndex and GetByIndex
p0mvn Jan 29, 2022
ee1ab84
revert GetWithIndex and GetByIndex
p0mvn Jan 29, 2022
2edb8cc
remove unused import
p0mvn Jan 29, 2022
9712e87
unit test re-upgrade protection and fix small issues
p0mvn Jan 30, 2022
eea8cfd
remove redundant setStorageVersion method
p0mvn Jan 31, 2022
cd0a61c
fix bug with appending to live stage version to storage version and n…
p0mvn Jan 31, 2022
8586d10
add comment for setFastStorageVersionToBatch
p0mvn Jan 31, 2022
c293938
refactor and improve unit tests for reupgrade protection
p0mvn Jan 31, 2022
eae9425
rename ndb's isFastStorageEnabled to hasUpgradedToFastStorage and add…
p0mvn Jan 31, 2022
2241204
comment out new implementation for GetNonMembershipProof
p0mvn Jan 31, 2022
31710b3
update comments in nodedb to reflect the difference between hasUpgrad…
p0mvn Jan 31, 2022
c0e4776
refactor nodedb tests
p0mvn Jan 31, 2022
51b6c2f
downgrade tendermint to 0.34.14 - osmosis's latest cosmos sdk does no…
p0mvn Feb 3, 2022
a7e7fb2
fix bug where fast storage was not enabled when version 0 was attempt…
p0mvn Feb 7, 2022
f78896b
implement unsaved fast iterator to be used in mutable tree (#16)
p0mvn Feb 7, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor non membership proof to use fast storage if available
p0mvn committed Jan 27, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 89633f7e8352517162774d0fccb3d9361a7480a8
79 changes: 73 additions & 6 deletions proof_ics23.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package iavl

import (
"bytes"
"encoding/binary"
"fmt"

@@ -28,7 +29,29 @@ func (t *ImmutableTree) GetMembershipProof(key []byte) (*ics23.CommitmentProof,
GetNonMembershipProof will produce a CommitmentProof that the given key doesn't exist in the iavl tree.
If the key exists in the tree, this will return an error.
*/
func (t *ImmutableTree) GetNonMembershipProof(key []byte) (*ics23.CommitmentProof, error) {
func (t *ImmutableTree) GetNonMembershipProof(key []byte) (proof *ics23.CommitmentProof, err error) {
var nonexist *ics23.NonExistenceProof
if t.IsFastCacheEnabled() {
nonexist, err = t.getNonMembershipProofFast(key)
} else {
nonexist, err = t.getNonMembershipProof(key)
}

if err != nil {
return nil, err
}

proof = &ics23.CommitmentProof{
Proof: &ics23.CommitmentProof_Nonexist{
Nonexist: nonexist,
},
}
return proof, nil
}

// getNonMembershipProof using regular strategy
// invariant: fast storage is enabled
func (t *ImmutableTree) getNonMembershipProof(key []byte) (*ics23.NonExistenceProof, error) {
// idx is one node right of what we want....
idx, val := t.GetWithIndex(key)
if val != nil {
@@ -57,12 +80,56 @@ func (t *ImmutableTree) GetNonMembershipProof(key []byte) (*ics23.CommitmentProo
}
}

proof := &ics23.CommitmentProof{
Proof: &ics23.CommitmentProof_Nonexist{
Nonexist: nonexist,
},
return nonexist, nil
}

// getNonMembershipProofFast using fast storage
// invariant: fast storage is enabled
func (t *ImmutableTree) getNonMembershipProofFast(key []byte) (*ics23.NonExistenceProof, error) {
index := 0
var prevKey []byte = nil
var nextKey []byte = nil

done := false
itr := t.Iterator(nil, nil, true)
Copy link
Member

@ValarDragon ValarDragon Jan 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hrmm, isn't iterating over all of fast node state going to be slower than a logarithmic number of file opens? Is there a way we can set a 'smarter' start bound?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can do this speedup in a non-db breaking way later, so may also be worth commenting out its invocation in GetNonMembershipProof and revisiting after we get the initial release out!

for ; !done && itr.Valid(); itr.Next() {
switch bytes.Compare(itr.Key(), key) {
case -1:
index++
prevKey = itr.Key()
case 1:
nextKey = itr.Key()
done = true
default:
done = true
}
}
return proof, nil

// If next was not set, that means we found the key during iterations above
if done && nextKey == nil {
return nil, fmt.Errorf("cannot create NonExistanceProof when Key in State")
}

var err error
nonexist := &ics23.NonExistenceProof{
Key: key,
}

if prevKey != nil {
nonexist.Left, err = createExistenceProof(t, prevKey)
if err != nil {
return nil, err
}
}

if nextKey != nil {
nonexist.Right, err = createExistenceProof(t, nextKey)
if err != nil {
return nil, err
}
}

return nonexist, nil
}

func createExistenceProof(tree *ImmutableTree, key []byte) (*ics23.ExistenceProof, error) {
44 changes: 33 additions & 11 deletions proof_ics23_test.go
Original file line number Diff line number Diff line change
@@ -72,22 +72,40 @@ func TestGetNonMembership(t *testing.T) {
"big right": {size: 5431, loc: Right},
}

performTest := func (tree *MutableTree, allKeys [][]byte, loc Where) {
key := GetNonKey(allKeys, loc)

proof, err := tree.GetNonMembershipProof(key)
require.NoError(t, err, "Creating Proof: %+v", err)

root := tree.Hash()
valid := ics23.VerifyNonMembership(ics23.IavlSpec, root, proof, key)
if !valid {
require.NoError(t, err, "Non Membership Proof Invalid")
}
}

for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
t.Run("fast-" + name, func (t *testing.T) {
tree, allkeys, err := BuildTree(tc.size)
require.NoError(t, err, "Creating tree: %+v", err)
// Save version to enable fast cache
_, _, err = tree.SaveVersion()
require.NoError(t, err)

key := GetNonKey(allkeys, tc.loc)
require.True(t, tree.IsFastCacheEnabled())

proof, err := tree.GetNonMembershipProof(key)
require.NoError(t, err, "Creating Proof: %+v", err)
performTest(tree, allkeys, tc.loc)
})

root := tree.Hash()
valid := ics23.VerifyNonMembership(ics23.IavlSpec, root, proof, key)
if !valid {
require.NoError(t, err, "Non Membership Proof Invalid")
}
t.Run("regular-" + name, func (t *testing.T) {
tree, allkeys, err := BuildTree(tc.size)
require.NoError(t, err, "Creating tree: %+v", err)
require.False(t, tree.IsFastCacheEnabled())


performTest(tree, allkeys, tc.loc)
})
}
}
@@ -110,6 +128,10 @@ func GenerateResult(size int, loc Where) (*Result, error) {
if err != nil {
return nil, err
}
_, _, err = tree.SaveVersion()
if err != nil {
return nil, err
}
key := GetKey(allkeys, loc)

value, proof, err := tree.GetWithProof(key)
@@ -173,7 +195,7 @@ func GetNonKey(allkeys [][]byte, loc Where) []byte {

// BuildTree creates random key/values and stores in tree
// returns a list of all keys in sorted order
func BuildTree(size int) (itree *ImmutableTree, keys [][]byte, err error) {
func BuildTree(size int) (itree *MutableTree, keys [][]byte, err error) {
tree, _ := NewMutableTree(db.NewMemDB(), 0)

// insert lots of info and store the bytes
@@ -191,5 +213,5 @@ func BuildTree(size int) (itree *ImmutableTree, keys [][]byte, err error) {
return bytes.Compare(keys[i], keys[j]) < 0
})

return tree.ImmutableTree, keys, nil
return tree, keys, nil
}