Fast Node Iteration (cosmos#7)

* propagate errors from nodedb and avoid panicking

* begin implementing fast node iteration

* resolve rebase issue in random tests

* fix iteration to deserialize fast node for extracting the correct value

* finalzie iteration and let all unit tests pass

* add benchmarks

* merge GetVersioned and GetVersionedFast

* remove fast node deletion from DeleteVersion and DeleteVersionRange and benchmark

* fix and unit test iteration on mutable and immutable trees

* implement tests for iterator and iterate, begin testing fast iterator

* fix and unit test fast iterator

benchmarks/bench_test.go

@@ -72,6 +72,24 @@ func runKnownQueries(b *testing.B, t *iavl.MutableTree, keys [][]byte) {
 	}
 }
 
+func runIteration(b *testing.B, t *iavl.MutableTree) {
+	itr := iavl.NewIterator(nil, nil, false, t.ImmutableTree)
+	defer itr.Close()
+
+	for i := 0; i < b.N && itr.Valid(); i++ {
+		itr.Next()
+	}
+}
+
+func runFastIteration(b *testing.B, t *iavl.MutableTree) {
+	itr := t.ImmutableTree.Iterator(nil, nil, false)
+	defer itr.Close()
+
+	for i := 0; i < b.N && itr.Valid(); i++ {
+		itr.Next()
+	}
+}
+
 // func runInsert(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int) *iavl.MutableTree {
 // 	for i := 1; i <= b.N; i++ {
 // 		t.Set(randBytes(keyLen), randBytes(dataLen))
@@ -146,6 +164,26 @@ func runBlock(b *testing.B, t *iavl.MutableTree, keyLen, dataLen, blockSize int,
 	return lastCommit
 }
 
+
+func BenchmarkIteration(b *testing.B) {
+	fmt.Printf("%s\n", iavl.GetVersionInfo())
+	benchmarks := []struct {
+		length int
+	}{
+		{4}, {16}, {32}, {100}, {1000},
+	}
+	for _, bench := range benchmarks {
+		bench := bench
+		name := fmt.Sprintf("random-%d", bench.length)
+		b.Run(name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				randBytes(bench.length)
+			}
+			runtime.GC()
+		})
+	}
+}
+
 func BenchmarkRandomBytes(b *testing.B) {
 	fmt.Printf("%s\n", iavl.GetVersionInfo())
 	benchmarks := []struct {
@@ -270,28 +308,37 @@ func runSuite(b *testing.B, d db.DB, initSize, blockSize, keyLen, dataLen int) {
 	runtime.GC()
 	init := memUseMB()
 
-	t, keys := prepareTree(b, d, initSize, keyLen, dataLen)
+	t, _ := prepareTree(b, d, initSize, keyLen, dataLen)
 	used := memUseMB() - init
 	fmt.Printf("Init Tree took %0.2f MB\n", used)
 
 	b.ResetTimer()
 
-	b.Run("query-no-in-tree-guarantee", func(sub *testing.B) {
-		sub.ReportAllocs()
-		runQueries(sub, t, keyLen)
-	})
-	b.Run("query-hits", func(sub *testing.B) {
-		sub.ReportAllocs()
-		runKnownQueries(sub, t, keys)
-	})
-	b.Run("update", func(sub *testing.B) {
+	// b.Run("query-no-in-tree-guarantee", func(sub *testing.B) {
+	// 	sub.ReportAllocs()
+	// 	runQueries(sub, t, keyLen)
+	// })
+	// b.Run("query-hits", func(sub *testing.B) {
+	// 	sub.ReportAllocs()
+	// 	runKnownQueries(sub, t, keys)
+	// })
+	b.Run("iteration", func(sub *testing.B) {
 		sub.ReportAllocs()
-		t = runUpdate(sub, t, dataLen, blockSize, keys)
+		runIteration(sub, t)
 	})
-	b.Run("block", func(sub *testing.B) {
+
+	b.Run("fast-iteration", func(sub *testing.B) {
 		sub.ReportAllocs()
-		t = runBlock(sub, t, keyLen, dataLen, blockSize, keys)
+		runFastIteration(sub, t)
 	})
+	// b.Run("update", func(sub *testing.B) {
+	// 	sub.ReportAllocs()
+	// 	t = runUpdate(sub, t, dataLen, blockSize, keys)
+	// })
+	// b.Run("block", func(sub *testing.B) {
+	// 	sub.ReportAllocs()
+	// 	t = runBlock(sub, t, keyLen, dataLen, blockSize, keys)
+	// })
 
 	// both of these edit size of the tree too much
 	// need to run with their own tree

fast_iterator.go

@@ -0,0 +1,132 @@
+package iavl
+
+import (
+	"errors"
+
+	dbm "github.com/tendermint/tm-db"
+)
+
+// Iterator is a dbm.Iterator for ImmutableTree
+type FastIterator struct {
+	start, end []byte
+
+	valid bool
+
+	ascending bool
+
+	err error
+
+	ndb *nodeDB
+
+	nextFastNode *FastNode
+
+	fastIterator dbm.Iterator
+}
+
+var _ dbm.Iterator = &FastIterator{}
+
+func NewFastIterator(start, end []byte, ascending bool, ndb *nodeDB) *FastIterator {
+	iter := &FastIterator{
+		start:       start,
+		end:         end,
+		err:         nil,
+		ascending:   ascending,
+		ndb:         ndb,
+		nextFastNode: nil,
+		fastIterator: nil,
+	}
+	iter.Next()
+	return iter
+}
+
+// Domain implements dbm.Iterator.
+func (iter *FastIterator) Domain() ([]byte, []byte) {
+	if iter.fastIterator == nil {
+		return iter.start, iter.end
+	}
+
+	start, end := iter.fastIterator.Domain()
+
+	if start != nil {
+		start = start[1:]
+		if len(start) == 0 {
+			start = nil
+		}
+	}
+
+	if end != nil {
+		end = end[1:]
+		if len(end) == 0 {
+			end = nil
+		}
+	}
+
+	return start, end
+}
+
+// Valid implements dbm.Iterator.
+func (iter *FastIterator) Valid() bool {
+	if iter.fastIterator == nil || !iter.fastIterator.Valid() {
+		return false
+	}
+
+	return iter.valid
+}
+
+// Key implements dbm.Iterator
+func (iter *FastIterator) Key() []byte {
+	if iter.valid {
+		return iter.nextFastNode.key
+	}
+	return nil
+}
+
+// Value implements dbm.Iterator
+func (iter *FastIterator) Value() []byte {
+	if iter.valid {
+		return iter.nextFastNode.value
+	}
+	return nil
+}
+
+// Next implements dbm.Iterator
+func (iter *FastIterator) Next() {
+	if iter.ndb == nil {
+		iter.err = errors.New("nodeDB is nil")
+		iter.valid = false
+		return
+	}
+
+	if iter.fastIterator == nil {
+		iter.fastIterator, iter.err = iter.ndb.getFastIterator(iter.start, iter.end, iter.ascending)
+		iter.valid = true
+	} else {
+		iter.fastIterator.Next()
+	}
+
+	if iter.err == nil {
+		iter.err = iter.fastIterator.Error()
+	}
+
+	iter.valid = iter.valid && iter.fastIterator.Valid()
+	if iter.valid {
+		iter.nextFastNode, iter.err = DeserializeFastNode(iter.fastIterator.Key()[1:], iter.fastIterator.Value())
+		iter.valid = iter.err == nil
+	}
+}
+
+// Close implements dbm.Iterator
+func (iter *FastIterator) Close() error {
+	iter.fastIterator = nil
+	iter.valid = false
+
+	if iter.fastIterator != nil {
+		iter.err = iter.fastIterator.Close()
+	}
+	return iter.err
+}
+
+// Error implements dbm.Iterator
+func (iter *FastIterator) Error() error {
+	return iter.err
+}

fast_node.go

@@ -25,13 +25,7 @@ func NewFastNode(key []byte, value []byte, version int64) *FastNode {
 }
 
 // DeserializeFastNode constructs an *FastNode from an encoded byte slice.
-func DeserializeFastNode(buf []byte) (*FastNode, error) {
-	key, n, cause := decodeBytes(buf)
-	if cause != nil {
-		return nil, errors.Wrap(cause, "decoding fastnode.key")
-	}
-	buf = buf[n:]
-
+func DeserializeFastNode(key []byte, buf []byte) (*FastNode, error) {
 	ver, n, cause := decodeVarint(buf)
 	if cause != nil {
 		return nil, errors.Wrap(cause, "decoding fastnode.version")
@@ -54,7 +48,6 @@ func DeserializeFastNode(buf []byte) (*FastNode, error) {
 
 func (node *FastNode) encodedSize() int {
 	n := 1 +
-		encodeBytesSize(node.key) +
 		encodeVarintSize(node.versionLastUpdatedAt) +
 		encodeBytesSize(node.value)
 	return n
@@ -65,11 +58,7 @@ func (node *FastNode) writeBytes(w io.Writer) error {
 	if node == nil {
 		return errors.New("cannot write nil node")
 	}
-	cause := encodeBytes(w, node.key)
-	if cause != nil {
-		return errors.Wrap(cause, "writing key")
-	}
-	cause = encodeVarint(w, node.versionLastUpdatedAt)
+	cause := encodeVarint(w, node.versionLastUpdatedAt)
 	if cause != nil {
 		return errors.Wrap(cause, "writing version last updated at")
 	}

immutable_tree.go

@@ -109,6 +109,11 @@ func (t *ImmutableTree) Version() int64 {
 	return t.version
 }
 
+// IsLatestVersion returns true if curren tree is of the latest version, false otherwise.
+func (t *ImmutableTree) IsLatestVersion() bool {
+	return t.version == t.ndb.getLatestVersion()
+}
+
 // Height returns the height of the tree.
 func (t *ImmutableTree) Height() int8 {
 	if t.root == nil {
@@ -211,12 +216,32 @@ func (t *ImmutableTree) GetByIndex(index int64) (key []byte, value []byte) {
 	return t.root.getByIndex(t, index)
 }
 
-// Iterate iterates over all keys of the tree, in order. The keys and values must not be modified,
+// Iterate iterates over all keys of the tree. The keys and values must not be modified,
 // since they may point to data stored within IAVL.
 func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) {
 	if t.root == nil {
 		return false
 	}
+
+	if t.version == t.ndb.getLatestVersion() {
+		stopped, err := t.ndb.traverseFastNodesWithStop(func(keyWithPrefix, v []byte) (bool, error) {
+			key := keyWithPrefix[1:]
+			fastNode, err := DeserializeFastNode(key, v)
+
+			if err != nil {
+				return false, err
+			}
+
+			return fn(fastNode.key, fastNode.value), nil
+		})
+
+		if err != nil {
+			panic(err)
+		}
+
+		return stopped
+	}
+
 	return t.root.traverse(t, true, func(node *Node) bool {
 		if node.height == 0 {
 			return fn(node.key, node.value)
@@ -225,6 +250,16 @@ func (t *ImmutableTree) Iterate(fn func(key []byte, value []byte) bool) (stopped
 	})
 }
 
+// Iterator returns an iterator over the immutable tree.
+func (t *ImmutableTree) Iterator(start, end []byte, ascending bool) dbm.Iterator {
+	isFastTraversal := t.IsLatestVersion()
+	if isFastTraversal {
+		return NewFastIterator(start, end, ascending, t.ndb)
+	} else {
+		return NewIterator(start, end, ascending, t)
+	}
+}
+
 // IterateRange makes a callback for all nodes with key between start and end non-inclusive.
 // If either are nil, then it is open on that side (nil, nil is the same as Iterate). The keys and
 // values must not be modified, since they may point to data stored within IAVL.