Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: replace instances of tree with trie #31

Merged
merged 2 commits into from
Dec 6, 2023
Merged

chore: replace instances of tree with trie #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
95cf936
chore: replace instances of tree with trie
h5law Dec 6, 2023
54e0721
chore: address linter errors
h5law Dec 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 41 additions & 9 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ SHELL := /bin/bash

.SILENT:

#####################
### General ###
#####################

.PHONY: help
.DEFAULT_GOAL := help
help: ## Prints all the targets in all the Makefiles
Expand All @@ -11,38 +15,66 @@ help: ## Prints all the targets in all the Makefiles
list: ## List all make targets
@${MAKE} -pRrn : -f $(MAKEFILE_LIST) 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | sort

## Ensure godoc is installed
.PHONY: check_godoc
# Internal helper target - check if godoc is installed
check_godoc:
{ \
if ( ! ( command -v godoc >/dev/null )); then \
echo "Seems like you don't have godoc installed. Make sure you install it via 'go install golang.org/x/tools/cmd/godoc@latest' before continuing"; \
exit 1; \
fi; \
}

#####################
### Documentation ###
#####################

.PHONY: go_docs
go_docs: check_godoc ## Generate documentation for the project
echo "Visit http://localhost:6060/pkg/github.com/pokt-network/smt/"
godoc -http=:6060

#####################
#### Testing ####
#####################

.PHONY: test_all
test_all: ## runs the test suite
go test -v -p 1 ./ -mod=readonly -race

#####################
### Benchmark ###
#####################

.PHONY: benchmark_all
bechmark_all: ## runs all benchmarks
go test -benchmem -run=^$ -bench Benchmark ./benchmarks -timeout 0

.PHONY: benchmark_smt
benchmark_smt: ## runs all benchmarks for the SMT
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTree ./benchmarks -timeout 0
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTrie ./benchmarks -timeout 0

.PHONY: benchmark_smt_fill
benchmark_smt_fill: ## runs a benchmark on filling the SMT with different amounts of values
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTree_Fill ./benchmarks -timeout 0 -benchtime 10x
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTrie_Fill ./benchmarks -timeout 0 -benchtime 10x

.PHONY: benchmark_smt_ops
benchmark_smt_ops: ## runs the benchmarks testing different operations on the SMT against different sized trees
go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleTree_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0
benchmark_smt_ops: ## runs the benchmarks testing different operations on the SMT against different sized tries
go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleTrie_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0

.PHONY: benchmark_smst
benchmark_smst: ## runs all benchmarks for the SMST
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTree ./benchmarks -timeout 0
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTrie ./benchmarks -timeout 0

.PHONY: benchmark_smst_fill
benchmark_smst_fill: ## runs a benchmark on filling the SMST with different amounts of values
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTree_Fill ./benchmarks -timeout 0 -benchtime 10x
go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTrie_Fill ./benchmarks -timeout 0 -benchtime 10x

.PHONY: benchmark_smst_ops
benchmark_smst_ops: ## runs the benchmarks testing different operations on the SMST against different sized trees
go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleSumTree_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0
benchmark_smst_ops: ## runs the benchmarks testing different operations on the SMST against different sized tries
go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleSumTrie_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0

.PHONY: bechmark_proof_sizes
benchmark_proof_sizes: ## runs the benchmarks testing the proof sizes for different sized trees
benchmark_proof_sizes: ## runs the benchmarks testing the proof sizes for different sized tries
go test -v ./benchmarks -run ProofSizes
24 changes: 12 additions & 12 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ Note: **Requires Go 1.20+**

## Overview

This is a Go library that implements a Sparse Merkle tree for a key-value map.
The tree implements the same optimisations specified in the [Libra whitepaper],
to reduce the number of hash operations required per tree operation to $O(k)$
where $k$ is the number of non-empty elements in the tree. And is implemented
This is a Go library that implements a Sparse Merkle Trie for a key-value map.
The trie implements the same optimisations specified in the [Libra whitepaper],
to reduce the number of hash operations required per trie operation to $O(k)$
where $k$ is the number of non-empty elements in the trie. And is implemented
in a similar way to the [JMT whitepaper], with additional features and proof
mechanics.

Expand Down Expand Up @@ -74,8 +74,8 @@ clarification.

#### Commit

- The `Commit` term refers to the `Commit` method of the tree. This takes all
changes (which are made in memory) to the tree and writes them to the
- The `Commit` term refers to the `Commit` method of the trie. This takes all
changes (which are made in memory) to the trie and writes them to the
underlying database.

#### Sizing
Expand All @@ -88,11 +88,11 @@ clarification.
- 5M = 5,000,000 (Five million)
- 10M = 10,000,000 (Ten million)
- These sizes refer to the number of key-value pairs or key-value-sum triples
inserted into the tree either beforehand or during the benchmark depending on
inserted into the trie either beforehand or during the benchmark depending on
which benchmark it is.

_NOTE: Unless otherwise stated the benchmarks in this document were ran on a
2023 14-inch Macbook Pro M2 Max with 32GB of RAM. The trees tested are using the
2023 14-inch Macbook Pro M2 Max with 32GB of RAM. The tries tested are using the
`sha256.New()` hasher._

_TODO: There is an opportunity to do a fuzz test where we commit every `N`
Expand All @@ -110,7 +110,7 @@ make benchmark_smt

The "fill" benchmarks cover the time taken to insert `N` key-value pairs into
the SMT, as well as how long it takes to do this and commit these changes to
disk. This gives us an insight into how long it takes to build a tree of a
disk. This gives us an insight into how long it takes to build a trie of a
certain size.

In order to run the SMT filling benchmarks use the following command:
Expand Down Expand Up @@ -190,7 +190,7 @@ make benchmark_smst

The "fill" benchmarks cover the time taken to insert `N` key-value-sum triples
into the SMST, as well as how long it takes to do this and commit these changes
to disk. This gives us an insight into how long it takes to build a tree of a
to disk. This gives us an insight into how long it takes to build a trie of a
certain size.

In order to run the SMST filling benchmarks use the following command:
Expand Down Expand Up @@ -260,7 +260,7 @@ make benchmark_smst_ops

### Proofs

To run the tests to average the proof size for numerous prefilled trees use the
To run the tests to average the proof size for numerous prefilled tries use the
following command:

```sh
Expand All @@ -287,5 +287,5 @@ make benchmark_proof_sizes
| 5,000,000 | 1166 | 975 | 2123 | 1169 | 1018 | 1388 |
| 10,000,000 | 1207 | 1026 | 2123 | 1210 | 1059 | 1429 |

[jmt whitepaper]: https://developers.diem.com/papers/jellyfish-merkle-tree/2021-01-14.pdf
[jmt whitepaper]: https://developers.diem.com/papers/jellyfish-merkle-trie/2021-01-14.pdf
[libra whitepaper]: https://diem-developers-components.netlify.app/papers/the-diem-blockchain/2020-05-26.pdf
34 changes: 18 additions & 16 deletions benchmarks/bench_leaf_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,35 @@ package smt
import (
"crypto/sha256"
"fmt"
"github.com/pokt-network/smt"
"github.com/stretchr/testify/require"
"strconv"
"testing"

"github.com/stretchr/testify/require"

"github.com/pokt-network/smt"
)

func BenchmarkSMTLeafSizes_Fill(b *testing.B) {
treeSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves
trieSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves
leafSizes := []int{256, 512, 1024, 2048, 4096, 8192, 16384} // number of bytes per leaf
nodes, err := smt.NewKVStore("")
require.NoError(b, err)
for _, treeSize := range treeSizes {
for _, trieSize := range trieSizes {
for _, leafSize := range leafSizes {
leaf := make([]byte, leafSize)
for _, operation := range []string{"Fill", "Fill & Commit"} {
tree := smt.NewSparseMerkleTree(nodes, sha256.New(), smt.WithValueHasher(nil))
trie := smt.NewSparseMerkleTrie(nodes, sha256.New(), smt.WithValueHasher(nil))
b.ResetTimer()
b.Run(
fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, treeSize),
fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, trieSize),
func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < treeSize; i++ {
require.NoError(b, tree.Update([]byte(strconv.Itoa(i)), leaf))
for i := 0; i < trieSize; i++ {
require.NoError(b, trie.Update([]byte(strconv.Itoa(i)), leaf))
}
if operation == "Fill & Commit" {
require.NoError(b, tree.Commit())
require.NoError(b, trie.Commit())
}
},
)
Expand All @@ -41,26 +43,26 @@ func BenchmarkSMTLeafSizes_Fill(b *testing.B) {
}

func BenchmarkSMSTLeafSizes_Fill(b *testing.B) {
treeSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves
trieSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves
leafSizes := []int{256, 512, 1024, 2048, 4096, 8192, 16384} // number of bytes per leaf
nodes, err := smt.NewKVStore("")
require.NoError(b, err)
for _, treeSize := range treeSizes {
for _, trieSize := range trieSizes {
for _, leafSize := range leafSizes {
leaf := make([]byte, leafSize)
for _, operation := range []string{"Fill", "Fill & Commit"} {
tree := smt.NewSparseMerkleSumTree(nodes, sha256.New(), smt.WithValueHasher(nil))
trie := smt.NewSparseMerkleSumTrie(nodes, sha256.New(), smt.WithValueHasher(nil))
b.ResetTimer()
b.Run(
fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, treeSize),
fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, trieSize),
func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < treeSize; i++ {
require.NoError(b, tree.Update([]byte(strconv.Itoa(i)), leaf, uint64(i)))
for i := 0; i < trieSize; i++ {
require.NoError(b, trie.Update([]byte(strconv.Itoa(i)), leaf, uint64(i)))
}
if operation == "Fill & Commit" {
require.NoError(b, tree.Commit())
require.NoError(b, trie.Commit())
}
},
)
Expand Down
Loading
Loading