From 5e7ed766138b9e7ec2e60613e1c7e2579a0e82e6 Mon Sep 17 00:00:00 2001 From: h5law Date: Tue, 19 Mar 2024 11:34:46 +0000 Subject: [PATCH 1/5] chore: add context on hashing algorithms used by the trie --- docs/smt.md | 116 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 68 insertions(+), 48 deletions(-) diff --git a/docs/smt.md b/docs/smt.md index bae85d2..19cdbf7 100644 --- a/docs/smt.md +++ b/docs/smt.md @@ -4,31 +4,31 @@ - [Overview](#overview) - [Implementation](#implementation) - * [Inner Nodes](#inner-nodes) - * [Extension Nodes](#extension-nodes) - * [Leaf Nodes](#leaf-nodes) - * [Lazy Nodes](#lazy-nodes) - * [Lazy Loading](#lazy-loading) - * [Visualisations](#visualisations) - + [General Trie Structure](#general-trie-structure) - + [Lazy Nodes](#lazy-nodes-1) + - [Inner Nodes](#inner-nodes) + - [Extension Nodes](#extension-nodes) + - [Leaf Nodes](#leaf-nodes) + - [Lazy Nodes](#lazy-nodes) + - [Lazy Loading](#lazy-loading) + - [Visualisations](#visualisations) + - [General Trie Structure](#general-trie-structure) + - [Lazy Nodes](#lazy-nodes-1) - [Paths](#paths) - * [Visualisation](#visualisation) + - [Visualisation](#visualisation) - [Values](#values) - * [Nil values](#nil-values) + - [Nil values](#nil-values) - [Hashers & Digests](#hashers--digests) - [Roots](#roots) - [Proofs](#proofs) - * [Verification](#verification) - * [Closest Proof](#closest-proof) - + [Closest Proof Use Cases](#closest-proof-use-cases) - * [Compression](#compression) - * [Serialisation](#serialisation) + - [Verification](#verification) + - [Closest Proof](#closest-proof) + - [Closest Proof Use Cases](#closest-proof-use-cases) + - [Compression](#compression) + - [Serialisation](#serialisation) - [Database](#database) - * [Database Submodules](#database-submodules) - + [SimpleMap](#simplemap) - + [Badger](#badger) - * [Data Loss](#data-loss) + - [Database Submodules](#database-submodules) + - [SimpleMap](#simplemap) + - [Badger](#badger) + - [Data Loss](#data-loss) - [Sparse Merkle Sum Trie](#sparse-merkle-sum-trie) @@ -44,6 +44,25 @@ make SMTs valuable in applications like blockchains, decentralized databases, and authenticated data structures, providing optimized and trustworthy data storage and verification. +Although any hash function that satisfies the `hash.Hash` interface can be used +to construct the trie it is **strongly recommended** to use a hashing function +that provides the following properties: + +- **Collision resistance**: The hash function must be collision resistant, in + order for the inputs of the SMT to be unique. +- **Preimage resistance**: The hash function must be preimage resistant, to + protect against the attack of the Merkle tree construction attacks where the + attacker can modify unknown data. +- **Efficiency**: The hash function must be efficient, as it is used to compute + the hash of many nodes in the trie. + +Therefore it is recommended to use a hashing function such as: + +- `sha256` +- `sha3_256`/`keccak256` + +Or another sufficiently secure hashing algorithm. + See [smt.go](../smt.go) for more details on the implementation. ## Implementation @@ -66,9 +85,9 @@ The SMT has 4 node types that are used to construct the trie: ### Inner Nodes -Inner nodes represent a branch in the trie with two **non-nil** child nodes. -The inner node has an internal `digest` which represents the hash of the child -nodes concatenated hashes. +Inner nodes represent a branch in the trie with two **non-nil** child nodes. The +inner node has an internal `digest` which represents the hash of the child nodes +concatenated hashes. ### Extension Nodes @@ -307,8 +326,8 @@ described in the [implementation](#implementation) section. The following diagram represents the creation of a leaf node in an abstracted and simplified manner. -_Note: This diagram is not entirely accurate regarding the process of creating -a leaf node, but is a good representation of the process._ +_Note: This diagram is not entirely accurate regarding the process of creating a +leaf node, but is a good representation of the process._ ```mermaid graph TD @@ -346,17 +365,17 @@ graph TD ## Roots The root of the tree is a slice of bytes. `MerkleRoot` is an alias for `[]byte`. -This design enables easily passing around the data (e.g. on-chain) -while maintaining primitive usage in different use cases (e.g. proofs). +This design enables easily passing around the data (e.g. on-chain) while +maintaining primitive usage in different use cases (e.g. proofs). `MerkleRoot` provides helpers, such as retrieving the `Sum() uint64` to -interface with data it captures. However, for the SMT it **always** panics, -as there is no sum. +interface with data it captures. However, for the SMT it **always** panics, as +there is no sum. ## Proofs -The `SparseMerkleProof` type contains the information required for inclusion -and exclusion proofs, depending on the key provided to the trie method +The `SparseMerkleProof` type contains the information required for inclusion and +exclusion proofs, depending on the key provided to the trie method `Prove(key []byte)` either an inclusion or exclusion proof will be generated. _NOTE: The inclusion and exclusion proof are the same type, just constructed @@ -397,29 +416,29 @@ using the `VerifyClosestProof` function which requires the proof and root hash of the trie. Since the `ClosestProof` method takes a hash as input, it is possible to place a -leaf in the trie according to the hash's path, if it is known. Depending on -the use case of this function this may expose a vulnerability. **It is not -intendend to be used as a general purpose proof mechanism**, but instead as a -**Commit and Reveal** mechanism, as detailed below. +leaf in the trie according to the hash's path, if it is known. Depending on the +use case of this function this may expose a vulnerability. **It is not intendend +to be used as a general purpose proof mechanism**, but instead as a **Commit and +Reveal** mechanism, as detailed below. #### Closest Proof Use Cases The `CloestProof` function is intended for use as a `commit & reveal` mechanism. Where there are two actors involved, the **prover** and **verifier**. -_NOTE: Throughout this document, `commitment` of the the trie's root hash is also -referred to as closing the trie, such that no more updates are made to it once -committed._ +_NOTE: Throughout this document, `commitment` of the the trie's root hash is +also referred to as closing the trie, such that no more updates are made to it +once committed._ Consider the following attack vector (**without** a commit prior to a reveal) into consideration: 1. The **verifier** picks the hash (i.e. a single branch) they intend to check -1. The **prover** inserts a leaf (i.e. a value) whose key (determined via the +2. The **prover** inserts a leaf (i.e. a value) whose key (determined via the hasher) has a longer common prefix than any other leaf in the trie. -1. Due to the deterministic nature of the `ClosestProof`, method this leaf will +3. Due to the deterministic nature of the `ClosestProof`, method this leaf will **always** be returned given the identified hash. -1. The **verifier** then verifies the revealed `ClosestProof`, which returns a +4. The **verifier** then verifies the revealed `ClosestProof`, which returns a branch the **prover** inserted after knowing which leaf was going to be checked. @@ -428,16 +447,16 @@ Consider the following normal flow (**with** a commit prior to reveal) as 1. The **prover** commits to the state of their trie by publishes their root hash, thereby _closing_ their trie and not being able to make further changes. -1. The **verifier** selects a hash to be used in the `commit & reveal` process +2. The **verifier** selects a hash to be used in the `commit & reveal` process that the **prover** must provide a closest proof for. -1. The **prover** utilises this hash and computes the `ClosestProof` on their +3. The **prover** utilises this hash and computes the `ClosestProof` on their _closed_ trie, producing a `ClosestProof`, thus revealing a deterministic, pseudo-random leaf that existed in the tree prior to commitment, yet -1. The **verifier** verifies the proof, in turn, verifying the commitment - made by the **prover** to the state of the trie in the first step. -1. The **prover** had no opportunity to insert a new leaf into the trie - after learning which hash the **verifier** was going to require a - `ClosestProof` for. +4. The **verifier** verifies the proof, in turn, verifying the commitment made + by the **prover** to the state of the trie in the first step. +5. The **prover** had no opportunity to insert a new leaf into the trie after + learning which hash the **verifier** was going to require a `ClosestProof` + for. ### Compression @@ -497,7 +516,8 @@ database. It's interface exposes numerous extra methods not used by the trie, However it can still be used as a node-store with both in-memory and persistent options. -See [badger-store.md](./badger-store.md.md) for the details of the implementation. +See [badger-store.md](./badger-store.md.md) for the details of the +implementation. ### Data Loss From 9e9d9b3f1437002c27196eacedf827d8ca717618 Mon Sep 17 00:00:00 2001 From: h5law Date: Tue, 19 Mar 2024 11:35:10 +0000 Subject: [PATCH 2/5] chore: add context on external kvstore writeability --- docs/mapstore.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/mapstore.md b/docs/mapstore.md index a47c670..d36fc95 100644 --- a/docs/mapstore.md +++ b/docs/mapstore.md @@ -3,8 +3,8 @@ - [Implementations](#implementations) - * [SimpleMap](#simplemap) - * [BadgerV4](#badgerv4) + - [SimpleMap](#simplemap) + - [BadgerV4](#badgerv4) @@ -12,6 +12,10 @@ The `MapStore` is a simple interface used by the SM(S)T to store, delete and retrieve key-value pairs. It is intentionally simple and minimalistic so as to enable different key-value engines to implement and back the trie database. +Any key-value store used by the tries should **not** be able to be externally +writeable in production. This opens the possibility to attacks where the writer +can modify the trie database and prove values that were not inserted. + See: [the interface](../kvstore/interfaces.go) for a more detailed description of the simple interface required by the SM(S)T. @@ -31,11 +35,11 @@ See [simplemap.go](../kvstore/simplemap/simplemap.go) for more details. ### BadgerV4 -This library provides a wrapper around [dgraph-io/badger][badgerv4] to adhere -to the `MapStore` interface. See the [full documentation](./badger-store.md) -for additional functionality and implementation details. +This library provides a wrapper around [dgraph-io/badger][badgerv4] to adhere to +the `MapStore` interface. See the [full documentation](./badger-store.md) for +additional functionality and implementation details. -See: [badger](../kvstore/badger/) for more details on the implementation of -this submodule. +See: [badger](../kvstore/badger/) for more details on the implementation of this +submodule. [badgerv4]: https://github.com/dgraph-io/badger From 4831b52ba5f12955b8ac94e0ae96ea189116d249 Mon Sep 17 00:00:00 2001 From: h5law Date: Tue, 19 Mar 2024 11:38:17 +0000 Subject: [PATCH 3/5] feat: consolidate ClosestProof verification and remove the NilPathHasher method --- options.go | 18 ------------- proofs.go | 66 ++++++++++++++++++++++++++++++++++++--------- smst_proofs_test.go | 46 ++++++++++++++++++++++--------- smt_proofs_test.go | 19 +++++++------ 4 files changed, 96 insertions(+), 53 deletions(-) diff --git a/options.go b/options.go index c4eb422..884d559 100644 --- a/options.go +++ b/options.go @@ -1,9 +1,5 @@ package smt -import ( - "hash" -) - // Option is a function that configures SparseMerkleTrie. type Option func(*TrieSpec) @@ -16,17 +12,3 @@ func WithPathHasher(ph PathHasher) Option { func WithValueHasher(vh ValueHasher) Option { return func(ts *TrieSpec) { ts.vh = vh } } - -// NoPrehashSpec returns a new TrieSpec that has a nil Value Hasher and a nil -// Path Hasher -// NOTE: This should only be used when values are already hashed and a path is -// used instead of a key during proof verification, otherwise these will be -// double hashed and produce an incorrect leaf digest invalidating the proof. -func NoPrehashSpec(hasher hash.Hash, sumTrie bool) *TrieSpec { - spec := newTrieSpec(hasher, sumTrie) - opt := WithPathHasher(newNilPathHasher(hasher.Size())) - opt(&spec) - opt = WithValueHasher(nil) - opt(&spec) - return &spec -} diff --git a/proofs.go b/proofs.go index 64ce171..3a2b690 100644 --- a/proofs.go +++ b/proofs.go @@ -61,7 +61,11 @@ func (proof *SparseMerkleProof) validateBasic(spec *TrieSpec) error { // Check that leaf data for non-membership proofs is a valid size. lps := len(leafPrefix) + spec.ph.PathSize() if proof.NonMembershipLeafData != nil && len(proof.NonMembershipLeafData) < lps { - return fmt.Errorf("invalid non-membership leaf data size: got %d but min is %d", len(proof.NonMembershipLeafData), lps) + return fmt.Errorf( + "invalid non-membership leaf data size: got %d but min is %d", + len(proof.NonMembershipLeafData), + lps, + ) } // Check that all supplied sidenodes are the correct size. @@ -133,7 +137,11 @@ func (proof *SparseCompactMerkleProof) validateBasic(spec *TrieSpec) error { // Compact proofs: check that NumSideNodes is within the right range. if proof.NumSideNodes < 0 || proof.NumSideNodes > spec.ph.PathSize()*8 { - return fmt.Errorf("invalid number of side nodes: got %d, min is 0 and max is %d", len(proof.SideNodes), spec.ph.PathSize()*8) + return fmt.Errorf( + "invalid number of side nodes: got %d, min is 0 and max is %d", + len(proof.SideNodes), + spec.ph.PathSize()*8, + ) } // Compact proofs: check that the length of the bit mask is as expected @@ -185,6 +193,17 @@ func (proof *SparseMerkleClosestProof) Unmarshal(bz []byte) error { return dec.Decode(proof) } +// GetValueHash returns the value hash of the closest proof. +func (proof *SparseMerkleClosestProof) GetValueHash(spec *TrieSpec) []byte { + if proof.ClosestValueHash == nil { + return nil + } + if spec.sumTrie { + return proof.ClosestValueHash[:len(proof.ClosestValueHash)-sumSize] + } + return proof.ClosestValueHash +} + func (proof *SparseMerkleClosestProof) validateBasic(spec *TrieSpec) error { // ensure the depth of the leaf node being proven is within the path size if proof.Depth < 0 || proof.Depth > spec.ph.PathSize()*8 { @@ -246,7 +265,12 @@ func (proof *SparseCompactMerkleClosestProof) validateBasic(spec *TrieSpec) erro } for i, b := range proof.FlippedBits { if len(b) > maxSliceLen { - return fmt.Errorf("invalid compressed flipped bit index %d: got length %d, max is %d]", i, bytesToInt(b), maxSliceLen) + return fmt.Errorf( + "invalid compressed flipped bit index %d: got length %d, max is %d]", + i, + bytesToInt(b), + maxSliceLen, + ) } } // perform a sanity check on the closest proof @@ -301,26 +325,38 @@ func VerifySumProof(proof *SparseMerkleProof, root, key, value []byte, sum uint6 // VerifyClosestProof verifies a Merkle proof for a proof of inclusion for a leaf // found to have the closest path to the one provided to the proof structure -// -// TO_AUDITOR: This is akin to an inclusion proof with N (num flipped bits) exclusion -// proof wrapped into one and needs to be reviewed from an algorithm POV. func VerifyClosestProof(proof *SparseMerkleClosestProof, root []byte, spec *TrieSpec) (bool, error) { if err := proof.validateBasic(spec); err != nil { return false, errors.Join(ErrBadProof, err) } - if !spec.sumTrie { - return VerifyProof(proof.ClosestProof, root, proof.ClosestPath, proof.ClosestValueHash, spec) + // Create a new TrieSpec with a nil path hasher - as the ClosestProof + // already contains a hashed path, double hashing it will invalidate the + // proof. + nilSpec := &TrieSpec{ + th: spec.th, + ph: newNilPathHasher(spec.ph.PathSize()), + vh: spec.vh, + sumTrie: spec.sumTrie, + } + if !nilSpec.sumTrie { + return VerifyProof(proof.ClosestProof, root, proof.ClosestPath, proof.ClosestValueHash, nilSpec) } if proof.ClosestValueHash == nil { - return VerifySumProof(proof.ClosestProof, root, proof.ClosestPath, nil, 0, spec) + return VerifySumProof(proof.ClosestProof, root, proof.ClosestPath, nil, 0, nilSpec) } sumBz := proof.ClosestValueHash[len(proof.ClosestValueHash)-sumSize:] sum := binary.BigEndian.Uint64(sumBz) valueHash := proof.ClosestValueHash[:len(proof.ClosestValueHash)-sumSize] - return VerifySumProof(proof.ClosestProof, root, proof.ClosestPath, valueHash, sum, spec) + return VerifySumProof(proof.ClosestProof, root, proof.ClosestPath, valueHash, sum, nilSpec) } -func verifyProofWithUpdates(proof *SparseMerkleProof, root []byte, key []byte, value []byte, spec *TrieSpec) (bool, [][][]byte, error) { +func verifyProofWithUpdates( + proof *SparseMerkleProof, + root []byte, + key []byte, + value []byte, + spec *TrieSpec, +) (bool, [][][]byte, error) { path := spec.ph.Path(key) if err := proof.validateBasic(spec); err != nil { @@ -384,7 +420,13 @@ func VerifyCompactProof(proof *SparseCompactMerkleProof, root []byte, key, value } // VerifyCompactSumProof is similar to VerifySumProof but for a compacted Merkle proof. -func VerifyCompactSumProof(proof *SparseCompactMerkleProof, root []byte, key, value []byte, sum uint64, spec *TrieSpec) (bool, error) { +func VerifyCompactSumProof( + proof *SparseCompactMerkleProof, + root []byte, + key, value []byte, + sum uint64, + spec *TrieSpec, +) (bool, error) { decompactedProof, err := DecompactProof(proof, spec) if err != nil { return false, errors.Join(ErrBadProof, err) diff --git a/smst_proofs_test.go b/smst_proofs_test.go index 21fd454..c909d23 100644 --- a/smst_proofs_test.go +++ b/smst_proofs_test.go @@ -79,7 +79,14 @@ func TestSMST_Proof_Operations(t *testing.T) { result, err = VerifySumProof(proof, root, []byte("testKey"), []byte("badValue"), 10, base) // wrong value and sum require.NoError(t, err) require.False(t, result) - result, err = VerifySumProof(randomiseSumProof(proof), root, []byte("testKey"), []byte("testValue"), 5, base) // invalid proof + result, err = VerifySumProof( + randomiseSumProof(proof), + root, + []byte("testKey"), + []byte("testValue"), + 5, + base, + ) // invalid proof require.NoError(t, err) require.False(t, result) @@ -98,7 +105,14 @@ func TestSMST_Proof_Operations(t *testing.T) { result, err = VerifySumProof(proof, root, []byte("testKey2"), []byte("badValue"), 10, base) // wrong value and sum require.NoError(t, err) require.False(t, result) - result, err = VerifySumProof(randomiseSumProof(proof), root, []byte("testKey2"), []byte("testValue"), 5, base) // invalid proof + result, err = VerifySumProof( + randomiseSumProof(proof), + root, + []byte("testKey2"), + []byte("testValue"), + 5, + base, + ) // invalid proof require.NoError(t, err) require.False(t, result) @@ -129,7 +143,14 @@ func TestSMST_Proof_Operations(t *testing.T) { result, err = VerifySumProof(proof, root, []byte("testKey3"), defaultValue, 5, base) // wrong sum require.NoError(t, err) require.False(t, result) - result, err = VerifySumProof(randomiseSumProof(proof), root, []byte("testKey3"), defaultValue, 0, base) // invalid proof + result, err = VerifySumProof( + randomiseSumProof(proof), + root, + []byte("testKey3"), + defaultValue, + 0, + base, + ) // invalid proof require.NoError(t, err) require.False(t, result) } @@ -204,7 +225,6 @@ func TestSMST_Proof_ValidateBasic(t *testing.T) { func TestSMST_ClosestProof_ValidateBasic(t *testing.T) { smn := simplemap.NewSimpleMap() smst := NewSparseMerkleSumTrie(smn, sha256.New()) - np := NoPrehashSpec(sha256.New(), true) base := smst.Spec() path := sha256.Sum256([]byte("testKey2")) flipPathBit(path[:], 3) @@ -227,14 +247,14 @@ func TestSMST_ClosestProof_ValidateBasic(t *testing.T) { require.NoError(t, err) proof.Depth = -1 require.EqualError(t, proof.validateBasic(base), "invalid depth: got -1, outside of [0, 256]") - result, err := VerifyClosestProof(proof, root, np) + result, err := VerifyClosestProof(proof, root, smst.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) require.Error(t, err) proof.Depth = 257 require.EqualError(t, proof.validateBasic(base), "invalid depth: got 257, outside of [0, 256]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -244,14 +264,14 @@ func TestSMST_ClosestProof_ValidateBasic(t *testing.T) { require.NoError(t, err) proof.FlippedBits[0] = -1 require.EqualError(t, proof.validateBasic(base), "invalid flipped bit index 0: got -1, outside of [0, 8]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) require.Error(t, err) proof.FlippedBits[0] = 9 require.EqualError(t, proof.validateBasic(base), "invalid flipped bit index 0: got 9, outside of [0, 8]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -265,7 +285,7 @@ func TestSMST_ClosestProof_ValidateBasic(t *testing.T) { proof.validateBasic(base), "invalid closest path: 8d13809f932d0296b88c1913231ab4b403f05c88363575476204fef6930f22ae (not equal at bit: 3)", ) - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -326,7 +346,7 @@ func TestSMST_ProveClosest(t *testing.T) { ClosestProof: proof.ClosestProof, // copy of proof as we are checking equality of other fields }) - result, err = VerifyClosestProof(proof, root, NoPrehashSpec(sha256.New(), true)) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.NoError(t, err) require.True(t, result) @@ -352,7 +372,7 @@ func TestSMST_ProveClosest(t *testing.T) { ClosestProof: proof.ClosestProof, // copy of proof as we are checking equality of other fields }) - result, err = VerifyClosestProof(proof, root, NoPrehashSpec(sha256.New(), true)) + result, err = VerifyClosestProof(proof, root, smst.Spec()) require.NoError(t, err) require.True(t, result) } @@ -381,7 +401,7 @@ func TestSMST_ProveClosest_Empty(t *testing.T) { ClosestProof: &SparseMerkleProof{}, }) - result, err := VerifyClosestProof(proof, smst.Root(), NoPrehashSpec(sha256.New(), true)) + result, err := VerifyClosestProof(proof, smst.Root(), smst.Spec()) require.NoError(t, err) require.True(t, result) } @@ -419,7 +439,7 @@ func TestSMST_ProveClosest_OneNode(t *testing.T) { ClosestProof: &SparseMerkleProof{}, }) - result, err := VerifyClosestProof(proof, smst.Root(), NoPrehashSpec(sha256.New(), true)) + result, err := VerifyClosestProof(proof, smst.Root(), smst.Spec()) require.NoError(t, err) require.True(t, result) } diff --git a/smt_proofs_test.go b/smt_proofs_test.go index 2cf70c8..1c353df 100644 --- a/smt_proofs_test.go +++ b/smt_proofs_test.go @@ -178,7 +178,6 @@ func TestSMT_Proof_ValidateBasic(t *testing.T) { func TestSMT_ClosestProof_ValidateBasic(t *testing.T) { smn := simplemap.NewSimpleMap() smt := NewSparseMerkleTrie(smn, sha256.New()) - np := NoPrehashSpec(sha256.New(), false) base := smt.Spec() path := sha256.Sum256([]byte("testKey2")) flipPathBit(path[:], 3) @@ -201,14 +200,14 @@ func TestSMT_ClosestProof_ValidateBasic(t *testing.T) { require.NoError(t, err) proof.Depth = -1 require.EqualError(t, proof.validateBasic(base), "invalid depth: got -1, outside of [0, 256]") - result, err := VerifyClosestProof(proof, root, np) + result, err := VerifyClosestProof(proof, root, smt.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) require.Error(t, err) proof.Depth = 257 require.EqualError(t, proof.validateBasic(base), "invalid depth: got 257, outside of [0, 256]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -218,14 +217,14 @@ func TestSMT_ClosestProof_ValidateBasic(t *testing.T) { require.NoError(t, err) proof.FlippedBits[0] = -1 require.EqualError(t, proof.validateBasic(base), "invalid flipped bit index 0: got -1, outside of [0, 8]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) require.Error(t, err) proof.FlippedBits[0] = 9 require.EqualError(t, proof.validateBasic(base), "invalid flipped bit index 0: got 9, outside of [0, 8]") - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -239,7 +238,7 @@ func TestSMT_ClosestProof_ValidateBasic(t *testing.T) { proof.validateBasic(base), "invalid closest path: 8d13809f932d0296b88c1913231ab4b403f05c88363575476204fef6930f22ae (not equal at bit: 3)", ) - result, err = VerifyClosestProof(proof, root, np) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.ErrorIs(t, err, ErrBadProof) require.False(t, result) _, err = CompactClosestProof(proof, base) @@ -287,7 +286,7 @@ func TestSMT_ProveClosest(t *testing.T) { checkClosestCompactEquivalence(t, proof, smt.Spec()) require.NotEqual(t, proof, &SparseMerkleClosestProof{}) - result, err = VerifyClosestProof(proof, root, NoPrehashSpec(sha256.New(), false)) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.NoError(t, err) require.True(t, result) closestPath := sha256.Sum256([]byte("testKey2")) @@ -304,7 +303,7 @@ func TestSMT_ProveClosest(t *testing.T) { checkClosestCompactEquivalence(t, proof, smt.Spec()) require.NotEqual(t, proof, &SparseMerkleClosestProof{}) - result, err = VerifyClosestProof(proof, root, NoPrehashSpec(sha256.New(), false)) + result, err = VerifyClosestProof(proof, root, smt.Spec()) require.NoError(t, err) require.True(t, result) closestPath = sha256.Sum256([]byte("testKey4")) @@ -336,7 +335,7 @@ func TestSMT_ProveClosest_Empty(t *testing.T) { ClosestProof: &SparseMerkleProof{}, }) - result, err := VerifyClosestProof(proof, smt.Root(), NoPrehashSpec(sha256.New(), false)) + result, err := VerifyClosestProof(proof, smt.Root(), smt.Spec()) require.NoError(t, err) require.True(t, result) } @@ -368,7 +367,7 @@ func TestSMT_ProveClosest_OneNode(t *testing.T) { ClosestProof: &SparseMerkleProof{}, }) - result, err := VerifyClosestProof(proof, smt.Root(), NoPrehashSpec(sha256.New(), false)) + result, err := VerifyClosestProof(proof, smt.Root(), smt.Spec()) require.NoError(t, err) require.True(t, result) } From 46dc5c53df57a0bb51ad3229ad3b8489d331b575 Mon Sep 17 00:00:00 2001 From: h5law Date: Tue, 19 Mar 2024 11:38:55 +0000 Subject: [PATCH 4/5] feat: reorganise extension node insertion to use separate pointers for the child node --- proofs.go | 11 ----------- smt.go | 29 +++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/proofs.go b/proofs.go index 3a2b690..0b8f11d 100644 --- a/proofs.go +++ b/proofs.go @@ -193,17 +193,6 @@ func (proof *SparseMerkleClosestProof) Unmarshal(bz []byte) error { return dec.Decode(proof) } -// GetValueHash returns the value hash of the closest proof. -func (proof *SparseMerkleClosestProof) GetValueHash(spec *TrieSpec) []byte { - if proof.ClosestValueHash == nil { - return nil - } - if spec.sumTrie { - return proof.ClosestValueHash[:len(proof.ClosestValueHash)-sumSize] - } - return proof.ClosestValueHash -} - func (proof *SparseMerkleClosestProof) validateBasic(spec *TrieSpec) error { // ensure the depth of the leaf node being proven is within the path size if proof.Depth < 0 || proof.Depth > spec.ph.PathSize()*8 { diff --git a/smt.go b/smt.go index 558db63..f55a3f8 100644 --- a/smt.go +++ b/smt.go @@ -178,19 +178,36 @@ func (smt *SMT) update( } // We insert an "extension" representing multiple single-branch inner nodes last := &node + var newInner *innerNode + if getPathBit(path, prefixlen) == left { + newInner = &innerNode{ + leftChild: newLeaf, + rightChild: leaf, + } + } else { + newInner = &innerNode{ + leftChild: leaf, + rightChild: newLeaf, + } + } + // Determine if we need to insert an extension or a branch if depth < prefixlen { // note: this keeps path slice alive - GC inefficiency? if depth > 0xff { panic("invalid depth") } - ext := extensionNode{path: path, pathBounds: [2]byte{byte(depth), byte(prefixlen)}} + ext := extensionNode{ + child: newInner, + path: path, + pathBounds: [2]byte{ + byte(depth), byte(prefixlen), + }, + } + // Dereference the last node to replace it with the extension node *last = &ext - last = &ext.child - } - if getPathBit(path, prefixlen) == left { - *last = &innerNode{leftChild: newLeaf, rightChild: leaf} } else { - *last = &innerNode{leftChild: leaf, rightChild: newLeaf} + // Dereference the last node to replace it with the new inner node + *last = newInner } return node, nil } From 51999693457b1f0014afa42d1c69765377a21c95 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 8 Apr 2024 16:00:05 -0700 Subject: [PATCH 5/5] Review submittd PR --- .gitignore | 3 +++ docs/mapstore.md | 20 ++++++++++---------- docs/smt.md | 42 +++++++++++++++++------------------------- proofs.go | 6 +++--- smt.go | 4 ++-- 5 files changed, 35 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index aa638db..55b4fed 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ # Ignore Goland and JetBrains IDE files .idea/ + +# Visual Studio Code +.vscode diff --git a/docs/mapstore.md b/docs/mapstore.md index d36fc95..ebaf965 100644 --- a/docs/mapstore.md +++ b/docs/mapstore.md @@ -1,21 +1,17 @@ -# MapStore - - +# MapStore +- [Introduction](#introduction) - [Implementations](#implementations) - [SimpleMap](#simplemap) - [BadgerV4](#badgerv4) +- [Note On External Writability](#note-on-external-writability) - +## Introduction The `MapStore` is a simple interface used by the SM(S)T to store, delete and retrieve key-value pairs. It is intentionally simple and minimalistic so as to enable different key-value engines to implement and back the trie database. -Any key-value store used by the tries should **not** be able to be externally -writeable in production. This opens the possibility to attacks where the writer -can modify the trie database and prove values that were not inserted. - See: [the interface](../kvstore/interfaces.go) for a more detailed description of the simple interface required by the SM(S)T. @@ -35,11 +31,15 @@ See [simplemap.go](../kvstore/simplemap/simplemap.go) for more details. ### BadgerV4 -This library provides a wrapper around [dgraph-io/badger][badgerv4] to adhere to +This library provides a wrapper around [dgraph-io/badger][https://github.com/dgraph-io/badger] to adhere to the `MapStore` interface. See the [full documentation](./badger-store.md) for additional functionality and implementation details. See: [badger](../kvstore/badger/) for more details on the implementation of this submodule. -[badgerv4]: https://github.com/dgraph-io/badger +## Note On External Writability + +Any key-value store used by the tries should **not** be able to be externally +writeable in production. This opens the possibility to attacks where the writer +can modify the trie database and prove values that were not inserted. diff --git a/docs/smt.md b/docs/smt.md index 19cdbf7..b7bdfad 100644 --- a/docs/smt.md +++ b/docs/smt.md @@ -1,6 +1,4 @@ -# smt - - +# smt - [Overview](#overview) - [Implementation](#implementation) @@ -16,7 +14,8 @@ - [Visualisation](#visualisation) - [Values](#values) - [Nil values](#nil-values) -- [Hashers & Digests](#hashers--digests) +- [Hashers \& Digests](#hashers--digests) + - [Hash Function Recommendations](#hash-function-recommendations) - [Roots](#roots) - [Proofs](#proofs) - [Verification](#verification) @@ -31,8 +30,6 @@ - [Data Loss](#data-loss) - [Sparse Merkle Sum Trie](#sparse-merkle-sum-trie) - - ## Overview Sparse Merkle Tries (SMTs) are efficient and secure data structures for storing @@ -44,25 +41,6 @@ make SMTs valuable in applications like blockchains, decentralized databases, and authenticated data structures, providing optimized and trustworthy data storage and verification. -Although any hash function that satisfies the `hash.Hash` interface can be used -to construct the trie it is **strongly recommended** to use a hashing function -that provides the following properties: - -- **Collision resistance**: The hash function must be collision resistant, in - order for the inputs of the SMT to be unique. -- **Preimage resistance**: The hash function must be preimage resistant, to - protect against the attack of the Merkle tree construction attacks where the - attacker can modify unknown data. -- **Efficiency**: The hash function must be efficient, as it is used to compute - the hash of many nodes in the trie. - -Therefore it is recommended to use a hashing function such as: - -- `sha256` -- `sha3_256`/`keccak256` - -Or another sufficiently secure hashing algorithm. - See [smt.go](../smt.go) for more details on the implementation. ## Implementation @@ -362,6 +340,20 @@ graph TD VH --ValueHash-->L ``` +### Hash Function Recommendations + +Although any hash function that satisfies the `hash.Hash` interface can be used +to construct the trie, it is **strongly recommended** to use a hashing function +that provides the following properties: + +- **Collision resistance**: The hash function must be collision resistant. This + is needed in order for the inputs of the SMT to be unique. +- **Preimage resistance**: The hash function must be preimage resistant. This + is needed to protect against the Merkle tree construction attacks where + the attacker can modify unknown data. +- **Efficiency**: The hash function must be efficient, as it is used to compute + the hash of many nodes in the trie. + ## Roots The root of the tree is a slice of bytes. `MerkleRoot` is an alias for `[]byte`. diff --git a/proofs.go b/proofs.go index 12d3de8..6a09fe9 100644 --- a/proofs.go +++ b/proofs.go @@ -341,9 +341,9 @@ func VerifyClosestProof(proof *SparseMerkleClosestProof, root []byte, spec *Trie if err := proof.validateBasic(spec); err != nil { return false, errors.Join(ErrBadProof, err) } - // Create a new TrieSpec with a nil path hasher - as the ClosestProof - // already contains a hashed path, double hashing it will invalidate the - // proof. + // Create a new TrieSpec with a nil path hasher. + // Since the ClosestProof already contains a hashed path, double hashing it + // will invalidate the proof. nilSpec := &TrieSpec{ th: spec.th, ph: newNilPathHasher(spec.ph.PathSize()), diff --git a/smt.go b/smt.go index 7928c40..0bcc6df 100644 --- a/smt.go +++ b/smt.go @@ -177,7 +177,6 @@ func (smt *SMT) update( return newLeaf, nil } // We insert an "extension" representing multiple single-branch inner nodes - last := &node var newInner *innerNode if getPathBit(path, prefixlen) == left { newInner = &innerNode{ @@ -191,6 +190,7 @@ func (smt *SMT) update( } } // Determine if we need to insert an extension or a branch + last := &node if depth < prefixlen { // note: this keeps path slice alive - GC inefficiency? if depth > 0xff { @@ -419,7 +419,7 @@ func (smt *SMT) Prove(key []byte) (proof *SparseMerkleProof, err error) { // node is encountered, the traversal backsteps and flips the path bit for that // depth (ie tries left if it tried right and vice versa). This guarantees that // a proof of inclusion is found that has the most common bits with the path -// provided, biased to the longest common prefix +// provided, biased to the longest common prefix. func (smt *SMT) ProveClosest(path []byte) ( proof *SparseMerkleClosestProof, // proof of the key-value pair found err error, // the error value encountered