From 5f8397656985187ff71b763854d5c6838b775d01 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 31 May 2022 21:30:24 +0800 Subject: [PATCH] Revert "[dev] update SMT library" This reverts commit 1a37cc229c2bd27f83061c415dda4f21b4aca795. --- go.mod | 3 - store/v2alpha1/multi/proof_test.go | 2 +- store/v2alpha1/multi/store.go | 47 ++++++++------- store/v2alpha1/multi/view_store.go | 10 ++-- store/v2alpha1/smt/ics23.go | 8 +-- store/v2alpha1/smt/ics23_test.go | 5 +- store/v2alpha1/smt/proof.go | 93 ++++++++++++++++++++++++++++++ store/v2alpha1/smt/store.go | 82 +++++++++++--------------- store/v2alpha1/smt/store_test.go | 7 +-- 9 files changed, 164 insertions(+), 93 deletions(-) create mode 100644 store/v2alpha1/smt/proof.go diff --git a/go.mod b/go.mod index 8ee20273f325..b1a3d22c9673 100644 --- a/go.mod +++ b/go.mod @@ -163,6 +163,3 @@ replace ( ) retract v0.43.0 - -// DEV -replace github.com/lazyledger/smt => ../smt diff --git a/store/v2alpha1/multi/proof_test.go b/store/v2alpha1/multi/proof_test.go index dc9e38fa15ec..13c69a5d616c 100644 --- a/store/v2alpha1/multi/proof_test.go +++ b/store/v2alpha1/multi/proof_test.go @@ -20,7 +20,7 @@ func keyPath(prefix, key string) string { func TestVerifySMTStoreProof(t *testing.T) { // Create main tree for testing. txn := memdb.NewDB().ReadWriter() - store := smt.NewStore(smt.StoreParams{TreeData: txn}) + store := smt.NewStore(txn) store.Set([]byte("MYKEY"), []byte("MYVALUE")) root := store.Root() diff --git a/store/v2alpha1/multi/store.go b/store/v2alpha1/multi/store.go index e86d264f5978..300a0d4e8ad7 100644 --- a/store/v2alpha1/multi/store.go +++ b/store/v2alpha1/multi/store.go @@ -504,27 +504,23 @@ func (rs *Store) getSubstore(key string) (*substore, error) { pfx := substorePrefix(key) stateRW := prefixdb.NewPrefixReadWriter(rs.stateTxn, pfx) stateCommitmentRW := prefixdb.NewPrefixReadWriter(rs.stateCommitmentTxn, pfx) - var stateCommitmentStore *smt.Store - data := prefixdb.NewPrefixReadWriter(stateRW, dataPrefix) + rootHash, err := stateRW.Get(substoreMerkleRootKey) if err != nil { return nil, err } if rootHash != nil { - stateCommitmentStore = loadSMT(stateCommitmentRW, data, rootHash) + stateCommitmentStore = loadSMT(stateCommitmentRW, rootHash) } else { - params := smt.StoreParams{ - TreeData: prefixdb.NewPrefixReadWriter(stateCommitmentRW, smtPrefix), - ValueData: data, - } - stateCommitmentStore = smt.NewStore(params) + smtdb := prefixdb.NewPrefixReadWriter(stateCommitmentRW, smtPrefix) + stateCommitmentStore = smt.NewStore(smtdb) } return &substore{ root: rs, name: key, - dataBucket: data, + dataBucket: prefixdb.NewPrefixReadWriter(stateRW, dataPrefix), stateCommitmentStore: stateCommitmentStore, }, nil } @@ -535,7 +531,7 @@ func (s *substore) refresh(rootHash []byte) { stateRW := prefixdb.NewPrefixReadWriter(s.root.stateTxn, pfx) stateCommitmentRW := prefixdb.NewPrefixReadWriter(s.root.stateCommitmentTxn, pfx) s.dataBucket = prefixdb.NewPrefixReadWriter(stateRW, dataPrefix) - s.stateCommitmentStore = loadSMT(stateCommitmentRW, s.dataBucket, rootHash) + s.stateCommitmentStore = loadSMT(stateCommitmentRW, rootHash) } // Commit implements Committer. @@ -587,20 +583,26 @@ func pruneVersions(current int64, opts pruningtypes.PruningOptions, prune func(i } } -// Calculates root hashes and commits to DB. Does not verify target version or perform pruning. -func (s *Store) commit(target uint64) (id *types.CommitID, err error) { - storeHashes := make(map[string][]byte, len(s.schema)) +func (s *Store) getMerkleRoots() (ret map[string][]byte, err error) { + ret = map[string][]byte{} for key := range s.schema { sub, has := s.substoreCache[key.Name()] if !has { - if sub, err = s.getSubstore(key.Name()); err != nil { + sub, err = s.getSubstore(key.Name()) + if err != nil { return } } - if err = sub.stateCommitmentStore.Commit(); err != nil { - return - } - storeHashes[key.Name()] = sub.stateCommitmentStore.Root() + ret[key.Name()] = sub.stateCommitmentStore.Root() + } + return +} + +// Calculates root hashes and commits to DB. Does not verify target version or perform pruning. +func (s *Store) commit(target uint64) (id *types.CommitID, err error) { + storeHashes, err := s.getMerkleRoots() + if err != nil { + return } // Update substore Merkle roots for key, storeHash := range storeHashes { @@ -844,12 +846,9 @@ func (rs *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { return res } -func loadSMT(sc, data dbm.DBReadWriter, root []byte) *smt.Store { - params := smt.StoreParams{ - TreeData: prefixdb.NewPrefixReadWriter(sc, smtPrefix), - ValueData: data, - } - return smt.LoadStore(params, root) +func loadSMT(stateCommitmentTxn dbm.DBReadWriter, root []byte) *smt.Store { + smtdb := prefixdb.NewPrefixReadWriter(stateCommitmentTxn, smtPrefix) + return smt.LoadStore(smtdb, root) } // Returns closest index and whether it's a match diff --git a/store/v2alpha1/multi/view_store.go b/store/v2alpha1/multi/view_store.go index 55c7eb598216..e9dda6259c66 100644 --- a/store/v2alpha1/multi/view_store.go +++ b/store/v2alpha1/multi/view_store.go @@ -57,13 +57,11 @@ func (vs *viewStore) getSubstore(key string) (*viewSubstore, error) { if err != nil { return nil, err } - data := prefixdb.NewPrefixReader(stateR, dataPrefix) return &viewSubstore{ - root: vs, - name: key, - dataBucket: data, - stateCommitmentStore: loadSMT( - dbm.ReaderAsReadWriter(stateCommitmentR), dbm.ReaderAsReadWriter(data), rootHash), + root: vs, + name: key, + dataBucket: prefixdb.NewPrefixReader(stateR, dataPrefix), + stateCommitmentStore: loadSMT(dbm.ReaderAsReadWriter(stateCommitmentR), rootHash), }, nil } diff --git a/store/v2alpha1/smt/ics23.go b/store/v2alpha1/smt/ics23.go index 22d98faa8f43..31d78f993dc0 100644 --- a/store/v2alpha1/smt/ics23.go +++ b/store/v2alpha1/smt/ics23.go @@ -15,12 +15,12 @@ import ( func createIcs23Proof(store *Store, key []byte) (*ics23.CommitmentProof, error) { ret := &ics23.CommitmentProof{} path := sha256.Sum256(key) - valueHash, err := store.tree.Get(key) + has, err := store.tree.Has(key) if err != nil { return nil, err } - if valueHash != nil { // Membership proof - value, err := store.values.Get(key) + if has { // Membership proof + value, err := store.values.Get(path[:]) if err != nil { return nil, err } @@ -52,7 +52,7 @@ func toNonExistenceProof(store *Store, path [32]byte) (*ics23.NonExistenceProof, getNext := func(it dbm.Iterator) (*ics23.ExistenceProof, error) { defer it.Close() if it.Next() { - value, err := store.values.Get(it.Value()) + value, err := store.values.Get(it.Key()) if err != nil { return nil, err } diff --git a/store/v2alpha1/smt/ics23_test.go b/store/v2alpha1/smt/ics23_test.go index 14fa8889675b..5667dbe16e78 100644 --- a/store/v2alpha1/smt/ics23_test.go +++ b/store/v2alpha1/smt/ics23_test.go @@ -14,7 +14,7 @@ import ( func TestProofICS23(t *testing.T) { txn := memdb.NewDB().ReadWriter() - s := store.NewStore(store.StoreParams{TreeData: txn}) + s := store.NewStore(txn) // pick keys whose hashes begin with different bits key00 := []byte("foo") // 00101100 = sha256(foo)[0] key01 := []byte("bill") // 01100010 @@ -74,8 +74,7 @@ func TestProofICS23(t *testing.T) { // Make sure proofs work with a loaded store root := s.Root() - assert.NoError(t, s.Commit()) - s = store.LoadStore(store.StoreParams{TreeData: txn}, root) + s = store.LoadStore(txn, root) proof, err = s.GetProofICS23(key10) assert.NoError(t, err) nonexist = proof.GetNonexist() diff --git a/store/v2alpha1/smt/proof.go b/store/v2alpha1/smt/proof.go new file mode 100644 index 000000000000..f013d1b2b995 --- /dev/null +++ b/store/v2alpha1/smt/proof.go @@ -0,0 +1,93 @@ +package smt + +import ( + "bytes" + "crypto/sha256" + "encoding/gob" + "hash" + + "github.com/cosmos/cosmos-sdk/store/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/lazyledger/smt" + "github.com/tendermint/tendermint/crypto/merkle" + tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto" +) + +type HasherType byte + +const ( + SHA256 HasherType = iota +) + +const ( + ProofType = "smt" +) + +type ProofOp struct { + Root []byte + Key []byte + Hasher HasherType + Proof smt.SparseMerkleProof +} + +var _ merkle.ProofOperator = (*ProofOp)(nil) + +// NewProofOp returns a ProofOp for a SparseMerkleProof. +func NewProofOp(root, key []byte, hasher HasherType, proof smt.SparseMerkleProof) *ProofOp { + return &ProofOp{ + Root: root, + Key: key, + Hasher: hasher, + Proof: proof, + } +} + +func (p *ProofOp) Run(args [][]byte) ([][]byte, error) { + switch len(args) { + case 0: // non-membership proof + if !smt.VerifyProof(p.Proof, p.Root, p.Key, []byte{}, getHasher(p.Hasher)) { + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "proof did not verify absence of key: %s", p.Key) + } + case 1: // membership proof + if !smt.VerifyProof(p.Proof, p.Root, p.Key, args[0], getHasher(p.Hasher)) { + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "proof did not verify existence of key %s with given value %x", p.Key, args[0]) + } + default: + return nil, sdkerrors.Wrapf(types.ErrInvalidProof, "args must be length 0 or 1, got: %d", len(args)) + } + return [][]byte{p.Root}, nil +} + +func (p *ProofOp) GetKey() []byte { + return p.Key +} + +func (p *ProofOp) ProofOp() tmmerkle.ProofOp { + var data bytes.Buffer + enc := gob.NewEncoder(&data) + enc.Encode(p) + return tmmerkle.ProofOp{ + Type: "smt", + Key: p.Key, + Data: data.Bytes(), + } +} + +func ProofDecoder(pop tmmerkle.ProofOp) (merkle.ProofOperator, error) { + dec := gob.NewDecoder(bytes.NewBuffer(pop.Data)) + var proof ProofOp + err := dec.Decode(&proof) + if err != nil { + return nil, err + } + return &proof, nil +} + +func getHasher(hasher HasherType) hash.Hash { + switch hasher { + case SHA256: + return sha256.New() + default: + return nil + } +} diff --git a/store/v2alpha1/smt/store.go b/store/v2alpha1/smt/store.go index 7b43be4d22b6..eb126fa9a27a 100644 --- a/store/v2alpha1/smt/store.go +++ b/store/v2alpha1/smt/store.go @@ -10,6 +10,7 @@ import ( ics23 "github.com/confio/ics23/go" "github.com/lazyledger/smt" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) var ( @@ -19,25 +20,18 @@ var ( var ( nodesPrefix = []byte{0} - preimagesPrefix = []byte{1} - valuesPrefix = []byte{2} + valuesPrefix = []byte{1} + preimagesPrefix = []byte{2} errKeyEmpty = errors.New("key is empty or nil") errValueNil = errors.New("value is nil") ) -// StoreParams defines how the SMT structural and value data are accessed internally. -type StoreParams struct { - TreeData dbm.DBReadWriter - ValueData dbm.DBReadWriter -} - // Store Implements types.BasicKVStore. type Store struct { - tree smt.SparseMerkleTree - // Maps value hash back to preimage - values dbm.DBReadWriter - // Maps hashed key (tree path) back to preimage + tree *smt.SparseMerkleTree + values dbm.DBReader + // Map hashed keys back to preimage preimages dbm.DBReadWriter } @@ -45,34 +39,40 @@ type Store struct { // smt.SparseMerkleTree expects this error to be returned when a key is not found type dbMapStore struct{ dbm.DBReadWriter } -func NewStore(par StoreParams) *Store { - nodes := prefix.NewPrefixReadWriter(par.TreeData, nodesPrefix) - preimages := prefix.NewPrefixReadWriter(par.TreeData, preimagesPrefix) - values := par.ValueData - if values == nil { - values = prefix.NewPrefixReadWriter(par.TreeData, valuesPrefix) - } +func NewStore(db dbm.DBReadWriter) *Store { + nodes := prefix.NewPrefixReadWriter(db, nodesPrefix) + values := prefix.NewPrefixReadWriter(db, valuesPrefix) + preimages := prefix.NewPrefixReadWriter(db, preimagesPrefix) return &Store{ - tree: smt.NewSMT(dbMapStore{nodes}, sha256.New()), + tree: smt.NewSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New()), values: values, preimages: preimages, } } -func LoadStore(par StoreParams, root []byte) *Store { - nodes := prefix.NewPrefixReadWriter(par.TreeData, nodesPrefix) - preimages := prefix.NewPrefixReadWriter(par.TreeData, preimagesPrefix) - values := par.ValueData - if values == nil { - values = prefix.NewPrefixReadWriter(par.TreeData, valuesPrefix) - } +func LoadStore(db dbm.DBReadWriter, root []byte) *Store { + nodes := prefix.NewPrefixReadWriter(db, nodesPrefix) + values := prefix.NewPrefixReadWriter(db, valuesPrefix) + preimages := prefix.NewPrefixReadWriter(db, preimagesPrefix) return &Store{ - tree: smt.ImportSMT(dbMapStore{nodes}, sha256.New(), root), + tree: smt.ImportSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New(), root), values: values, preimages: preimages, } } +func (s *Store) GetProof(key []byte) (*tmcrypto.ProofOps, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } + proof, err := s.tree.Prove(key) + if err != nil { + return nil, err + } + op := NewProofOp(s.tree.Root(), key, SHA256, proof) + return &tmcrypto.ProofOps{Ops: []tmcrypto.ProofOp{op.ProofOp()}}, nil +} + func (s *Store) GetProofICS23(key []byte) (*ics23.CommitmentProof, error) { return createIcs23Proof(s, key) } @@ -86,7 +86,7 @@ func (s *Store) Get(key []byte) []byte { if len(key) == 0 { panic(errKeyEmpty) } - val, err := s.values.Get(key) + val, err := s.tree.Get(key) if err != nil { panic(err) } @@ -98,7 +98,7 @@ func (s *Store) Has(key []byte) bool { if len(key) == 0 { panic(errKeyEmpty) } - has, err := s.values.Has(key) + has, err := s.tree.Has(key) if err != nil { panic(err) } @@ -113,17 +113,12 @@ func (s *Store) Set(key []byte, value []byte) { if value == nil { panic(errValueNil) } - if err := s.tree.Update(key, value); err != nil { - panic(err) - } - if err := s.values.Set(key, value); err != nil { + _, err := s.tree.Update(key, value) + if err != nil { panic(err) } - // TODO: plug into the SMT's hashers path := sha256.Sum256(key) - if err := s.preimages.Set(path[:], key); err != nil { - panic(err) - } + s.preimages.Set(path[:], key) } // Delete deletes the key. Panics on nil key. @@ -131,20 +126,11 @@ func (s *Store) Delete(key []byte) { if len(key) == 0 { panic(errKeyEmpty) } - if err := s.tree.Delete(key); err != nil && err != smt.ErrKeyNotPresent { - panic(err) - } - if err := s.values.Delete(key); err != nil { - panic(err) - } + _, _ = s.tree.Delete(key) path := sha256.Sum256(key) s.preimages.Delete(path[:]) } -func (s *Store) Commit() error { - return s.tree.Commit() -} - func (ms dbMapStore) Get(key []byte) ([]byte, error) { val, err := ms.DBReadWriter.Get(key) if err != nil { diff --git a/store/v2alpha1/smt/store_test.go b/store/v2alpha1/smt/store_test.go index 91511f5d9c95..3024343fbf95 100644 --- a/store/v2alpha1/smt/store_test.go +++ b/store/v2alpha1/smt/store_test.go @@ -11,7 +11,7 @@ import ( func TestGetSetHasDelete(t *testing.T) { db := memdb.NewDB() - s := store.NewStore(store.StoreParams{TreeData: db.ReadWriter()}) + s := store.NewStore(db.ReadWriter()) s.Set([]byte("foo"), []byte("bar")) assert.Equal(t, []byte("bar"), s.Get([]byte("foo"))) @@ -33,15 +33,14 @@ func TestGetSetHasDelete(t *testing.T) { func TestLoadStore(t *testing.T) { db := memdb.NewDB() txn := db.ReadWriter() - s := store.NewStore(store.StoreParams{TreeData: txn}) + s := store.NewStore(txn) s.Set([]byte{0}, []byte{0}) s.Set([]byte{1}, []byte{1}) s.Delete([]byte{1}) root := s.Root() - assert.NoError(t, s.Commit()) - s = store.LoadStore(store.StoreParams{TreeData: txn}, root) + s = store.LoadStore(txn, root) assert.Equal(t, []byte{0}, s.Get([]byte{0})) assert.False(t, s.Has([]byte{1})) s.Set([]byte{2}, []byte{2})