Snapshot sync: use hasher for chunk hashes (#7215)

go.mod

@@ -42,6 +42,7 @@ require (
 	github.com/tendermint/go-amino v0.15.1
 	github.com/tendermint/tendermint v0.34.0-rc3
 	github.com/tendermint/tm-db v0.6.2
+	github.com/zeebo/blake3 v0.0.4
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	google.golang.org/genproto v0.0.0-20200825200019-8632dd797987
 	google.golang.org/grpc v1.31.1

go.sum

@@ -593,6 +593,11 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
+github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
+github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
 github.com/zondax/hid v0.9.0 h1:eiT3P6vNxAEVxXMw66eZUAAnU2zD33JBkfG/EnfAKl8=
 github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

snapshots/helpers_test.go

@@ -2,7 +2,6 @@ package snapshots_test
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"errors"
 	"io"
 	"io/ioutil"
@@ -12,26 +11,25 @@ import (
 
 	"github.com/stretchr/testify/require"
 	db "github.com/tendermint/tm-db"
+	"github.com/zeebo/blake3"
 
 	"github.com/cosmos/cosmos-sdk/snapshots"
 	"github.com/cosmos/cosmos-sdk/snapshots/types"
 )
 
-func checksum(b []byte) []byte {
-	hash := sha256.Sum256(b)
-	return hash[:]
-}
-
 func checksums(slice [][]byte) [][]byte {
-	checksums := [][]byte{}
-	for _, chunk := range slice {
-		checksums = append(checksums, checksum(chunk))
+	hasher := blake3.New()
+	checksums := make([][]byte, len(slice))
+	for i, chunk := range slice {
+		hasher.Write(chunk)
+		checksums[i] = hasher.Sum(nil)
+		hasher.Reset()
 	}
 	return checksums
 }
 
 func hash(chunks [][]byte) []byte {
-	hasher := sha256.New()
+	hasher := blake3.New()
 	for _, chunk := range chunks {
 		hasher.Write(chunk)
 	}

snapshots/manager.go

@@ -2,13 +2,13 @@ package snapshots
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"io"
 	"io/ioutil"
 	"sync"
 
 	"github.com/cosmos/cosmos-sdk/snapshots/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
+	"github.com/zeebo/blake3"
 )
 
 const (
@@ -231,9 +231,11 @@ func (m *Manager) RestoreChunk(chunk []byte) (bool, error) {
 	}
 
 	// Verify the chunk hash.
-	hash := sha256.Sum256(chunk)
+	hasher := blake3.New()
+	hasher.Write(chunk)
+	hash := hasher.Sum(nil)
 	expected := m.restoreChunkHashes[m.restoreChunkIndex]
-	if !bytes.Equal(hash[:], expected) {
+	if !bytes.Equal(hash, expected) {
 		return false, sdkerrors.Wrapf(types.ErrChunkHashMismatch,
 			"expected %x, got %x", hash, expected)
 	}

snapshots/manager_test.go

@@ -82,13 +82,10 @@ func TestManager_Take(t *testing.T) {
 		Height: 5,
 		Format: types.CurrentFormat,
 		Chunks: 3,
-		Hash:   []uint8{0x47, 0xe4, 0xee, 0x7f, 0x21, 0x1f, 0x73, 0x26, 0x5d, 0xd1, 0x76, 0x58, 0xf6, 0xe2, 0x1c, 0x13, 0x18, 0xbd, 0x6c, 0x81, 0xf3, 0x75, 0x98, 0xe2, 0xa, 0x27, 0x56, 0x29, 0x95, 0x42, 0xef, 0xcf},
+		Hash:   []uint8{0x6a, 0xaf, 0x8d, 0x61, 0xb6, 0x4a, 0xbd, 0xd5, 0xcc, 0x79, 0xe9, 0x11, 0x22, 0x7e, 0x8d, 0x1, 0x7f, 0xcf, 0xa1, 0xe6, 0xfa, 0xaf, 0xc8, 0x55, 0x4c, 0x1, 0x9c, 0xad, 0xb, 0x4b, 0xd9, 0xf8},
 		Metadata: types.Metadata{
-			ChunkHashes: [][]byte{
-				checksum([]byte{1, 2, 3}),
-				checksum([]byte{4, 5, 6}),
-				checksum([]byte{7, 8, 9}),
-			},
+			ChunkHashes: checksums([][]byte{
+				{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}),
 		},
 	}, snapshot)
 

snapshots/store.go

@@ -1,7 +1,6 @@
 package snapshots
 
 import (
-	"crypto/sha256"
 	"encoding/binary"
 	"io"
 	"math"
@@ -12,6 +11,7 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	db "github.com/tendermint/tm-db"
+	"github.com/zeebo/blake3"
 
 	"github.com/cosmos/cosmos-sdk/snapshots/types"
 	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
@@ -63,11 +63,8 @@ func (s *Store) Delete(height uint64, format uint32) error {
 			height, format)
 	}
 	err = os.RemoveAll(s.pathSnapshot(height, format))
-	if err != nil {
-		return sdkerrors.Wrapf(err, "failed to delete snapshot chunks for height %v format %v",
-			height, format)
-	}
-	return nil
+	return sdkerrors.Wrapf(err, "failed to delete snapshot chunks for height %v format %v",
+		height, format)
 }
 
 // Get fetches snapshot info from the database.
@@ -109,10 +106,7 @@ func (s *Store) GetLatest() (*types.Snapshot, error) {
 		}
 	}
 	err = iter.Error()
-	if err != nil {
-		return nil, sdkerrors.Wrap(err, "failed to find latest snapshot")
-	}
-	return snapshot, nil
+	return snapshot, sdkerrors.Wrap(err, "failed to find latest snapshot")
 }
 
 // List lists snapshots, in reverse order (newest first).
@@ -132,23 +126,16 @@ func (s *Store) List() ([]*types.Snapshot, error) {
 		}
 		snapshots = append(snapshots, snapshot)
 	}
-	err = iter.Error()
-	if err != nil {
-		return nil, err
-	}
-	return snapshots, nil
+	return snapshots, iter.Error()
 }
 
 // Load loads a snapshot (both metadata and binary chunks). The chunks must be consumed and closed.
 // Returns nil if the snapshot does not exist.
 func (s *Store) Load(height uint64, format uint32) (*types.Snapshot, <-chan io.ReadCloser, error) {
 	snapshot, err := s.Get(height, format)
-	if err != nil {
+	if snapshot == nil || err != nil {
 		return nil, nil, err
 	}
-	if snapshot == nil {
-		return nil, nil, nil
-	}
 
 	ch := make(chan io.ReadCloser)
 	go func() {
@@ -189,11 +176,7 @@ func (s *Store) LoadChunk(height uint64, format uint32, chunk uint32) (io.ReadCl
 // loadChunkFile loads a chunk from disk, and errors if it does not exist.
 func (s *Store) loadChunkFile(height uint64, format uint32, chunk uint32) (io.ReadCloser, error) {
 	path := s.pathChunk(height, format, chunk)
-	file, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	return file, nil
+	return os.Open(path)
 }
 
 // Prune removes old snapshots. The given number of most recent heights (regardless of format) are retained.
@@ -233,11 +216,7 @@ func (s *Store) Prune(retain uint32) (uint64, error) {
 			}
 		}
 	}
-	err = iter.Error()
-	if err != nil {
-		return 0, err
-	}
-	return pruned, nil
+	return pruned, iter.Error()
 }
 
 // Save saves a snapshot to disk, returning it.
@@ -277,8 +256,8 @@ func (s *Store) Save(
 		Format: format,
 	}
 	index := uint32(0)
-	snapshotHasher := sha256.New()
-	chunkHasher := sha256.New()
+	snapshotHasher := blake3.New()
+	chunkHasher := blake3.New()
 	for chunkBody := range chunks {
 		defer chunkBody.Close() // nolint: staticcheck
 		dir := s.pathSnapshot(height, format)
@@ -292,6 +271,7 @@ func (s *Store) Save(
 			return nil, sdkerrors.Wrapf(err, "failed to create snapshot chunk file %q", path)
 		}
 		defer file.Close() // nolint: staticcheck
+
 		chunkHasher.Reset()
 		_, err = io.Copy(io.MultiWriter(file, chunkHasher, snapshotHasher), chunkBody)
 		if err != nil {
@@ -310,11 +290,7 @@ func (s *Store) Save(
 	}
 	snapshot.Chunks = index
 	snapshot.Hash = snapshotHasher.Sum(nil)
-	err = s.saveSnapshot(snapshot)
-	if err != nil {
-		return nil, err
-	}
-	return snapshot, nil
+	return snapshot, s.saveSnapshot(snapshot)
 }
 
 // saveSnapshot saves snapshot metadata to the database.
@@ -324,10 +300,7 @@ func (s *Store) saveSnapshot(snapshot *types.Snapshot) error {
 		return sdkerrors.Wrap(err, "failed to encode snapshot metadata")
 	}
 	err = s.db.SetSync(encodeKey(snapshot.Height, snapshot.Format), value)
-	if err != nil {
-		return sdkerrors.Wrap(err, "failed to store snapshot")
-	}
-	return nil
+	return sdkerrors.Wrap(err, "failed to store snapshot")
 }
 
 // pathHeight generates the path to a height, containing multiple snapshot formats.

snapshots/store_test.go

@@ -131,10 +131,8 @@ func TestStore_Get(t *testing.T) {
 		Chunks: 2,
 		Hash:   hash([][]byte{{2, 1, 0}, {2, 1, 1}}),
 		Metadata: types.Metadata{
-			ChunkHashes: [][]byte{
-				checksum([]byte{2, 1, 0}),
-				checksum([]byte{2, 1, 1}),
-			},
+			ChunkHashes: checksums([][]byte{
+				{2, 1, 0}, {2, 1, 1}}),
 		},
 	}, snapshot)
 }
@@ -207,10 +205,8 @@ func TestStore_Load(t *testing.T) {
 		Chunks: 2,
 		Hash:   hash([][]byte{{2, 1, 0}, {2, 1, 1}}),
 		Metadata: types.Metadata{
-			ChunkHashes: [][]byte{
-				checksum([]byte{2, 1, 0}),
-				checksum([]byte{2, 1, 1}),
-			},
+			ChunkHashes: checksums([][]byte{
+				{2, 1, 0}, {2, 1, 1}}),
 		},
 	}, snapshot)
 
@@ -306,10 +302,7 @@ func TestStore_Save(t *testing.T) {
 		Chunks: 2,
 		Hash:   hash([][]byte{{1}, {2}}),
 		Metadata: types.Metadata{
-			ChunkHashes: [][]byte{
-				checksum([]byte{1}),
-				checksum([]byte{2}),
-			},
+			ChunkHashes: checksums([][]byte{{1}, {2}}),
 		},
 	}, snapshot)
 	loaded, err := store.Get(snapshot.Height, snapshot.Format)

store/rootmulti/store_test.go

@@ -1,7 +1,6 @@
 package rootmulti
 
 import (
-	"crypto/sha256"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
@@ -15,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 	abci "github.com/tendermint/tendermint/abci/types"
 	dbm "github.com/tendermint/tm-db"
+	"github.com/zeebo/blake3"
 
 	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
 	"github.com/cosmos/cosmos-sdk/store/iavl"
@@ -527,12 +527,12 @@ func TestMultistoreSnapshot_Checksum(t *testing.T) {
 		chunkHashes []string
 	}{
 		{1, []string{
-			"503e5b51b657055b77e88169fadae543619368744ad15f1de0736c0a20482f24",
-			"e1a0daaa738eeb43e778aefd2805e3dd720798288a410b06da4b8459c4d8f72e",
-			"aa048b4ee0f484965d7b3b06822cf0772cdcaad02f3b1b9055e69f2cb365ef3c",
-			"7921eaa3ed4921341e504d9308a9877986a879fe216a099c86e8db66fcba4c63",
-			"a4a864e6c02c9fca5837ec80dc84f650b25276ed7e4820cf7516ced9f9901b86",
-			"ca2879ac6e7205d257440131ba7e72bef784cd61642e32b847729e543c1928b9",
+			"549f0bf509242ec785297db0a565f177215dd06985e1ec11960d9f675a387bbf",
+			"9358f48120b936f3e821439dfbeefb6dd71c3d60168e59c35319f96ad30ffd5e",
+			"349f477bf4225ac2a09d83963e7b12c78d1dd6fb50a810d6c6b4f80d74a4fdc8",
+			"418027f332603531248015634e2613d399b531f3d49e88ef93a85ae79c0cf8ed",
+			"6de71204263d43fded1216142382952b0baa0bcc899ddcc97bc7d7c622ff1dcb",
+			"d26306884098a2875ff1c4da340270a817c8de9c2d4c60369c37089e40e9b9ae",
 		}},
 	}
 	for _, tc := range testcases {
@@ -541,8 +541,9 @@ func TestMultistoreSnapshot_Checksum(t *testing.T) {
 			chunks, err := store.Snapshot(version, tc.format)
 			require.NoError(t, err)
 			hashes := []string{}
+			hasher := blake3.New()
 			for chunk := range chunks {
-				hasher := sha256.New()
+				hasher.Reset()
 				_, err := io.Copy(hasher, chunk)
 				require.NoError(t, err)
 				hashes = append(hashes, hex.EncodeToString(hasher.Sum(nil)))

x/slashing/client/rest/grpc_query_test.go

@@ -67,7 +67,7 @@ func (s *IntegrationTestSuite) TestGRPCQueries() {
 			&types.QuerySigningInfosResponse{},
 			&types.QuerySigningInfosResponse{
 				Info: []types.ValidatorSigningInfo{
-					types.ValidatorSigningInfo{
+					{
 						Address:     sdk.ConsAddress(val.PubKey.Address()),
 						JailedUntil: time.Unix(0, 0),
 					},