From 6de685805dfcc4b3d80de0e2774b3753981eb74c Mon Sep 17 00:00:00 2001 From: Robert Zaremba Date: Thu, 10 Dec 2020 20:02:17 +0100 Subject: [PATCH] Snapshot sync: use hasher for chunk hashes (#7215) (#7259) * snapshot sync: use blake3 for chunk hashes (7215) Blake3 improves on security and speed compared to sha256. https://blake3.io/ In this PR: + use optimized blake3 hashes (with dedicated SIMD code) instead of sha256 + reuse resources on hashing chunks. + cleaned few error return statements. * linter issues fixes * revert blake2 hashing to sha256 * revert go.mod Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- snapshots/helpers_test.go | 14 +++++------ snapshots/manager_test.go | 7 ++---- snapshots/store.go | 47 ++++++++--------------------------- snapshots/store_test.go | 17 ++++--------- store/rootmulti/store_test.go | 3 ++- 5 files changed, 25 insertions(+), 63 deletions(-) diff --git a/snapshots/helpers_test.go b/snapshots/helpers_test.go index dcb7dfabffd2..5dbdd317536a 100644 --- a/snapshots/helpers_test.go +++ b/snapshots/helpers_test.go @@ -16,15 +16,13 @@ import ( "github.com/cosmos/cosmos-sdk/snapshots/types" ) -func checksum(b []byte) []byte { - hash := sha256.Sum256(b) - return hash[:] -} - func checksums(slice [][]byte) [][]byte { - checksums := [][]byte{} - for _, chunk := range slice { - checksums = append(checksums, checksum(chunk)) + hasher := sha256.New() + checksums := make([][]byte, len(slice)) + for i, chunk := range slice { + hasher.Write(chunk) + checksums[i] = hasher.Sum(nil) + hasher.Reset() } return checksums } diff --git a/snapshots/manager_test.go b/snapshots/manager_test.go index 662dbc383388..6069666f14f5 100644 --- a/snapshots/manager_test.go +++ b/snapshots/manager_test.go @@ -79,11 +79,8 @@ func TestManager_Take(t *testing.T) { Chunks: 3, Hash: []uint8{0x47, 0xe4, 0xee, 0x7f, 0x21, 0x1f, 0x73, 0x26, 0x5d, 0xd1, 0x76, 0x58, 0xf6, 0xe2, 0x1c, 0x13, 0x18, 0xbd, 0x6c, 0x81, 0xf3, 0x75, 0x98, 0xe2, 0xa, 0x27, 0x56, 0x29, 0x95, 0x42, 0xef, 0xcf}, Metadata: types.Metadata{ - ChunkHashes: [][]byte{ - checksum([]byte{1, 2, 3}), - checksum([]byte{4, 5, 6}), - checksum([]byte{7, 8, 9}), - }, + ChunkHashes: checksums([][]byte{ + {1, 2, 3}, {4, 5, 6}, {7, 8, 9}}), }, }, snapshot) diff --git a/snapshots/store.go b/snapshots/store.go index 687653a220fd..e0bcbe6bd570 100644 --- a/snapshots/store.go +++ b/snapshots/store.go @@ -63,11 +63,8 @@ func (s *Store) Delete(height uint64, format uint32) error { height, format) } err = os.RemoveAll(s.pathSnapshot(height, format)) - if err != nil { - return sdkerrors.Wrapf(err, "failed to delete snapshot chunks for height %v format %v", - height, format) - } - return nil + return sdkerrors.Wrapf(err, "failed to delete snapshot chunks for height %v format %v", + height, format) } // Get fetches snapshot info from the database. @@ -109,10 +106,7 @@ func (s *Store) GetLatest() (*types.Snapshot, error) { } } err = iter.Error() - if err != nil { - return nil, sdkerrors.Wrap(err, "failed to find latest snapshot") - } - return snapshot, nil + return snapshot, sdkerrors.Wrap(err, "failed to find latest snapshot") } // List lists snapshots, in reverse order (newest first). @@ -132,23 +126,16 @@ func (s *Store) List() ([]*types.Snapshot, error) { } snapshots = append(snapshots, snapshot) } - err = iter.Error() - if err != nil { - return nil, err - } - return snapshots, nil + return snapshots, iter.Error() } // Load loads a snapshot (both metadata and binary chunks). The chunks must be consumed and closed. // Returns nil if the snapshot does not exist. func (s *Store) Load(height uint64, format uint32) (*types.Snapshot, <-chan io.ReadCloser, error) { snapshot, err := s.Get(height, format) - if err != nil { + if snapshot == nil || err != nil { return nil, nil, err } - if snapshot == nil { - return nil, nil, nil - } ch := make(chan io.ReadCloser) go func() { @@ -189,11 +176,7 @@ func (s *Store) LoadChunk(height uint64, format uint32, chunk uint32) (io.ReadCl // loadChunkFile loads a chunk from disk, and errors if it does not exist. func (s *Store) loadChunkFile(height uint64, format uint32, chunk uint32) (io.ReadCloser, error) { path := s.pathChunk(height, format, chunk) - file, err := os.Open(path) - if err != nil { - return nil, err - } - return file, nil + return os.Open(path) } // Prune removes old snapshots. The given number of most recent heights (regardless of format) are retained. @@ -233,11 +216,7 @@ func (s *Store) Prune(retain uint32) (uint64, error) { } } } - err = iter.Error() - if err != nil { - return 0, err - } - return pruned, nil + return pruned, iter.Error() } // Save saves a snapshot to disk, returning it. @@ -292,6 +271,7 @@ func (s *Store) Save( return nil, sdkerrors.Wrapf(err, "failed to create snapshot chunk file %q", path) } defer file.Close() // nolint: staticcheck + chunkHasher.Reset() _, err = io.Copy(io.MultiWriter(file, chunkHasher, snapshotHasher), chunkBody) if err != nil { @@ -310,11 +290,7 @@ func (s *Store) Save( } snapshot.Chunks = index snapshot.Hash = snapshotHasher.Sum(nil) - err = s.saveSnapshot(snapshot) - if err != nil { - return nil, err - } - return snapshot, nil + return snapshot, s.saveSnapshot(snapshot) } // saveSnapshot saves snapshot metadata to the database. @@ -324,10 +300,7 @@ func (s *Store) saveSnapshot(snapshot *types.Snapshot) error { return sdkerrors.Wrap(err, "failed to encode snapshot metadata") } err = s.db.SetSync(encodeKey(snapshot.Height, snapshot.Format), value) - if err != nil { - return sdkerrors.Wrap(err, "failed to store snapshot") - } - return nil + return sdkerrors.Wrap(err, "failed to store snapshot") } // pathHeight generates the path to a height, containing multiple snapshot formats. diff --git a/snapshots/store_test.go b/snapshots/store_test.go index 5c0b77b8536d..1a04a393c1ed 100644 --- a/snapshots/store_test.go +++ b/snapshots/store_test.go @@ -112,10 +112,8 @@ func TestStore_Get(t *testing.T) { Chunks: 2, Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), Metadata: types.Metadata{ - ChunkHashes: [][]byte{ - checksum([]byte{2, 1, 0}), - checksum([]byte{2, 1, 1}), - }, + ChunkHashes: checksums([][]byte{ + {2, 1, 0}, {2, 1, 1}}), }, }, snapshot) } @@ -182,10 +180,8 @@ func TestStore_Load(t *testing.T) { Chunks: 2, Hash: hash([][]byte{{2, 1, 0}, {2, 1, 1}}), Metadata: types.Metadata{ - ChunkHashes: [][]byte{ - checksum([]byte{2, 1, 0}), - checksum([]byte{2, 1, 1}), - }, + ChunkHashes: checksums([][]byte{ + {2, 1, 0}, {2, 1, 1}}), }, }, snapshot) @@ -275,10 +271,7 @@ func TestStore_Save(t *testing.T) { Chunks: 2, Hash: hash([][]byte{{1}, {2}}), Metadata: types.Metadata{ - ChunkHashes: [][]byte{ - checksum([]byte{1}), - checksum([]byte{2}), - }, + ChunkHashes: checksums([][]byte{{1}, {2}}), }, }, snapshot) loaded, err := store.Get(snapshot.Height, snapshot.Format) diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index 7934b514d168..e4654f410d0d 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -566,8 +566,9 @@ func TestMultistoreSnapshot_Checksum(t *testing.T) { chunks, err := store.Snapshot(version, tc.format) require.NoError(t, err) hashes := []string{} + hasher := sha256.New() for chunk := range chunks { - hasher := sha256.New() + hasher.Reset() _, err := io.Copy(hasher, chunk) require.NoError(t, err) hashes = append(hashes, hex.EncodeToString(hasher.Sum(nil)))