git

.github/workflows/coverage.yml

@@ -4,7 +4,6 @@ on:
   push:
     paths:
       - "**.go"
-      - "!test/"
     branches:
       - master
       - release/**
@@ -89,7 +88,7 @@ jobs:
           go-version: 1.16
       - name: test & coverage report creation
         run: |
-          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic
+          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out
         if: env.GIT_DIFF
       - uses: actions/upload-artifact@v2
         with:
@@ -125,7 +124,7 @@ jobs:
           name: "${{ github.sha }}-03-coverage"
         if: env.GIT_DIFF
       - run: |
-          cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt
+          cat ./*profile.out | grep -v "mode: set" >> coverage.txt
         if: env.GIT_DIFF
       - uses: codecov/[email protected]
         with:

CHANGELOG_PENDING.md

@@ -1,10 +1,12 @@
 # Unreleased Changes
 
+Friendly reminder: We have a [bug bounty program](https://hackerone.com/cosmos).
+
 ## vX.X
 
-Special thanks to external contributors on this release:
+Month, DD, YYYY
 
-Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermint).
+Special thanks to external contributors on this release:
 
 ### BREAKING CHANGES
 
@@ -32,4 +34,3 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
 ### IMPROVEMENTS
 
 ### BUG FIXES
-

go.mod

@@ -41,5 +41,6 @@ require (
 	golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	google.golang.org/grpc v1.41.0
+	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	pgregory.net/rapid v0.4.7
 )

go.sum

@@ -147,14 +147,6 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
-github.com/celestiaorg/go-leopard v0.1.0 h1:28z2EkvKJIez5J9CEaiiUEC+OxalRLtTGJJ1oScfE1g=
-github.com/celestiaorg/go-leopard v0.1.0/go.mod h1:NtO/rjlB8dw2aq7jr06vZFKGvryQcTDXaNHelmPNOAM=
-github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=
-github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA=
-github.com/celestiaorg/nmt v0.7.0 h1:XRYh7F7TH6ewD3Gybk1L1zjyvgkWPt1iGqWtbMtB2Lk=
-github.com/celestiaorg/nmt v0.7.0/go.mod h1:3bqzTj8xKj0DgQUpOgZzoxvtNkC3MS/hTbQ6dn8SIa0=
-github.com/celestiaorg/rsmt2d v0.3.0 h1:c82dh9J3DBcIQu9OND+5HViUKb5NmpKYVvf8z3zjlfA=
-github.com/celestiaorg/rsmt2d v0.3.0/go.mod h1:2Frw4GEYUnVu6Mvlo+CUzuC2/8wn+zLwVVtp+muN6vg=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
@@ -1035,7 +1027,6 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

internal/p2p/pex/pex_reactor.go

@@ -349,11 +349,10 @@ func (r *Reactor) receiveRequest(src Peer) error {
 // request out for this peer.
 func (r *Reactor) RequestAddrs(p Peer) {
 	id := string(p.ID())
-	if r.requestsSent.Has(id) {
+	if _, exists := r.requestsSent.GetOrSet(id, struct{}{}); exists {
 		return
 	}
 	r.Logger.Debug("Request addrs", "from", p)
-	r.requestsSent.Set(id, struct{}{})
 	p.Send(PexChannel, mustEncode(&tmp2p.PexRequest{}))
 }
 

internal/p2p/switch.go

@@ -432,10 +432,9 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) {
 //  - ie. if we're getting ErrDuplicatePeer we can stop
 //  	because the addrbook got us the peer back already
 func (sw *Switch) reconnectToPeer(addr *NetAddress) {
-	if sw.reconnecting.Has(string(addr.ID)) {
+	if _, exists := sw.reconnecting.GetOrSet(string(addr.ID), addr); exists {
 		return
 	}
-	sw.reconnecting.Set(string(addr.ID), addr)
 	defer sw.reconnecting.Delete(string(addr.ID))
 
 	start := time.Now()

internal/statesync/reactor.go

@@ -1006,14 +1006,26 @@ func (r *Reactor) waitForEnoughPeers(ctx context.Context, numPeers int) error {
 	startAt := time.Now()
 	t := time.NewTicker(100 * time.Millisecond)
 	defer t.Stop()
+	logT := time.NewTicker(time.Minute)
+	defer logT.Stop()
+	var iter int
 	for r.peers.Len() < numPeers {
+		iter++
 		select {
 		case <-ctx.Done():
 			return fmt.Errorf("operation canceled while waiting for peers after %s", time.Since(startAt))
 		case <-r.closeCh:
 			return fmt.Errorf("shutdown while waiting for peers after %s", time.Since(startAt))
 		case <-t.C:
 			continue
+		case <-logT.C:
+			r.Logger.Info("waiting for sufficient peers to start statesync",
+				"duration", time.Since(startAt).String(),
+				"target", numPeers,
+				"peers", r.peers.Len(),
+				"iters", iter,
+			)
+			continue
 		}
 	}
 	return nil

libs/cmap/cmap.go

@@ -22,6 +22,20 @@ func (cm *CMap) Set(key string, value interface{}) {
 	cm.l.Unlock()
 }
 
+// GetOrSet returns the existing value if present. Othewise, it stores `newValue` and returns it.
+func (cm *CMap) GetOrSet(key string, newValue interface{}) (value interface{}, alreadyExists bool) {
+
+	cm.l.Lock()
+	defer cm.l.Unlock()
+
+	if v, ok := cm.m[key]; ok {
+		return v, true
+	}
+
+	cm.m[key] = newValue
+	return newValue, false
+}
+
 func (cm *CMap) Get(key string) interface{} {
 	cm.l.Lock()
 	val := cm.m[key]

libs/cmap/cmap_test.go

@@ -3,6 +3,7 @@ package cmap
 import (
 	"fmt"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -67,3 +68,46 @@ func BenchmarkCMapHas(b *testing.B) {
 		m.Has(string(rune(i)))
 	}
 }
+
+func TestCMap_GetOrSet_Parallel(t *testing.T) {
+
+	tests := []struct {
+		name        string
+		newValue    interface{}
+		parallelism int
+	}{
+		{"test1", "a", 4},
+		{"test2", "a", 40},
+		{"test3", "a", 1},
+	}
+
+	//nolint:scopelint
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cm := NewCMap()
+
+			wg := sync.WaitGroup{}
+			wg.Add(tt.parallelism)
+			for i := 0; i < tt.parallelism; i++ {
+				go func() {
+					defer wg.Done()
+					gotValue, _ := cm.GetOrSet(tt.name, tt.newValue)
+					assert.EqualValues(t, tt.newValue, gotValue)
+				}()
+			}
+			wg.Wait()
+		})
+	}
+}
+
+func TestCMap_GetOrSet_Exists(t *testing.T) {
+	cm := NewCMap()
+
+	gotValue, exists := cm.GetOrSet("key", 1000)
+	assert.False(t, exists)
+	assert.EqualValues(t, 1000, gotValue)
+
+	gotValue, exists = cm.GetOrSet("key", 2000)
+	assert.True(t, exists)
+	assert.EqualValues(t, 1000, gotValue)
+}

test/e2e/generator/generate.go

@@ -63,7 +63,7 @@ var (
 		"restart":    0.1,
 	}
 	evidence = uniformChoice{0, 1, 10}
-	txSize   = uniformChoice{1024, 10240} // either 1kb or 10kb
+	txSize   = uniformChoice{1024, 4096} // either 1kb or 4kb
 	ipv6     = uniformChoice{false, true}
 	keyType  = uniformChoice{types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1}
 )
@@ -73,7 +73,15 @@ func Generate(r *rand.Rand, opts Options) ([]e2e.Manifest, error) {
 	manifests := []e2e.Manifest{}
 	switch opts.P2P {
 	case NewP2PMode, LegacyP2PMode, HybridP2PMode:
+		defer func() {
+			// avoid modifying the global state.
+			original := make([]interface{}, len(testnetCombinations["p2p"]))
+			copy(original, testnetCombinations["p2p"])
+			testnetCombinations["p2p"] = original
+		}()
+
 		testnetCombinations["p2p"] = []interface{}{opts.P2P}
+
 	default:
 		testnetCombinations["p2p"] = []interface{}{NewP2PMode, LegacyP2PMode, HybridP2PMode}
 	}
@@ -153,14 +161,16 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		numValidators = 4
 	case "large":
 		// FIXME Networks are kept small since large ones use too much CPU.
-		numSeeds = r.Intn(2)
-		numLightClients = r.Intn(3)
+		numSeeds = r.Intn(1)
+		numLightClients = r.Intn(2)
 		numValidators = 4 + r.Intn(4)
 		numFulls = r.Intn(4)
 	default:
 		return manifest, fmt.Errorf("unknown topology %q", opt["topology"])
 	}
 
+	const legacyP2PFactor float64 = 0.5
+
 	// First we generate seed nodes, starting at the initial height.
 	for i := 1; i <= numSeeds; i++ {
 		node := generateNode(r, manifest, e2e.ModeSeed, 0, false)
@@ -169,12 +179,18 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() < legacyP2PFactor
 		}
 
 		manifest.Nodes[fmt.Sprintf("seed%02d", i)] = node
 	}
 
+	var (
+		numSyncingNodes = 0
+		hybridNumNew    = 0
+		hybridNumLegacy = 0
+	)
+
 	// Next, we generate validators. We make sure a BFT quorum of validators start
 	// at the initial height, and that we have two archive nodes. We also set up
 	// the initial validator set, and validator set updates for delayed nodes.
@@ -195,7 +211,23 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() < legacyP2PFactor
+			if node.UseLegacyP2P {
+				hybridNumLegacy++
+				if hybridNumNew == 0 {
+					hybridNumNew++
+					hybridNumLegacy--
+					node.UseLegacyP2P = false
+				}
+			} else {
+				hybridNumNew++
+				if hybridNumLegacy == 0 {
+					hybridNumNew--
+					hybridNumLegacy++
+					node.UseLegacyP2P = true
+
+				}
+			}
 		}
 
 		manifest.Nodes[name] = node
@@ -222,7 +254,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 	// Finally, we generate random full nodes.
 	for i := 1; i <= numFulls; i++ {
 		startAt := int64(0)
-		if r.Float64() >= 0.5 {
+		if numSyncingNodes < 2 && r.Float64() >= 0.5 {
+			numSyncingNodes++
 			startAt = nextStartAt
 			nextStartAt += 5
 		}
@@ -232,7 +265,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() > legacyP2PFactor
 		}
 
 		manifest.Nodes[fmt.Sprintf("full%02d", i)] = node
@@ -275,7 +308,17 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		}
 	})
 	for i, name := range peerNames {
-		if len(seedNames) > 0 && (i == 0 || r.Float64() >= 0.5) {
+		// there are seeds, statesync is disabled, and it's
+		// either the first peer by the sort order, and
+		// (randomly half of the remaining peers use a seed
+		// node; otherwise, choose some remaining set of the
+		// peers.
+
+		if len(seedNames) > 0 &&
+			manifest.Nodes[name].StateSync == e2e.StateSyncDisabled &&
+			(i == 0 || r.Float64() >= 0.5) {
+
+			// choose one of the seeds
 			manifest.Nodes[name].Seeds = uniformSetChoice(seedNames).Choose(r)
 		} else if i > 0 {
 			peers := uniformSetChoice(peerNames[:i])