txHandler: batch transactions to the verifier

agreement/asyncVoteVerifier_test.go

@@ -34,6 +34,9 @@ func (fp *expiredExecPool) EnqueueBacklog(enqueueCtx context.Context, t execpool
 	// generate an error, to see if we correctly report that on the verifyVote() call.
 	return context.Canceled
 }
+func (fp *expiredExecPool) BufferSize() (length, capacity int) {
+	return
+}
 
 // Test async vote verifier against a full execution pool.
 func TestVerificationAgainstFullExecutionPool(t *testing.T) {

cmd/goal/clerk.go

@@ -422,7 +422,7 @@ var sendCmd = &cobra.Command{
 					CurrentProtocol: proto,
 				},
 			}
-			groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, blockHeader, nil)
+			groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, &blockHeader, nil)
 			if err == nil {
 				err = verify.LogicSigSanityCheck(&uncheckedTxn, 0, groupCtx)
 			}
@@ -823,7 +823,7 @@ var signCmd = &cobra.Command{
 			}
 			var groupCtx *verify.GroupContext
 			if lsig.Logic != nil {
-				groupCtx, err = verify.PrepareGroupContext(txnGroup, contextHdr, nil)
+				groupCtx, err = verify.PrepareGroupContext(txnGroup, &contextHdr, nil)
 				if err != nil {
 					// this error has to be unsupported protocol
 					reportErrorf("%s: %v", txFilename, err)

crypto/batchverifier.go

@@ -53,7 +53,7 @@ const minBatchVerifierAlloc = 16
 
 // Batch verifications errors
 var (
-	ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification")
+	ErrBatchHasFailedSigs = errors.New("At least one signature didn't pass verification")
 )
 
 //export ed25519_randombytes_unsafe
@@ -104,8 +104,8 @@ func (b *BatchVerifier) expand() {
 	b.signatures = signatures
 }
 
-// getNumberOfEnqueuedSignatures returns the number of signatures current enqueue onto the batch verifier object
-func (b *BatchVerifier) getNumberOfEnqueuedSignatures() int {
+// GetNumberOfEnqueuedSignatures returns the number of signatures currently enqueued into the BatchVerifier
+func (b *BatchVerifier) GetNumberOfEnqueuedSignatures() int {
 	return len(b.messages)
 }
 
@@ -120,18 +120,18 @@ func (b *BatchVerifier) Verify() error {
 // if some signatures are invalid, true will be set in failed at the corresponding indexes, and
 // ErrBatchVerificationFailed for err
 func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) {
-	if b.getNumberOfEnqueuedSignatures() == 0 {
+	if b.GetNumberOfEnqueuedSignatures() == 0 {
 		return nil, nil
 	}
-	var messages = make([][]byte, b.getNumberOfEnqueuedSignatures())
-	for i, m := range b.messages {
-		messages[i] = HashRep(m)
+	var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures())
+	for i := range b.messages {
+		messages[i] = HashRep(b.messages[i])
 	}
 	allValid, failed := batchVerificationImpl(messages, b.publicKeys, b.signatures)
 	if allValid {
 		return failed, nil
 	}
-	return failed, ErrBatchVerificationFailed
+	return failed, ErrBatchHasFailedSigs
 }
 
 // batchVerificationImpl invokes the ed25519 batch verification algorithm.

crypto/batchverifier_test.go

@@ -17,6 +17,7 @@
 package crypto
 
 import (
+	"fmt"
 	"math/rand"
 	"runtime"
 	"testing"
@@ -64,7 +65,7 @@ func TestBatchVerifierBulk(t *testing.T) {
 			sig := sigSecrets.Sign(msg)
 			bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig)
 		}
-		require.Equal(t, n, bv.getNumberOfEnqueuedSignatures())
+		require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures())
 		require.NoError(t, bv.Verify())
 	}
 
@@ -121,6 +122,67 @@ func BenchmarkBatchVerifier(b *testing.B) {
 	require.NoError(b, bv.Verify())
 }
 
+// BenchmarkBatchVerifierBig with b.N over 1000 will report the expected performance
+// gain as the batchsize increases. All sigs are valid.
+func BenchmarkBatchVerifierBig(b *testing.B) {
+	c := makeCurve25519Secret()
+	for batchSize := 1; batchSize <= 96; batchSize++ {
+		bv := MakeBatchVerifierWithHint(batchSize)
+		for i := 0; i < batchSize; i++ {
+			str := randString()
+			bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str))
+		}
+		b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) {
+			totalTransactions := b.N
+			count := totalTransactions / batchSize
+			if count*batchSize < totalTransactions {
+				count++
+			}
+			for x := 0; x < count; x++ {
+				require.NoError(b, bv.Verify())
+			}
+		})
+	}
+}
+
+// BenchmarkBatchVerifierBigWithInvalid builds over BenchmarkBatchVerifierBig by introducing
+// invalid sigs to even numbered batch sizes. This shows the impact of invalid sigs on the
+// performance. Basically, all the gains from batching disappear.
+func BenchmarkBatchVerifierBigWithInvalid(b *testing.B) {
+	c := makeCurve25519Secret()
+	badSig := Signature{}
+	for batchSize := 1; batchSize <= 96; batchSize++ {
+		bv := MakeBatchVerifierWithHint(batchSize)
+		for i := 0; i < batchSize; i++ {
+			str := randString()
+			if batchSize%2 == 0 && (i == 0 || rand.Float32() < 0.1) {
+				bv.EnqueueSignature(c.SignatureVerifier, str, badSig)
+			} else {
+				bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str))
+			}
+		}
+		b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) {
+			totalTransactions := b.N
+			count := totalTransactions / batchSize
+			if count*batchSize < totalTransactions {
+				count++
+			}
+			for x := 0; x < count; x++ {
+				failed, err := bv.VerifyWithFeedback()
+				if err != nil {
+					for i, f := range failed {
+						if bv.signatures[i] == badSig {
+							require.True(b, f)
+						} else {
+							require.False(b, f)
+						}
+					}
+				}
+			}
+		})
+	}
+}
+
 func TestEmpty(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	bv := MakeBatchVerifier()
@@ -155,10 +217,10 @@ func TestBatchVerifierIndividualResults(t *testing.T) {
 			}
 			bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig)
 		}
-		require.Equal(t, n, bv.getNumberOfEnqueuedSignatures())
+		require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures())
 		failed, err := bv.VerifyWithFeedback()
 		if hasBadSig {
-			require.ErrorIs(t, err, ErrBatchVerificationFailed)
+			require.ErrorIs(t, err, ErrBatchHasFailedSigs)
 		} else {
 			require.NoError(t, err)
 		}
@@ -185,10 +247,10 @@ func TestBatchVerifierIndividualResultsAllValid(t *testing.T) {
 			sig := sigSecrets.Sign(msg)
 			bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig)
 		}
-		require.Equal(t, n, bv.getNumberOfEnqueuedSignatures())
+		require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures())
 		failed, err := bv.VerifyWithFeedback()
 		require.NoError(t, err)
-		require.Equal(t, bv.getNumberOfEnqueuedSignatures(), len(failed))
+		require.Equal(t, bv.GetNumberOfEnqueuedSignatures(), len(failed))
 		for _, f := range failed {
 			require.False(t, f)
 		}

data/ledger.go

@@ -80,7 +80,7 @@ type roundSeed struct {
 func LoadLedger(
 	log logging.Logger, dbFilenamePrefix string, memory bool,
 	genesisProto protocol.ConsensusVersion, genesisBal bookkeeping.GenesisBalances, genesisID string, genesisHash crypto.Digest,
-	blockListeners []ledger.BlockListener, cfg config.Local,
+	blockListeners []ledgercore.BlockListener, cfg config.Local,
 ) (*Ledger, error) {
 	if genesisBal.Balances == nil {
 		genesisBal.Balances = make(map[basics.Address]basics.AccountData)

data/transactions/verify/artifact_test.go

@@ -77,7 +77,7 @@ func BenchmarkTinyMan(b *testing.B) {
 
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, err := TxnGroup(stxnss[i], hdr, nil, &logic.NoHeaderLedger{})
+			_, err := TxnGroup(stxnss[i], &hdr, nil, &logic.NoHeaderLedger{})
 			require.NoError(b, err)
 		}
 	})
@@ -93,7 +93,7 @@ func BenchmarkTinyMan(b *testing.B) {
 
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, err := TxnGroup(stxns, hdr, nil, &logic.NoHeaderLedger{})
+			_, err := TxnGroup(stxns, &hdr, nil, &logic.NoHeaderLedger{})
 			require.NoError(b, err)
 		}
 	})