From 3608cf9c1d74a09bcc87f8657810828cac5631e3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Sep 2022 11:07:54 -0400 Subject: [PATCH 001/156] rename and unexport local functions --- data/transactions/verify/txn.go | 48 +++++++++++---------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 885d0882ec..4fbf312973 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -99,42 +99,26 @@ func (g *GroupContext) Equal(other *GroupContext) bool { g.minAvmVersion == other.minAvmVersion } -// Txn verifies a SignedTxn as being signed and having no obviously inconsistent data. +// txnBatchVerifyPrep verifies a SignedTxn having no obviously inconsistent data. // Block-assembly time checks of LogicSig and accounting rules may still block the txn. -func Txn(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext) error { - batchVerifier := crypto.MakeBatchVerifier() - - if err := TxnBatchVerify(s, txnIdx, groupCtx, batchVerifier); err != nil { - return err - } - - // this case is used for comapact certificate where no signature is supplied - if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { - return nil - } - return batchVerifier.Verify() -} - -// TxnBatchVerify verifies a SignedTxn having no obviously inconsistent data. -// Block-assembly time checks of LogicSig and accounting rules may still block the txn. -// it is the caller responsibility to call batchVerifier.verify() -func TxnBatchVerify(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, verifier *crypto.BatchVerifier) error { +// it is the caller responsibility to call batchVerifier.Verify() +func txnBatchVerifyPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, verifier *crypto.BatchVerifier) error { if !groupCtx.consensusParams.SupportRekeying && (s.AuthAddr != basics.Address{}) { - return errors.New("nonempty AuthAddr but rekeying not supported") + return errors.New("nonempty AuthAddr but rekeying is not supported") } if err := s.Txn.WellFormed(groupCtx.specAddrs, groupCtx.consensusParams); err != nil { return err } - return stxnVerifyCore(s, txnIdx, groupCtx, verifier) + return stxnCoreChecks(s, txnIdx, groupCtx, verifier) } // TxnGroup verifies a []SignedTxn as being signed and having no obviously inconsistent data. func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature) (groupCtx *GroupContext, err error) { batchVerifier := crypto.MakeBatchVerifier() - if groupCtx, err = TxnGroupBatchVerify(stxs, contextHdr, cache, ledger, batchVerifier); err != nil { + if groupCtx, err = txnGroupBatchVerifyPrep(stxs, contextHdr, cache, ledger, batchVerifier); err != nil { return nil, err } @@ -149,9 +133,9 @@ func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, return } -// TxnGroupBatchVerify verifies a []SignedTxn having no obviously inconsistent data. -// it is the caller responsibility to call batchVerifier.verify() -func TxnGroupBatchVerify(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { +// txnGroupBatchVerifyPrep verifies a []SignedTxn having no obviously inconsistent data. +// it is the caller responsibility to call batchVerifier.Verify() +func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { groupCtx, err = PrepareGroupContext(stxs, contextHdr, ledger) if err != nil { return nil, err @@ -160,7 +144,7 @@ func TxnGroupBatchVerify(stxs []transactions.SignedTxn, contextHdr bookkeeping.B minFeeCount := uint64(0) feesPaid := uint64(0) for i, stxn := range stxs { - err = TxnBatchVerify(&stxn, i, groupCtx, verifier) + err = txnBatchVerifyPrep(&stxn, i, groupCtx, verifier) if err != nil { err = fmt.Errorf("transaction %+v invalid : %w", stxn, err) return @@ -190,7 +174,7 @@ func TxnGroupBatchVerify(stxs []transactions.SignedTxn, contextHdr bookkeeping.B return } -func stxnVerifyCore(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { +func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { numSigs := 0 hasSig := false hasMsig := false @@ -246,7 +230,7 @@ func stxnVerifyCore(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex func LogicSigSanityCheck(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { batchVerifier := crypto.MakeBatchVerifier() - if err := LogicSigSanityCheckBatchVerify(txn, groupIndex, groupCtx, batchVerifier); err != nil { + if err := logicSigSanityCheckBatchVerifyPrep(txn, groupIndex, groupCtx, batchVerifier); err != nil { return err } @@ -258,10 +242,10 @@ func LogicSigSanityCheck(txn *transactions.SignedTxn, groupIndex int, groupCtx * return batchVerifier.Verify() } -// LogicSigSanityCheckBatchVerify checks that the signature is valid and that the program is basically well formed. +// logicSigSanityCheckBatchVerifyPrep checks that the signature is valid and that the program is basically well formed. // It does not evaluate the logic. -// it is the caller responsibility to call batchVerifier.verify() -func LogicSigSanityCheckBatchVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { +// it is the caller responsibility to call batchVerifier.Verify() +func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { lsig := txn.Lsig if groupCtx.consensusParams.LogicSigVersion == 0 { @@ -404,7 +388,7 @@ func PaysetGroups(ctx context.Context, payset [][]transactions.SignedTxn, blkHea batchVerifier := crypto.MakeBatchVerifierWithHint(len(payset)) for i, signTxnsGrp := range txnGroups { - groupCtxs[i], grpErr = TxnGroupBatchVerify(signTxnsGrp, blkHeader, nil, ledger, batchVerifier) + groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(signTxnsGrp, blkHeader, nil, ledger, batchVerifier) // abort only if it's a non-cache error. if grpErr != nil { return grpErr From 05d63ae87cfd29c23721c653f202cf6c03f3d736 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Sep 2022 11:52:05 -0400 Subject: [PATCH 002/156] update the tests --- data/transactions/verify/txn_test.go | 44 ++++++++++++++++++---------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 3998352114..44108ad5bf 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -51,6 +51,20 @@ var spec = transactions.SpecialAddresses{ RewardsPool: poolAddr, } +func verifyTxn(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext) error { + batchVerifier := crypto.MakeBatchVerifier() + + if err := txnBatchVerifyPrep(s, txnIdx, groupCtx, batchVerifier); err != nil { + return err + } + + // this case is used for comapact certificate where no signature is supplied + if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { + return nil + } + return batchVerifier.Verify() +} + func keypair() *crypto.SignatureSecrets { var seed crypto.Seed crypto.RandBytes(seed[:]) @@ -117,14 +131,14 @@ func TestSignedPayment(t *testing.T) { groupCtx, err := PrepareGroupContext(stxns, blockHeader, nil) require.NoError(t, err) require.NoError(t, payment.WellFormed(spec, proto), "generateTestObjects generated an invalid payment") - require.NoError(t, Txn(&stxn, 0, groupCtx), "generateTestObjects generated a bad signedtxn") + require.NoError(t, verifyTxn(&stxn, 0, groupCtx), "generateTestObjects generated a bad signedtxn") stxn2 := payment.Sign(secret) require.Equal(t, stxn2.Sig, stxn.Sig, "got two different signatures for the same transaction (our signing function is deterministic)") stxn2.MessUpSigForTesting() require.Equal(t, stxn.ID(), stxn2.ID(), "changing sig caused txid to change") - require.Error(t, Txn(&stxn2, 0, groupCtx), "verify succeeded with bad sig") + require.Error(t, verifyTxn(&stxn2, 0, groupCtx), "verify succeeded with bad sig") require.True(t, crypto.SignatureVerifier(addr).Verify(payment, stxn.Sig), "signature on the transaction is not the signature of the hash of the transaction under the spender's key") } @@ -137,7 +151,7 @@ func TestTxnValidationEncodeDecode(t *testing.T) { for _, txn := range signed { groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{txn}, blockHeader, nil) require.NoError(t, err) - if Txn(&txn, 0, groupCtx) != nil { + if verifyTxn(&txn, 0, groupCtx) != nil { t.Errorf("signed transaction %#v did not verify", txn) } @@ -145,7 +159,7 @@ func TestTxnValidationEncodeDecode(t *testing.T) { var signedTx transactions.SignedTxn protocol.Decode(x, &signedTx) - if Txn(&signedTx, 0, groupCtx) != nil { + if verifyTxn(&signedTx, 0, groupCtx) != nil { t.Errorf("signed transaction %#v did not verify", txn) } } @@ -159,14 +173,14 @@ func TestTxnValidationEmptySig(t *testing.T) { for _, txn := range signed { groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{txn}, blockHeader, nil) require.NoError(t, err) - if Txn(&txn, 0, groupCtx) != nil { + if verifyTxn(&txn, 0, groupCtx) != nil { t.Errorf("signed transaction %#v did not verify", txn) } txn.Sig = crypto.Signature{} txn.Msig = crypto.MultisigSig{} txn.Lsig = transactions.LogicSig{} - if Txn(&txn, 0, groupCtx) == nil { + if verifyTxn(&txn, 0, groupCtx) == nil { t.Errorf("transaction %#v verified without sig", txn) } } @@ -205,13 +219,13 @@ func TestTxnValidationStateProof(t *testing.T) { groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{stxn}, blockHeader, nil) require.NoError(t, err) - err = Txn(&stxn, 0, groupCtx) + err = verifyTxn(&stxn, 0, groupCtx) require.NoError(t, err, "state proof txn %#v did not verify", stxn) stxn2 := stxn stxn2.Txn.Type = protocol.PaymentTx stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "payment txn %#v verified from StateProofSender", stxn2) secret := keypair() @@ -219,28 +233,28 @@ func TestTxnValidationStateProof(t *testing.T) { stxn2.Txn.Header.Sender = basics.Address(secret.SignatureVerifier) stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} stxn2 = stxn2.Txn.Sign(secret) - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "state proof txn %#v verified from non-StateProofSender", stxn2) // state proof txns are not allowed to have non-zero values for many fields stxn2 = stxn stxn2.Txn.Header.Fee = basics.MicroAlgos{Raw: proto.MinTxnFee} - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "state proof txn %#v verified", stxn2) stxn2 = stxn stxn2.Txn.Header.Note = []byte{'A'} - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "state proof txn %#v verified", stxn2) stxn2 = stxn stxn2.Txn.Lease[0] = 1 - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "state proof txn %#v verified", stxn2) stxn2 = stxn stxn2.Txn.RekeyTo = basics.Address(secret.SignatureVerifier) - err = Txn(&stxn2, 0, groupCtx) + err = verifyTxn(&stxn2, 0, groupCtx) require.Error(t, err, "state proof txn %#v verified", stxn2) } @@ -258,7 +272,7 @@ func TestDecodeNil(t *testing.T) { // This used to panic when run on a zero value of SignedTxn. groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{st}, blockHeader, nil) require.NoError(t, err) - Txn(&st, 0, groupCtx) + verifyTxn(&st, 0, groupCtx) } } @@ -425,7 +439,7 @@ func BenchmarkTxn(b *testing.B) { groupCtx, err := PrepareGroupContext(txnGroup, blk.BlockHeader, nil) require.NoError(b, err) for i, txn := range txnGroup { - err := Txn(&txn, i, groupCtx) + err := verifyTxn(&txn, i, groupCtx) require.NoError(b, err) } } From 58e4d3f71e0b7bd936cae54a330adbcbe42ab207 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Sep 2022 12:11:31 -0400 Subject: [PATCH 003/156] update a test --- test/e2e-go/upgrades/rekey_support_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e-go/upgrades/rekey_support_test.go b/test/e2e-go/upgrades/rekey_support_test.go index 4988b79c4c..0032e569dc 100644 --- a/test/e2e-go/upgrades/rekey_support_test.go +++ b/test/e2e-go/upgrades/rekey_support_test.go @@ -128,8 +128,8 @@ func TestRekeyUpgrade(t *testing.T) { _, err = client.BroadcastTransaction(rekeyed) // non empty err means the upgrade have not happened yet (as expected), ensure the error if err != nil { - // should be either "nonempty AuthAddr but rekeying not supported" or "txn dead" - if !strings.Contains(err.Error(), "nonempty AuthAddr but rekeying not supported") && + // should be either "nonempty AuthAddr but rekeying is not supported" or "txn dead" + if !strings.Contains(err.Error(), "nonempty AuthAddr but rekeying is not supported") && !strings.Contains(err.Error(), "txn dead") { a.NoErrorf(err, "error message should be one of :\n%s\n%s", "nonempty AuthAddr but rekeying not supported", "txn dead") } From 1e96dd683d346d53a8b96d40c19db34e68cb37b6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Sep 2022 12:19:30 -0400 Subject: [PATCH 004/156] update test err msg --- test/e2e-go/upgrades/rekey_support_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-go/upgrades/rekey_support_test.go b/test/e2e-go/upgrades/rekey_support_test.go index 0032e569dc..56f5b6408b 100644 --- a/test/e2e-go/upgrades/rekey_support_test.go +++ b/test/e2e-go/upgrades/rekey_support_test.go @@ -131,7 +131,7 @@ func TestRekeyUpgrade(t *testing.T) { // should be either "nonempty AuthAddr but rekeying is not supported" or "txn dead" if !strings.Contains(err.Error(), "nonempty AuthAddr but rekeying is not supported") && !strings.Contains(err.Error(), "txn dead") { - a.NoErrorf(err, "error message should be one of :\n%s\n%s", "nonempty AuthAddr but rekeying not supported", "txn dead") + a.NoErrorf(err, "error message should be one of :\n%s\n%s", "nonempty AuthAddr but rekeying is not supported", "txn dead") } } else { // if we had no error it must mean that we've upgraded already. Verify that. From 2db4ee5454ce7016e9b3909c9bb03b430282927d Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 26 Sep 2022 13:18:51 -0400 Subject: [PATCH 005/156] CR updates --- data/transactions/verify/txn.go | 6 ++---- data/transactions/verify/txn_test.go | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4fbf312973..49d56ea5c8 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -220,7 +220,7 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex return errors.New("multisig validation failed") } if hasLogicSig { - return logicSigBatchVerify(s, txnIdx, groupCtx) + return logicSigVerify(s, txnIdx, groupCtx) } return errors.New("has one mystery sig. WAT?") } @@ -314,9 +314,7 @@ func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex return nil } -// logicSigBatchVerify checks that the signature is valid, executing the program. -// it is the caller responsibility to call batchVerifier.verify() -func logicSigBatchVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { +func logicSigVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { err := LogicSigSanityCheck(txn, groupIndex, groupCtx) if err != nil { return err diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 44108ad5bf..6cec7f238d 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -58,7 +58,6 @@ func verifyTxn(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext) er return err } - // this case is used for comapact certificate where no signature is supplied if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { return nil } From b0ef3a0228e8802cde04bf0bed1b328c74aeafda Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 15:28:20 -0400 Subject: [PATCH 006/156] add test and refactoring --- data/transactions/verify/txn.go | 19 ++++--- data/transactions/verify/txn_test.go | 76 +++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4fbf312973..781669fb93 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -118,7 +118,7 @@ func txnBatchVerifyPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupCo func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature) (groupCtx *GroupContext, err error) { batchVerifier := crypto.MakeBatchVerifier() - if groupCtx, err = txnGroupBatchVerifyPrep(stxs, contextHdr, cache, ledger, batchVerifier); err != nil { + if groupCtx, err = txnGroupBatchVerifyPrep(stxs, contextHdr, ledger, batchVerifier); err != nil { return nil, err } @@ -130,12 +130,16 @@ func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, return nil, err } + if cache != nil { + cache.Add(stxs, groupCtx) + } + return } // txnGroupBatchVerifyPrep verifies a []SignedTxn having no obviously inconsistent data. // it is the caller responsibility to call batchVerifier.Verify() -func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { +func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { groupCtx, err = PrepareGroupContext(stxs, contextHdr, ledger) if err != nil { return nil, err @@ -168,9 +172,6 @@ func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeepi return } - if cache != nil { - cache.Add(stxs, groupCtx) - } return } @@ -220,7 +221,7 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex return errors.New("multisig validation failed") } if hasLogicSig { - return logicSigBatchVerify(s, txnIdx, groupCtx) + return logicSigVerify(s, txnIdx, groupCtx) } return errors.New("has one mystery sig. WAT?") } @@ -314,9 +315,7 @@ func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex return nil } -// logicSigBatchVerify checks that the signature is valid, executing the program. -// it is the caller responsibility to call batchVerifier.verify() -func logicSigBatchVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { +func logicSigVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { err := LogicSigSanityCheck(txn, groupIndex, groupCtx) if err != nil { return err @@ -388,7 +387,7 @@ func PaysetGroups(ctx context.Context, payset [][]transactions.SignedTxn, blkHea batchVerifier := crypto.MakeBatchVerifierWithHint(len(payset)) for i, signTxnsGrp := range txnGroups { - groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(signTxnsGrp, blkHeader, nil, ledger, batchVerifier) + groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(signTxnsGrp, blkHeader, ledger, batchVerifier) // abort only if it's a non-cache error. if grpErr != nil { return grpErr diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 44108ad5bf..7bd3c68279 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -58,7 +58,6 @@ func verifyTxn(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext) er return err } - // this case is used for comapact certificate where no signature is supplied if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { return nil } @@ -445,3 +444,78 @@ func BenchmarkTxn(b *testing.B) { } b.StopTimer() } + +// TestTxnGroupCacheUpdate uses TxnGroup to verify txns and add them to the +// cache. Then makes sure that only the valid txns are verified and added to +// the cache. +func TestTxnGroupCacheUpdate(t *testing.T) { + partitiontest.PartitionTest(t) + + _, signedTxn, secrets, addrs := generateTestObjects(100, 20, 50) + blkHdr := bookkeeping.BlockHeader{ + Round: 50, + GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + RewardsState: bookkeeping.RewardsState{ + FeeSink: feeSink, + RewardsPool: poolAddr, + }, + } + + txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + cache := MakeVerifiedTransactionCache(1000) + + // break the signature and see if it fails. + txnGroups[0][0].Sig[0] = txnGroups[0][0].Sig[0] + 1 + + _, err := TxnGroup(txnGroups[0], blkHdr, cache, nil) + require.Error(t, err) + + // The txns should not be in the cache + unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups[:1], spec, protocol.ConsensusCurrentVersion) + require.Equal(t, 1, len(unverifiedGroups)) + + unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) + require.Equal(t, 2, len(unverifiedGroups)) + + _, err = TxnGroup(txnGroups[1], blkHdr, cache, nil) + require.NoError(t, err) + + // Only the second txn should be in the cache + unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) + require.Equal(t, 1, len(unverifiedGroups)) + + // Fix the signature + txnGroups[0][0].Sig[0] = txnGroups[0][0].Sig[0] - 1 + + _, err = TxnGroup(txnGroups[0], blkHdr, cache, nil) + require.NoError(t, err) + + // Both transactions should be in the cache + unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) + require.Equal(t, 0, len(unverifiedGroups)) + + // Break a random signature + txgIdx := rand.Intn(len(txnGroups)) + txIdx := rand.Intn(len(txnGroups[txgIdx])) + txnGroups[txgIdx][txIdx].Sig[0] = txnGroups[0][0].Sig[0] + 1 + + numFailed := 0 + + // Add them to the cache by verifying them + for _, txng := range txnGroups { + _, err = TxnGroup(txng, blkHdr, cache, nil) + if err != nil { + numFailed++ + } + } + require.Equal(t, 1, numFailed) + + // Onle one transaction should not be in cache + unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) + require.Equal(t, 1, len(unverifiedGroups)) + + require.Equal(t, unverifiedGroups[0], txnGroups[txgIdx]) +} From caca719dc80ca6aceb99cd7fbc4afe5132ee8b32 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 16:43:03 -0400 Subject: [PATCH 007/156] CR comments --- data/transactions/verify/txn.go | 1 + data/transactions/verify/txn_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 781669fb93..cd951346e8 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -315,6 +315,7 @@ func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex return nil } +// logicSigVerify checks that the signature is valid, executing the program. func logicSigVerify(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { err := LogicSigSanityCheck(txn, groupIndex, groupCtx) if err != nil { diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 7bd3c68279..05ad1e4659 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -475,17 +475,17 @@ func TestTxnGroupCacheUpdate(t *testing.T) { // The txns should not be in the cache unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups[:1], spec, protocol.ConsensusCurrentVersion) - require.Equal(t, 1, len(unverifiedGroups)) + require.Len(t, unverifiedGroups, 1) unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) - require.Equal(t, 2, len(unverifiedGroups)) + require.Len(t, unverifiedGroups, 2) _, err = TxnGroup(txnGroups[1], blkHdr, cache, nil) require.NoError(t, err) // Only the second txn should be in the cache unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) - require.Equal(t, 1, len(unverifiedGroups)) + require.Len(t, unverifiedGroups, 1) // Fix the signature txnGroups[0][0].Sig[0] = txnGroups[0][0].Sig[0] - 1 @@ -495,7 +495,7 @@ func TestTxnGroupCacheUpdate(t *testing.T) { // Both transactions should be in the cache unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups[:2], spec, protocol.ConsensusCurrentVersion) - require.Equal(t, 0, len(unverifiedGroups)) + require.Len(t, unverifiedGroups, 0) // Break a random signature txgIdx := rand.Intn(len(txnGroups)) @@ -515,7 +515,7 @@ func TestTxnGroupCacheUpdate(t *testing.T) { // Onle one transaction should not be in cache unverifiedGroups = cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) - require.Equal(t, 1, len(unverifiedGroups)) + require.Len(t, unverifiedGroups, 1) require.Equal(t, unverifiedGroups[0], txnGroups[txgIdx]) } From 754f0fbf83f9658414293aa88fa1b83d479701cb Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 21:40:45 -0400 Subject: [PATCH 008/156] cache txn when only logicsig --- crypto/batchverifier.go | 3 +-- crypto/batchverifier_test.go | 2 +- crypto/multisig.go | 4 ++-- crypto/multisig_test.go | 12 ++++++------ data/transactions/verify/txn.go | 8 ++------ 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index cce8e06d7e..bb02075910 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,7 +53,6 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") - ErrZeroTransactionInBatch = errors.New("Could not validate empty signature set") ) //export ed25519_randombytes_unsafe @@ -113,7 +112,7 @@ func (b *BatchVerifier) GetNumberOfEnqueuedSignatures() int { // if the batch is zero an appropriate error is return. func (b *BatchVerifier) Verify() error { if b.GetNumberOfEnqueuedSignatures() == 0 { - return ErrZeroTransactionInBatch + return nil } var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 781a80e174..8e76580db9 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -122,5 +122,5 @@ func BenchmarkBatchVerifier(b *testing.B) { func TestEmpty(t *testing.T) { partitiontest.PartitionTest(t) bv := MakeBatchVerifier() - require.Error(t, bv.Verify()) + require.NoError(t, bv.Verify()) } diff --git a/crypto/multisig.go b/crypto/multisig.go index 53386ebc97..e5470f51cd 100644 --- a/crypto/multisig.go +++ b/crypto/multisig.go @@ -219,7 +219,7 @@ func MultisigAssemble(unisig []MultisigSig) (msig MultisigSig, err error) { func MultisigVerify(msg Hashable, addr Digest, sig MultisigSig) (verified bool, err error) { batchVerifier := MakeBatchVerifier() - if verified, err = MultisigBatchVerify(msg, addr, sig, batchVerifier); err != nil { + if verified, err = MultisigBatchVerifyPrep(msg, addr, sig, batchVerifier); err != nil { return } if !verified { @@ -236,7 +236,7 @@ func MultisigVerify(msg Hashable, addr Digest, sig MultisigSig) (verified bool, // MultisigBatchVerify verifies an assembled MultisigSig. // it is the caller responsibility to call batchVerifier.verify() -func MultisigBatchVerify(msg Hashable, addr Digest, sig MultisigSig, batchVerifier *BatchVerifier) (verified bool, err error) { +func MultisigBatchVerifyPrep(msg Hashable, addr Digest, sig MultisigSig, batchVerifier *BatchVerifier) (verified bool, err error) { verified = false // short circuit: if msig doesn't have subsigs or if Subsigs are empty // then terminate (the upper layer should now verify the unisig) diff --git a/crypto/multisig_test.go b/crypto/multisig_test.go index 28eec24598..0e070e550b 100644 --- a/crypto/multisig_test.go +++ b/crypto/multisig_test.go @@ -142,7 +142,7 @@ func TestMultisig(t *testing.T) { //test3: use the batch verification br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, msig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, msig, br) require.NoError(t, err, "Multisig: unexpected verification failure with err") require.True(t, verify, "Multisig: verification failed, verify flag was false") res := br.Verify() @@ -258,7 +258,7 @@ func TestEmptyMultisig(t *testing.T) { require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, emptyMutliSig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, emptyMutliSig, br) require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") } @@ -286,7 +286,7 @@ func TestIncorrectAddrresInMultisig(t *testing.T) { require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, MutliSig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, MutliSig, br) require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") @@ -325,7 +325,7 @@ func TestMoreThanMaxSigsInMultisig(t *testing.T) { require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, msig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, msig, br) require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") } @@ -364,7 +364,7 @@ func TestOneSignatureIsEmpty(t *testing.T) { require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, msig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, msig, br) require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") } @@ -405,7 +405,7 @@ func TestOneSignatureIsInvalid(t *testing.T) { require.False(t, verify, "Multisig: verification succeeded, it should failed") require.Error(t, err, "Multisig: did not return error as expected") br := MakeBatchVerifier() - verify, err = MultisigBatchVerify(txid, addr, msig, br) + verify, err = MultisigBatchVerifyPrep(txid, addr, msig, br) require.NoError(t, err, "Multisig: did not return error as expected") require.True(t, verify, "Multisig: verification succeeded, it should failed") res := br.Verify() diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index cd951346e8..a9417489b8 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -122,10 +122,6 @@ func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, return nil, err } - if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { - return groupCtx, nil - } - if err := batchVerifier.Verify(); err != nil { return nil, err } @@ -212,7 +208,7 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex return nil } if hasMsig { - if ok, _ := crypto.MultisigBatchVerify(s.Txn, + if ok, _ := crypto.MultisigBatchVerifyPrep(s.Txn, crypto.Digest(s.Authorizer()), s.Msig, batchVerifier); ok { @@ -308,7 +304,7 @@ func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex batchVerifier.EnqueueSignature(crypto.PublicKey(txn.Authorizer()), &program, lsig.Sig) } else { program := logic.Program(lsig.Logic) - if ok, _ := crypto.MultisigBatchVerify(&program, crypto.Digest(txn.Authorizer()), lsig.Msig, batchVerifier); !ok { + if ok, _ := crypto.MultisigBatchVerifyPrep(&program, crypto.Digest(txn.Authorizer()), lsig.Msig, batchVerifier); !ok { return errors.New("logic multisig validation failed") } } From d2f8e16484fb381f2ebb6bb1cf0c3d09ad42ecab Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 21:48:24 -0400 Subject: [PATCH 009/156] fix comment --- crypto/multisig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/multisig.go b/crypto/multisig.go index e5470f51cd..c1bb6b9abd 100644 --- a/crypto/multisig.go +++ b/crypto/multisig.go @@ -234,7 +234,7 @@ func MultisigVerify(msg Hashable, addr Digest, sig MultisigSig) (verified bool, return true, nil } -// MultisigBatchVerify verifies an assembled MultisigSig. +// MultisigBatchVerifyPrep verifies an assembled MultisigSig. // it is the caller responsibility to call batchVerifier.verify() func MultisigBatchVerifyPrep(msg Hashable, addr Digest, sig MultisigSig, batchVerifier *BatchVerifier) (verified bool, err error) { verified = false From bd3697a3481fec3e6e16abbd272ed37d34fb1353 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 26 Sep 2022 17:53:51 -0400 Subject: [PATCH 010/156] draft --- crypto/batchverifier.go | 35 ++++++++++++++++++++++--- crypto/onetimesig.go | 2 +- data/transactions/verify/txn.go | 45 +++++++++++++++++++++++++++++++++ ledger/internal/eval.go | 2 +- 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index bb02075910..499ef3c471 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,6 +53,7 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") + errInvalidResultSize = errors.New("results size is not equal to the number of enqueued signatures") ) //export ed25519_randombytes_unsafe @@ -119,7 +120,31 @@ func (b *BatchVerifier) Verify() error { for i, m := range b.messages { messages[i] = HashRep(m) } - if batchVerificationImpl(messages, b.publicKeys, b.signatures) { + if batchVerificationImpl(messages, b.publicKeys, b.signatures, nil) { + return nil + } + return ErrBatchVerificationFailed + +} + +// VerifyWithFeedback verifies that all the signatures are valid. +// results slice should have len equal to GetNumberOfEnqueuedSignatures +// if all sigs are valid, nil will be returned +// if some txns are invalid, false will be set at the appropriate index in results +func (b *BatchVerifier) VerifyWithFeedback(results []bool) error { + if b.GetNumberOfEnqueuedSignatures() == 0 { + return ErrZeroTransactionInBatch + } + + if len(results) != b.GetNumberOfEnqueuedSignatures() { + return errInvalidResultSize + } + + var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) + for i, m := range b.messages { + messages[i] = HashRep(m) + } + if batchVerificationImpl(messages, b.publicKeys, b.signatures, results) { return nil } return ErrBatchVerificationFailed @@ -128,7 +153,7 @@ func (b *BatchVerifier) Verify() error { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners -func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) bool { +func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, results []bool) bool { numberOfSignatures := len(messages) @@ -163,6 +188,10 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si (**C.uchar)(unsafe.Pointer(signaturesAllocation)), C.size_t(len(messages)), (*C.int)(unsafe.Pointer(valid))) - + if allValid != 0 && results != nil { + for i := 0; i < numberOfSignatures; i++ { + results[i] = valid[i] + } + } return allValid == 0 } diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index cf1c256429..c028a7c895 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -322,7 +322,7 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message return batchVerificationImpl( [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, - []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, + []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, nil, ) } diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a9417489b8..26953e616d 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -459,3 +459,48 @@ func (w *worksetBuilder) next() (txnGroups [][]transactions.SignedTxn) { func (w *worksetBuilder) completed() bool { return w.idx >= len(w.payset) } + +type VerificationElement struct { + txnGroup []transactions.SignedTxn + contextHdr bookkeeping.BlockHeader +} + +type VerificationResult struct { + txnGroup []transactions.SignedTxn + verified bool +} + +func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, + stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult) { + + // wait time for another txn should satisfy the following inequality: + // [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] + // since these are difficult to estimate, the simplified version could be to assume: + // [validation time added to the group by one more txn] = [validation time of a single txn] / 2 + // This gives us: + // [wait time] <= [validation time of a single txn] / 2 + singelTxnValidationTime := 100 * time.Millisecond + + batchVerifier := crypto.MakeBatchVerifier() + timer := time.NewTicker(singelTxnValidationTime) + + go func() { + select { + case stx := <-stxnChan: + groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(stx.txnGroup, stx.contextHdr, cache, + ledger, batchVerifier) + case <-timer.C: + numSigs := batchVerifier.GetNumberOfEnqueuedSignatures() + if numSigs != 0 { + results := make([]bool, numSigs, numSigs) + verifyError := batchVerifier.VerifyWithFeedback(results) + } + timer = time.NewTicker(singelTxnValidationTime) + batchVerifier = crypto.MakeBatchVerifier() + case <-ctx.Done(): + return ctx.Err() + + } + }() + +} diff --git a/ledger/internal/eval.go b/ledger/internal/eval.go index f2750d8a08..7ad4a54b0d 100644 --- a/ledger/internal/eval.go +++ b/ledger/internal/eval.go @@ -1490,7 +1490,7 @@ func (validator *evalTxValidator) run() { // Validate: Eval(ctx, l, blk, true, txcache, executionPool) // AddBlock: Eval(context.Background(), l, blk, false, txcache, nil) // tracker: Eval(context.Background(), l, blk, false, txcache, nil) -func Eval(ctx context.Context, l LedgerForEvaluator, blk bookkeeping.Block, validate bool, txcache verify.VerifiedTransactionCache, executionPool execpool.BacklogPool) (ledgercore.StateDelta, error) { +func XEval(ctx context.Context, l LedgerForEvaluator, blk bookkeeping.Block, validate bool, txcache verify.VerifiedTransactionCache, executionPool execpool.BacklogPool) (ledgercore.StateDelta, error) { eval, err := StartEvaluator(l, blk.BlockHeader, EvaluatorOptions{ PaysetHint: len(blk.Payset), From 179d064a35c9f982f0abae017ef32939ab0a7cf5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 10:09:40 -0400 Subject: [PATCH 011/156] fixes --- ledger/internal/eval.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/internal/eval.go b/ledger/internal/eval.go index 7ad4a54b0d..f2750d8a08 100644 --- a/ledger/internal/eval.go +++ b/ledger/internal/eval.go @@ -1490,7 +1490,7 @@ func (validator *evalTxValidator) run() { // Validate: Eval(ctx, l, blk, true, txcache, executionPool) // AddBlock: Eval(context.Background(), l, blk, false, txcache, nil) // tracker: Eval(context.Background(), l, blk, false, txcache, nil) -func XEval(ctx context.Context, l LedgerForEvaluator, blk bookkeeping.Block, validate bool, txcache verify.VerifiedTransactionCache, executionPool execpool.BacklogPool) (ledgercore.StateDelta, error) { +func Eval(ctx context.Context, l LedgerForEvaluator, blk bookkeeping.Block, validate bool, txcache verify.VerifiedTransactionCache, executionPool execpool.BacklogPool) (ledgercore.StateDelta, error) { eval, err := StartEvaluator(l, blk.BlockHeader, EvaluatorOptions{ PaysetHint: len(blk.Payset), From 363e7cdf40615abb29018b62b733b17e58e6ea35 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 23:17:38 -0400 Subject: [PATCH 012/156] get Verify failed sig indexes. Add tests --- crypto/batchverifier.go | 17 ++++---- crypto/batchverifier_test.go | 69 +++++++++++++++++++++++++++++++++ data/transactions/verify/txn.go | 3 +- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 499ef3c471..906eb5ac59 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,7 +53,7 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") - errInvalidResultSize = errors.New("results size is not equal to the number of enqueued signatures") + errInvalidResultSize = errors.New("results size is not equal to the number of enqueued signatures") ) //export ed25519_randombytes_unsafe @@ -131,20 +131,20 @@ func (b *BatchVerifier) Verify() error { // results slice should have len equal to GetNumberOfEnqueuedSignatures // if all sigs are valid, nil will be returned // if some txns are invalid, false will be set at the appropriate index in results -func (b *BatchVerifier) VerifyWithFeedback(results []bool) error { +func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { if b.GetNumberOfEnqueuedSignatures() == 0 { return ErrZeroTransactionInBatch } - if len(results) != b.GetNumberOfEnqueuedSignatures() { + if len(failed) != b.GetNumberOfEnqueuedSignatures() { return errInvalidResultSize } - + var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) for i, m := range b.messages { messages[i] = HashRep(m) } - if batchVerificationImpl(messages, b.publicKeys, b.signatures, results) { + if batchVerificationImpl(messages, b.publicKeys, b.signatures, failed) { return nil } return ErrBatchVerificationFailed @@ -153,7 +153,7 @@ func (b *BatchVerifier) VerifyWithFeedback(results []bool) error { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners -func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, results []bool) bool { +func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, failed []bool) bool { numberOfSignatures := len(messages) @@ -188,9 +188,10 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si (**C.uchar)(unsafe.Pointer(signaturesAllocation)), C.size_t(len(messages)), (*C.int)(unsafe.Pointer(valid))) - if allValid != 0 && results != nil { + if allValid != 0 && failed != nil { for i := 0; i < numberOfSignatures; i++ { - results[i] = valid[i] + cint := *(*C.int)(unsafe.Pointer(uintptr(valid) + uintptr(i*C.sizeof_int))) + failed[i] = (cint == 0) } } return allValid == 0 diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 8e76580db9..cfd8ff2bd6 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -17,6 +17,7 @@ package crypto import ( + "math/rand" "testing" "github.com/stretchr/testify/require" @@ -124,3 +125,71 @@ func TestEmpty(t *testing.T) { bv := MakeBatchVerifier() require.NoError(t, bv.Verify()) } + +// TestBatchVerifierBadFailedArray tests that VerifyWithFeedback returns +// the expected error if the failed array size is not correct +func TestBatchVerifierBadFailedArray(t *testing.T) { + partitiontest.PartitionTest(t) + + var s Seed + bv := MakeBatchVerifierWithHint(4) + for i := 0; i < 4; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, 4, bv.GetNumberOfEnqueuedSignatures()) + failed := make([]bool, 4, 6) + err := bv.VerifyWithFeedback(failed) + require.NoError(t, err) + + failed = make([]bool, 3, 4) + err = bv.VerifyWithFeedback(failed) + require.Error(t, err) + require.Equal(t, errInvalidResultSize, err) + + failed = make([]bool, 5, 5) + err = bv.VerifyWithFeedback(failed) + require.Error(t, err) + require.Equal(t, errInvalidResultSize, err) +} + +// TestBatchVerifierIndividualResults tests that VerifyWithFeedback +// returns the correct failed signature indexes +func TestBatchVerifierIndividualResults(t *testing.T) { + partitiontest.PartitionTest(t) + + for i := 1; i < 64*2+3; i++ { + n := i + bv := MakeBatchVerifierWithHint(n) + var s Seed + badSigs := make([]bool, n, n) + failed := make([]bool, n, n) + hasBadSig := false + for i := 0; i < n; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + if rand.Float32() > 0.5 { + // make a bad sig + sig[0] = sig[0] + 1 + badSigs[i] = true + hasBadSig = true + } + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) + err := bv.VerifyWithFeedback(failed) + if hasBadSig { + require.Error(t, err) + } else { + require.NoError(t, err) + } + for i := range badSigs { + require.Equal(t, badSigs[i], failed[i]) + } + } +} diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 26953e616d..e4e1bb17aa 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -469,7 +469,7 @@ type VerificationResult struct { txnGroup []transactions.SignedTxn verified bool } - +/* func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult) { @@ -504,3 +504,4 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le }() } +*/ From eaa8dc43c9aee159a9e1a1f46b87e674a7dbc678 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 23:38:13 -0400 Subject: [PATCH 013/156] rebase --- crypto/batchverifier.go | 9 +++++---- crypto/batchverifier_test.go | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 906eb5ac59..dbea7144a2 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,7 +53,7 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") - errInvalidResultSize = errors.New("results size is not equal to the number of enqueued signatures") + errInvalidFailedSlice = errors.New("failed slice size is not equal to the number of enqueued signatures") ) //export ed25519_randombytes_unsafe @@ -128,16 +128,16 @@ func (b *BatchVerifier) Verify() error { } // VerifyWithFeedback verifies that all the signatures are valid. -// results slice should have len equal to GetNumberOfEnqueuedSignatures +// failed slice should have len equal to GetNumberOfEnqueuedSignatures // if all sigs are valid, nil will be returned -// if some txns are invalid, false will be set at the appropriate index in results +// if some txns are invalid, true will be set at the appropriate index in failed func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { if b.GetNumberOfEnqueuedSignatures() == 0 { return ErrZeroTransactionInBatch } if len(failed) != b.GetNumberOfEnqueuedSignatures() { - return errInvalidResultSize + return errInvalidFailedSlice } var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) @@ -153,6 +153,7 @@ func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners +// otherwise, returns an error, and sets the indexes of the failed sigs in failed func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, failed []bool) bool { numberOfSignatures := len(messages) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index cfd8ff2bd6..ea5e88b4bd 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -148,12 +148,12 @@ func TestBatchVerifierBadFailedArray(t *testing.T) { failed = make([]bool, 3, 4) err = bv.VerifyWithFeedback(failed) require.Error(t, err) - require.Equal(t, errInvalidResultSize, err) + require.Equal(t, errInvalidFailedSlice, err) failed = make([]bool, 5, 5) err = bv.VerifyWithFeedback(failed) require.Error(t, err) - require.Equal(t, errInvalidResultSize, err) + require.Equal(t, errInvalidFailedSlice, err) } // TestBatchVerifierIndividualResults tests that VerifyWithFeedback From 204c710eb72a0edbf97eb759492fd8a5b8da7b06 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 30 Sep 2022 01:10:31 -0400 Subject: [PATCH 014/156] initial draft --- crypto/batchverifier.go | 2 +- data/transactions/verify/txn.go | 104 ++++++++++++++++++++++++++------ 2 files changed, 87 insertions(+), 19 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index dbea7144a2..3a5f25b5fa 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -133,7 +133,7 @@ func (b *BatchVerifier) Verify() error { // if some txns are invalid, true will be set at the appropriate index in failed func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { if b.GetNumberOfEnqueuedSignatures() == 0 { - return ErrZeroTransactionInBatch + return nil } if len(failed) != b.GetNumberOfEnqueuedSignatures() { diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index e4e1bb17aa..91c0bb4702 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" @@ -469,9 +470,17 @@ type VerificationResult struct { txnGroup []transactions.SignedTxn verified bool } -/* + +type streamManager struct { + returnChans []chan interface{} + poolSeats chan int + verificationPool execpool.BacklogPool + ctx context.Context + cache VerifiedTransactionCache +} + func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, - stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult) { + stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { // wait time for another txn should satisfy the following inequality: // [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] @@ -481,27 +490,86 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le // [wait time] <= [validation time of a single txn] / 2 singelTxnValidationTime := 100 * time.Millisecond + numberOfExecPoolSeats := 4 + + sm := streamManager{ + returnChans: make([]chan interface{}, 4, 4), + poolSeats: make(chan int, numberOfExecPoolSeats), + verificationPool: verificationPool, + ctx: ctx, + cache: cache, + } + batchVerifier := crypto.MakeBatchVerifier() - timer := time.NewTicker(singelTxnValidationTime) + var timer *time.Ticker go func() { - select { - case stx := <-stxnChan: - groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(stx.txnGroup, stx.contextHdr, cache, - ledger, batchVerifier) - case <-timer.C: - numSigs := batchVerifier.GetNumberOfEnqueuedSignatures() - if numSigs != 0 { - results := make([]bool, numSigs, numSigs) - verifyError := batchVerifier.VerifyWithFeedback(results) - } - timer = time.NewTicker(singelTxnValidationTime) - batchVerifier = crypto.MakeBatchVerifier() - case <-ctx.Done(): - return ctx.Err() + var groupCtxs []*GroupContext + var txnGroups [][]transactions.SignedTxn + var messagesForTxn []int + for { + select { + case stx := <-stxnChan: + if timer == nil { + timer = time.NewTicker(singelTxnValidationTime) + } + // TODO: separate operations here, and get the sig verification inside LogicSig outside + groupCtx, err := txnGroupBatchVerifyPrep(stx.txnGroup, stx.contextHdr, + ledger, batchVerifier) + //TODO: report the error ctx.Err() + fmt.Println(err) + if err != nil { + continue + } + groupCtxs = append(groupCtxs, groupCtx) + txnGroups = append(txnGroups, stx.txnGroup) + messagesForTxn = append(messagesForTxn, batchVerifier.GetNumberOfEnqueuedSignatures()) + case <-timer.C: + select { + // try to pick a seat in the pool + case seatID := <-sm.poolSeats: + sm.addVerificationTaskToThePool(batchVerifier, txnGroups, groupCtxs, messagesForTxn, seatID) + // TODO: queue to the pool. + // fmt.Println(err) + default: + // if no free seats, wait some more for more txns + timer = time.NewTicker(singelTxnValidationTime / 2) + continue + } + batchVerifier = crypto.MakeBatchVerifier() + groupCtxs = groupCtxs[:0] + txnGroups = txnGroups[:0] + messagesForTxn = messagesForTxn[:0] + timer = nil + case <-ctx.Done(): + return //TODO: report the error ctx.Err() + } } }() } -*/ + +func (sm *streamManager) addVerificationTaskToThePool( + batchVerifier *crypto.BatchVerifier, txnGroups [][]transactions.SignedTxn, + groupCtxs []*GroupContext, messagesForTxn []int, seatID int) error { + + function := func(arg interface{}) interface{} { + // var grpErr error + // check if we've canceled the request while this was in the queue. + if sm.ctx.Err() != nil { + return sm.ctx.Err() + } + numSigs := batchVerifier.GetNumberOfEnqueuedSignatures() + failed := make([]bool, numSigs, numSigs) + _ = batchVerifier.VerifyWithFeedback(failed) + + for txIdx := range txnGroups { + if + } + sm.cache.AddPayset(txnGroups, groupCtxs) + return nil + } + err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, nil, sm.returnChans[seatID]) + return err +} From 620c271194f80ac8576f21c7e341aaf1e0e07a3f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Sep 2022 23:45:17 -0400 Subject: [PATCH 015/156] Verify: chekcout failed sig indexes --- crypto/batchverifier.go | 37 +++++++++++++++++-- crypto/batchverifier_test.go | 69 ++++++++++++++++++++++++++++++++++++ crypto/onetimesig.go | 2 +- 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index db8ffe6426..adb16c554d 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,6 +53,7 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") + errInvalidFailedSlice = errors.New("failed slice size is not equal to the number of enqueued signatures") ) //export ed25519_randombytes_unsafe @@ -119,7 +120,31 @@ func (b *BatchVerifier) Verify() error { for i, m := range b.messages { messages[i] = HashRep(m) } - if batchVerificationImpl(messages, b.publicKeys, b.signatures) { + if batchVerificationImpl(messages, b.publicKeys, b.signatures, nil) { + return nil + } + return ErrBatchVerificationFailed + +} + +// VerifyWithFeedback verifies that all the signatures are valid. +// failed slice should have len equal to GetNumberOfEnqueuedSignatures +// if all sigs are valid, nil will be returned +// if some txns are invalid, true will be set at the appropriate index in failed +func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { + if b.GetNumberOfEnqueuedSignatures() == 0 { + return nil + } + + if len(failed) != b.GetNumberOfEnqueuedSignatures() { + return errInvalidFailedSlice + } + + var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) + for i, m := range b.messages { + messages[i] = HashRep(m) + } + if batchVerificationImpl(messages, b.publicKeys, b.signatures, failed) { return nil } return ErrBatchVerificationFailed @@ -128,7 +153,8 @@ func (b *BatchVerifier) Verify() error { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners -func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) bool { +// otherwise, returns an error, and sets the indexes of the failed sigs in failed +func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, failed []bool) bool { numberOfSignatures := len(messages) @@ -163,6 +189,11 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si (**C.uchar)(unsafe.Pointer(signaturesAllocation)), C.size_t(len(messages)), (*C.int)(unsafe.Pointer(valid))) - + if allValid != 0 && failed != nil { + for i := 0; i < numberOfSignatures; i++ { + cint := *(*C.int)(unsafe.Pointer(uintptr(valid) + uintptr(i*C.sizeof_int))) + failed[i] = (cint == 0) + } + } return allValid == 0 } diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 4469da400b..8cda805d89 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -17,6 +17,7 @@ package crypto import ( + "math/rand" "testing" "github.com/stretchr/testify/require" @@ -124,3 +125,71 @@ func TestEmpty(t *testing.T) { bv := MakeBatchVerifier() require.NoError(t, bv.Verify()) } + +// TestBatchVerifierBadFailedArray tests that VerifyWithFeedback returns +// the expected error if the failed array size is not correct +func TestBatchVerifierBadFailedArray(t *testing.T) { + partitiontest.PartitionTest(t) + + var s Seed + bv := MakeBatchVerifierWithHint(4) + for i := 0; i < 4; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, 4, bv.GetNumberOfEnqueuedSignatures()) + failed := make([]bool, 4, 6) + err := bv.VerifyWithFeedback(failed) + require.NoError(t, err) + + failed = make([]bool, 3, 4) + err = bv.VerifyWithFeedback(failed) + require.Error(t, err) + require.Equal(t, errInvalidFailedSlice, err) + + failed = make([]bool, 5, 5) + err = bv.VerifyWithFeedback(failed) + require.Error(t, err) + require.Equal(t, errInvalidFailedSlice, err) +} + +// TestBatchVerifierIndividualResults tests that VerifyWithFeedback +// returns the correct failed signature indexes +func TestBatchVerifierIndividualResults(t *testing.T) { + partitiontest.PartitionTest(t) + + for i := 1; i < 64*2+3; i++ { + n := i + bv := MakeBatchVerifierWithHint(n) + var s Seed + badSigs := make([]bool, n, n) + failed := make([]bool, n, n) + hasBadSig := false + for i := 0; i < n; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + if rand.Float32() > 0.5 { + // make a bad sig + sig[0] = sig[0] + 1 + badSigs[i] = true + hasBadSig = true + } + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) + err := bv.VerifyWithFeedback(failed) + if hasBadSig { + require.Error(t, err) + } else { + require.NoError(t, err) + } + for i := range badSigs { + require.Equal(t, badSigs[i], failed[i]) + } + } +} diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index cf1c256429..c028a7c895 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -322,7 +322,7 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message return batchVerificationImpl( [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, - []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, + []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, nil, ) } From 0b61671aad174fd32d7d85b45a891f084547f73b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 30 Sep 2022 12:37:10 -0400 Subject: [PATCH 016/156] reuse the func --- crypto/batchverifier.go | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index adb16c554d..f3e8d7ebec 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -112,35 +112,23 @@ func (b *BatchVerifier) getNumberOfEnqueuedSignatures() int { // Verify verifies that all the signatures are valid. in that case nil is returned // if the batch is zero an appropriate error is return. func (b *BatchVerifier) Verify() error { - if b.getNumberOfEnqueuedSignatures() == 0 { - return nil - } - - var messages = make([][]byte, b.getNumberOfEnqueuedSignatures()) - for i, m := range b.messages { - messages[i] = HashRep(m) - } - if batchVerificationImpl(messages, b.publicKeys, b.signatures, nil) { - return nil - } - return ErrBatchVerificationFailed - + return b.VerifyWithFeedback(nil) } // VerifyWithFeedback verifies that all the signatures are valid. -// failed slice should have len equal to GetNumberOfEnqueuedSignatures +// failed slice should have len equal to getNumberOfEnqueuedSignatures // if all sigs are valid, nil will be returned // if some txns are invalid, true will be set at the appropriate index in failed func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { - if b.GetNumberOfEnqueuedSignatures() == 0 { - return nil + if failed != nil && len(failed) != b.getNumberOfEnqueuedSignatures() { + return errInvalidFailedSlice } - if len(failed) != b.GetNumberOfEnqueuedSignatures() { - return errInvalidFailedSlice + if b.getNumberOfEnqueuedSignatures() == 0 { + return nil } - var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) + var messages = make([][]byte, b.getNumberOfEnqueuedSignatures()) for i, m := range b.messages { messages[i] = HashRep(m) } @@ -148,7 +136,6 @@ func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { return nil } return ErrBatchVerificationFailed - } // batchVerificationImpl invokes the ed25519 batch verification algorithm. From bf128b1bca10437549dcc5c70d6056f98f940e6c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 30 Sep 2022 12:40:00 -0400 Subject: [PATCH 017/156] fix code merge issue --- crypto/batchverifier_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 8cda805d89..3ca36b1186 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -140,7 +140,7 @@ func TestBatchVerifierBadFailedArray(t *testing.T) { sig := sigSecrets.Sign(msg) bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) } - require.Equal(t, 4, bv.GetNumberOfEnqueuedSignatures()) + require.Equal(t, 4, bv.getNumberOfEnqueuedSignatures()) failed := make([]bool, 4, 6) err := bv.VerifyWithFeedback(failed) require.NoError(t, err) @@ -181,7 +181,7 @@ func TestBatchVerifierIndividualResults(t *testing.T) { } bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) } - require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) + require.Equal(t, n, bv.getNumberOfEnqueuedSignatures()) err := bv.VerifyWithFeedback(failed) if hasBadSig { require.Error(t, err) From 18a23975f60ec91ab89edea6642d6d8baf34a7a5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 30 Sep 2022 23:35:28 -0400 Subject: [PATCH 018/156] testing --- data/transactions/verify/txn.go | 137 ++++++++++++++------------- data/transactions/verify/txn_test.go | 66 +++++++++++++ 2 files changed, 138 insertions(+), 65 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 5a47895fb9..956dd499c3 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -100,10 +100,10 @@ func (g *GroupContext) Equal(other *GroupContext) bool { g.minAvmVersion == other.minAvmVersion } -// txnBatchVerifyPrep verifies a SignedTxn having no obviously inconsistent data. +// txnBatchPrep verifies a SignedTxn having no obviously inconsistent data. // Block-assembly time checks of LogicSig and accounting rules may still block the txn. // it is the caller responsibility to call batchVerifier.Verify() -func txnBatchVerifyPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, verifier *crypto.BatchVerifier) error { +func txnBatchPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, verifier *crypto.BatchVerifier) error { if !groupCtx.consensusParams.SupportRekeying && (s.AuthAddr != basics.Address{}) { return errors.New("nonempty AuthAddr but rekeying is not supported") } @@ -119,7 +119,7 @@ func txnBatchVerifyPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupCo func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature) (groupCtx *GroupContext, err error) { batchVerifier := crypto.MakeBatchVerifier() - if groupCtx, err = txnGroupBatchVerifyPrep(stxs, contextHdr, ledger, batchVerifier); err != nil { + if groupCtx, err = txnGroupBatchPrep(stxs, contextHdr, ledger, batchVerifier); err != nil { return nil, err } @@ -134,9 +134,9 @@ func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, return } -// txnGroupBatchVerifyPrep verifies a []SignedTxn having no obviously inconsistent data. +// txnGroupBatchPrep verifies a []SignedTxn having no obviously inconsistent data. // it is the caller responsibility to call batchVerifier.Verify() -func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { +func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { groupCtx, err = PrepareGroupContext(stxs, contextHdr, ledger) if err != nil { return nil, err @@ -145,7 +145,7 @@ func txnGroupBatchVerifyPrep(stxs []transactions.SignedTxn, contextHdr bookkeepi minFeeCount := uint64(0) feesPaid := uint64(0) for i, stxn := range stxs { - err = txnBatchVerifyPrep(&stxn, i, groupCtx, verifier) + err = txnBatchPrep(&stxn, i, groupCtx, verifier) if err != nil { err = fmt.Errorf("transaction %+v invalid : %w", stxn, err) return @@ -209,13 +209,10 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex return nil } if hasMsig { - if ok, _ := crypto.MultisigBatchVerifyPrep(s.Txn, - crypto.Digest(s.Authorizer()), - s.Msig, - batchVerifier); ok { - return nil + if err := crypto.MultisigBatchPrep(s.Txn, crypto.Digest(s.Authorizer()), s.Msig, batchVerifier); err != nil { + return fmt.Errorf("multisig validation failed: %w", err) } - return errors.New("multisig validation failed") + return nil } if hasLogicSig { return logicSigVerify(s, txnIdx, groupCtx) @@ -228,22 +225,16 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex func LogicSigSanityCheck(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext) error { batchVerifier := crypto.MakeBatchVerifier() - if err := logicSigSanityCheckBatchVerifyPrep(txn, groupIndex, groupCtx, batchVerifier); err != nil { + if err := logicSigSanityCheckBatchPrep(txn, groupIndex, groupCtx, batchVerifier); err != nil { return err } - - // in case of contract account the signature len might 0. that's ok - if batchVerifier.GetNumberOfEnqueuedSignatures() == 0 { - return nil - } - return batchVerifier.Verify() } -// logicSigSanityCheckBatchVerifyPrep checks that the signature is valid and that the program is basically well formed. +// logicSigSanityCheckBatchPrep checks that the signature is valid and that the program is basically well formed. // It does not evaluate the logic. // it is the caller responsibility to call batchVerifier.Verify() -func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { +func logicSigSanityCheckBatchPrep(txn *transactions.SignedTxn, groupIndex int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) error { lsig := txn.Lsig if groupCtx.consensusParams.LogicSigVersion == 0 { @@ -305,8 +296,8 @@ func logicSigSanityCheckBatchVerifyPrep(txn *transactions.SignedTxn, groupIndex batchVerifier.EnqueueSignature(crypto.PublicKey(txn.Authorizer()), &program, lsig.Sig) } else { program := logic.Program(lsig.Logic) - if ok, _ := crypto.MultisigBatchVerifyPrep(&program, crypto.Digest(txn.Authorizer()), lsig.Msig, batchVerifier); !ok { - return errors.New("logic multisig validation failed") + if err := crypto.MultisigBatchPrep(&program, crypto.Digest(txn.Authorizer()), lsig.Msig, batchVerifier); err != nil { + return fmt.Errorf("logic multisig validation failed: %w", err) } } return nil @@ -385,17 +376,15 @@ func PaysetGroups(ctx context.Context, payset [][]transactions.SignedTxn, blkHea batchVerifier := crypto.MakeBatchVerifierWithHint(len(payset)) for i, signTxnsGrp := range txnGroups { - groupCtxs[i], grpErr = txnGroupBatchVerifyPrep(signTxnsGrp, blkHeader, ledger, batchVerifier) + groupCtxs[i], grpErr = txnGroupBatchPrep(signTxnsGrp, blkHeader, ledger, batchVerifier) // abort only if it's a non-cache error. if grpErr != nil { return grpErr } } - if batchVerifier.GetNumberOfEnqueuedSignatures() != 0 { - verifyErr := batchVerifier.Verify() - if verifyErr != nil { - return verifyErr - } + verifyErr := batchVerifier.Verify() + if verifyErr != nil { + return verifyErr } cache.AddPayset(txnGroups, groupCtxs) return nil @@ -480,29 +469,34 @@ type streamManager struct { cache VerifiedTransactionCache } +// wait time for another txn should satisfy the following inequality: +// [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] +// since these are difficult to estimate, the simplified version could be to assume: +// [validation time added to the group by one more txn] = [validation time of a single txn] / 2 +// This gives us: +// [wait time] <= [validation time of a single txn] / 2 +const singelTxnValidationTime = 100 * time.Millisecond + func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { - // wait time for another txn should satisfy the following inequality: - // [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] - // since these are difficult to estimate, the simplified version could be to assume: - // [validation time added to the group by one more txn] = [validation time of a single txn] / 2 - // This gives us: - // [wait time] <= [validation time of a single txn] / 2 - singelTxnValidationTime := 100 * time.Millisecond - numberOfExecPoolSeats := 4 sm := streamManager{ returnChans: make([]chan interface{}, 4, 4), poolSeats: make(chan int, numberOfExecPoolSeats), + resultChan: resultChan, verificationPool: verificationPool, ctx: ctx, cache: cache, } + for x := 0; x < numberOfExecPoolSeats; x++ { + sm.poolSeats <- x + } + batchVerifier := crypto.MakeBatchVerifier() - var timer *time.Ticker + timer := time.NewTicker(singelTxnValidationTime / 2) go func() { var groupCtxs []*GroupContext @@ -511,46 +505,57 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le for { select { case stx := <-stxnChan: - if timer == nil { - timer = time.NewTicker(singelTxnValidationTime) - } + timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchVerifyPrep(stx.txnGroup, stx.contextHdr, + groupCtx, err := txnGroupBatchPrep(stx.txnGroup, stx.contextHdr, ledger, batchVerifier) //TODO: report the error ctx.Err() - fmt.Println(err) + if err != nil { continue } groupCtxs = append(groupCtxs, groupCtx) txnGroups = append(txnGroups, stx.txnGroup) messagesForTxn = append(messagesForTxn, batchVerifier.GetNumberOfEnqueuedSignatures()) + if len(groupCtxs) == txnPerWorksetThreshold { + timer = sm.processBatch(batchVerifier, txnGroups, groupCtxs, messagesForTxn) + } case <-timer.C: - select { - // try to pick a seat in the pool - case seatID := <-sm.poolSeats: - sm.addVerificationTaskToThePool(batchVerifier, txnGroups, groupCtxs, messagesForTxn, seatID) - // TODO: queue to the pool. - // fmt.Println(err) - default: - // if no free seats, wait some more for more txns + if len(groupCtxs) == 0 { + // nothing yet... wait some more timer = time.NewTicker(singelTxnValidationTime / 2) continue } - batchVerifier = crypto.MakeBatchVerifier() - groupCtxs = groupCtxs[:0] - txnGroups = txnGroups[:0] - messagesForTxn = messagesForTxn[:0] - timer = nil + timer = sm.processBatch(batchVerifier, txnGroups, groupCtxs, messagesForTxn) case <-ctx.Done(): return //TODO: report the error ctx.Err() - } } }() } +func (sm *streamManager) processBatch(batchVerifier *crypto.BatchVerifier, + txnGroups [][]transactions.SignedTxn, groupCtxs []*GroupContext, + messagesForTxn []int) (timer *time.Ticker) { + select { + // try to pick a seat in the pool + case seatID := <-sm.poolSeats: + sm.addVerificationTaskToThePool(batchVerifier, txnGroups, groupCtxs, messagesForTxn, seatID) + timer = time.NewTicker(singelTxnValidationTime / 2) + // TODO: queue to the pool. + // fmt.Println(err) + default: + // if no free seats, wait some more for more txns + timer = time.NewTicker(singelTxnValidationTime / 2) + } + batchVerifier = crypto.MakeBatchVerifier() + groupCtxs = groupCtxs[:0] + txnGroups = txnGroups[:0] + messagesForTxn = messagesForTxn[:0] + return +} + func (sm *streamManager) addVerificationTaskToThePool( batchVerifier *crypto.BatchVerifier, txnGroups [][]transactions.SignedTxn, groupCtxs []*GroupContext, messagesForTxn []int, seatID int) error { @@ -564,7 +569,7 @@ func (sm *streamManager) addVerificationTaskToThePool( numSigs := batchVerifier.GetNumberOfEnqueuedSignatures() failed := make([]bool, numSigs, numSigs) err := batchVerifier.VerifyWithFeedback(failed) - if err != nil && err != ErrBatchHasFailedSigs { + if err != nil && err != crypto.ErrBatchHasFailedSigs { // something bad happened // TODO: report error and discard the batch } @@ -573,7 +578,7 @@ func (sm *streamManager) addVerificationTaskToThePool( verifiedGroupCtxs := make([]*GroupContext, len(groupCtxs)) failedSigIdx := 0 for txgIdx := range txnGroups { - txGroupSigFailed = false + txGroupSigFailed := false // if err == nil, means all sigs are verified, no need to check for the failed for err != nil && failedSigIdx < messagesForTxn[txgIdx] { if failed[failedSigIdx] { @@ -587,7 +592,7 @@ func (sm *streamManager) addVerificationTaskToThePool( verifiedGroupCtxs = append(verifiedGroupCtxs, groupCtxs[txgIdx]) vr = VerificationResult{ txnGroup: txnGroups[txgIdx], - verified: false, + verified: true, } } else { vr = VerificationResult{ @@ -596,11 +601,13 @@ func (sm *streamManager) addVerificationTaskToThePool( } } // send the txn result out the pipe - select{ - case sm.resultChan<- vr: - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: + select { + case sm.resultChan <- vr: + fmt.Println("sent!") + // if the channel is not accepting, should not block here + // report dropped txn. caching is fine, if it comes back in the block + default: + fmt.Println("skipped!!") //TODO: report this } } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index ea3f05b87e..0829a04b1a 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -18,6 +18,7 @@ package verify import ( "context" + "fmt" "math/rand" "testing" "time" @@ -515,3 +516,68 @@ func TestTxnGroupCacheUpdate(t *testing.T) { require.Equal(t, unverifiedGroups[0], txnGroups[txgIdx]) } + +func TestStreamVerifier(t *testing.T) { + partitiontest.PartitionTest(t) + + _, signedTxn, secrets, addrs := generateTestObjects(10000, 20, 50) + blkHdr := bookkeeping.BlockHeader{ + Round: 50, + GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + RewardsState: bookkeeping.RewardsState{ + FeeSink: feeSink, + RewardsPool: poolAddr, + }, + } + + execPool := execpool.MakePool(t) + verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + defer verificationPool.Shutdown() + + txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + + ctx := context.Background() + cache := MakeVerifiedTransactionCache(50000) + stxnChan := make(chan VerificationElement, 3) + resultChan := make(chan VerificationResult, 3) + + defer ctx.Done() + Stream(ctx, cache, nil, stxnChan, resultChan, verificationPool) + + badTxnGroups := make(map[crypto.Signature]struct{}) + + for tgi := range txnGroups { + if rand.Float32() > 0.7 { + // make a bad sig + t := rand.Intn(len(txnGroups[tgi])) + txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 + badTxnGroups[txnGroups[tgi][0].Sig] = struct{}{} + } + } + + lenTGs := len(txnGroups) + tgi := 0 + for x := 0; x < lenTGs; { + select { + case result := <-resultChan: + x++ + if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { + fmt.Printf("%d ", len(badTxnGroups)) + delete(badTxnGroups, result.txnGroup[0].Sig) + fmt.Printf("%dn ", len(badTxnGroups)) + require.False(t, result.verified) + } else { + require.True(t, result.verified) + } + default: + if tgi < lenTGs { + stxnChan <- VerificationElement{txnGroup: txnGroups[tgi], contextHdr: blkHdr} + tgi++ + } + } + } + require.Equal(t, 0, len(badTxnGroups)) +} From f7fc5f4a3418e07b7e05d38bcc1726b7d4826218 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 1 Oct 2022 11:39:47 -0400 Subject: [PATCH 019/156] more progress --- data/transactions/verify/txn.go | 49 ++++++++++++++++------- data/transactions/verify/txn_test.go | 60 ++++++++++++++++++---------- 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 956dd499c3..99e56502f5 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -469,6 +469,28 @@ type streamManager struct { cache VerifiedTransactionCache } +type batchLoad struct { + batchVerifier *crypto.BatchVerifier + groupCtxs []*GroupContext + txnGroups [][]transactions.SignedTxn + messagesForTxn []int +} + +func makeBatchLoad() { + bl := batchLoad{} + bl.batchVerifier = crypto.MakeBatchVerifier() + bl.groupCtxs = make([]*GroupContext, 0) + bl.txnGroups = make([][]transactions.SignedTxn, 0) + bl.messagesForTxn = make([]int, 0) +} + +func (bl batchLoad) reset() { + bl.batchVerifier = crypto.MakeBatchVerifier() + bl.groupCtxs = groupCtxs[:0] + bl.txnGroups = txnGroups[:0] + bl.messagesForTxn = messagesForTxn[:0] +} + // wait time for another txn should satisfy the following inequality: // [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] // since these are difficult to estimate, the simplified version could be to assume: @@ -494,14 +516,9 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le for x := 0; x < numberOfExecPoolSeats; x++ { sm.poolSeats <- x } - - batchVerifier := crypto.MakeBatchVerifier() timer := time.NewTicker(singelTxnValidationTime / 2) go func() { - var groupCtxs []*GroupContext - var txnGroups [][]transactions.SignedTxn - var messagesForTxn []int for { select { case stx := <-stxnChan: @@ -549,10 +566,6 @@ func (sm *streamManager) processBatch(batchVerifier *crypto.BatchVerifier, // if no free seats, wait some more for more txns timer = time.NewTicker(singelTxnValidationTime / 2) } - batchVerifier = crypto.MakeBatchVerifier() - groupCtxs = groupCtxs[:0] - txnGroups = txnGroups[:0] - messagesForTxn = messagesForTxn[:0] return } @@ -582,8 +595,13 @@ func (sm *streamManager) addVerificationTaskToThePool( // if err == nil, means all sigs are verified, no need to check for the failed for err != nil && failedSigIdx < messagesForTxn[txgIdx] { if failed[failedSigIdx] { + // if there is a failed sig check, then no need to check the rest of the + // sigs for this txnGroup failedSigIdx = messagesForTxn[txgIdx] txGroupSigFailed = true + } else { + // proceed to check the next sig belonging to this txnGroup + failedSigIdx++ } } var vr VerificationResult @@ -604,12 +622,15 @@ func (sm *streamManager) addVerificationTaskToThePool( select { case sm.resultChan <- vr: fmt.Println("sent!") - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: - fmt.Println("skipped!!") - //TODO: report this + /* + // if the channel is not accepting, should not block here + // report dropped txn. caching is fine, if it comes back in the block + default: + fmt.Println("skipped!!") + //TODO: report this + */ } + } // loading them all at once to lock the cache once sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 0829a04b1a..5b9e15feed 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -520,7 +520,8 @@ func TestTxnGroupCacheUpdate(t *testing.T) { func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) - _, signedTxn, secrets, addrs := generateTestObjects(10000, 20, 50) + numOfTxnGroups := 10000 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxnGroups, 20, 50) blkHdr := bookkeeping.BlockHeader{ Round: 50, GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), @@ -539,12 +540,12 @@ func TestStreamVerifier(t *testing.T) { txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) stxnChan := make(chan VerificationElement, 3) resultChan := make(chan VerificationResult, 3) - defer ctx.Done() + defer cancel() Stream(ctx, cache, nil, stxnChan, resultChan, verificationPool) badTxnGroups := make(map[crypto.Signature]struct{}) @@ -558,26 +559,45 @@ func TestStreamVerifier(t *testing.T) { } } - lenTGs := len(txnGroups) - tgi := 0 - for x := 0; x < lenTGs; { - select { - case result := <-resultChan: - x++ - if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { - fmt.Printf("%d ", len(badTxnGroups)) - delete(badTxnGroups, result.txnGroup[0].Sig) - fmt.Printf("%dn ", len(badTxnGroups)) - require.False(t, result.verified) - } else { - require.True(t, result.verified) + errChan := make(chan error) + go func() { + defer close(errChan) + // process the thecked signatures + for x := 0; x < numOfTxnGroups; x++ { + fmt.Printf("receiving x=%d\n", x) + select { + case result := <-resultChan: + if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { + fmt.Printf("%d ", len(badTxnGroups)) + delete(badTxnGroups, result.txnGroup[0].Sig) + fmt.Printf("%dn ", len(badTxnGroups)) + err := fmt.Errorf("%dth transaction varified with a bad sig", x) + errChan <- err + } else { + err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) + errChan <- err + } + case <-ctx.Done(): + break } - default: - if tgi < lenTGs { - stxnChan <- VerificationElement{txnGroup: txnGroups[tgi], contextHdr: blkHdr} - tgi++ + } + }() + + // send txn groups to be verified + go func() { + for _, tg := range txnGroups { + select { + case <-ctx.Done(): + break + default: + stxnChan <- VerificationElement{txnGroup: tg, contextHdr: blkHdr} } } + }() + + for err := range errChan { + require.NoError(t, err) } + require.Equal(t, 0, len(badTxnGroups)) } From b89e82f1884b31dcb338767a10941d43cd13a331 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 1 Oct 2022 20:35:07 -0400 Subject: [PATCH 020/156] fixes --- data/transactions/verify/txn.go | 75 ++++++++++++---------------- data/transactions/verify/txn_test.go | 21 ++++---- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 99e56502f5..8cffc46238 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -461,7 +461,7 @@ type VerificationResult struct { } type streamManager struct { - returnChans []chan interface{} + seatReturnChan chan interface{} poolSeats chan int resultChan chan<- VerificationResult verificationPool execpool.BacklogPool @@ -471,24 +471,18 @@ type streamManager struct { type batchLoad struct { batchVerifier *crypto.BatchVerifier - groupCtxs []*GroupContext txnGroups [][]transactions.SignedTxn + groupCtxs []*GroupContext messagesForTxn []int } -func makeBatchLoad() { +func makeBatchLoad() batchLoad { bl := batchLoad{} bl.batchVerifier = crypto.MakeBatchVerifier() bl.groupCtxs = make([]*GroupContext, 0) bl.txnGroups = make([][]transactions.SignedTxn, 0) bl.messagesForTxn = make([]int, 0) -} - -func (bl batchLoad) reset() { - bl.batchVerifier = crypto.MakeBatchVerifier() - bl.groupCtxs = groupCtxs[:0] - bl.txnGroups = txnGroups[:0] - bl.messagesForTxn = messagesForTxn[:0] + return bl } // wait time for another txn should satisfy the following inequality: @@ -503,9 +497,8 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { numberOfExecPoolSeats := 4 - sm := streamManager{ - returnChans: make([]chan interface{}, 4, 4), + seatReturnChan: make(chan interface{}), poolSeats: make(chan int, numberOfExecPoolSeats), resultChan: resultChan, verificationPool: verificationPool, @@ -517,33 +510,34 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le sm.poolSeats <- x } timer := time.NewTicker(singelTxnValidationTime / 2) - go func() { + bl := makeBatchLoad() for { select { case stx := <-stxnChan: timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchPrep(stx.txnGroup, stx.contextHdr, - ledger, batchVerifier) + groupCtx, err := txnGroupBatchPrep(stx.txnGroup, stx.contextHdr, ledger, bl.batchVerifier) //TODO: report the error ctx.Err() if err != nil { continue } - groupCtxs = append(groupCtxs, groupCtx) - txnGroups = append(txnGroups, stx.txnGroup) - messagesForTxn = append(messagesForTxn, batchVerifier.GetNumberOfEnqueuedSignatures()) - if len(groupCtxs) == txnPerWorksetThreshold { - timer = sm.processBatch(batchVerifier, txnGroups, groupCtxs, messagesForTxn) + bl.groupCtxs = append(bl.groupCtxs, groupCtx) + bl.txnGroups = append(bl.txnGroups, stx.txnGroup) + bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) + if len(bl.groupCtxs) == txnPerWorksetThreshold { + timer = sm.processBatch(bl) + bl = makeBatchLoad() } case <-timer.C: - if len(groupCtxs) == 0 { + if len(bl.groupCtxs) == 0 { // nothing yet... wait some more timer = time.NewTicker(singelTxnValidationTime / 2) continue } - timer = sm.processBatch(batchVerifier, txnGroups, groupCtxs, messagesForTxn) + timer = sm.processBatch(bl) + bl = makeBatchLoad() case <-ctx.Done(): return //TODO: report the error ctx.Err() } @@ -552,13 +546,11 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } -func (sm *streamManager) processBatch(batchVerifier *crypto.BatchVerifier, - txnGroups [][]transactions.SignedTxn, groupCtxs []*GroupContext, - messagesForTxn []int) (timer *time.Ticker) { +func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker) { select { // try to pick a seat in the pool case seatID := <-sm.poolSeats: - sm.addVerificationTaskToThePool(batchVerifier, txnGroups, groupCtxs, messagesForTxn, seatID) + sm.addVerificationTaskToThePool(bl, seatID) timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: queue to the pool. // fmt.Println(err) @@ -569,35 +561,34 @@ func (sm *streamManager) processBatch(batchVerifier *crypto.BatchVerifier, return } -func (sm *streamManager) addVerificationTaskToThePool( - batchVerifier *crypto.BatchVerifier, txnGroups [][]transactions.SignedTxn, - groupCtxs []*GroupContext, messagesForTxn []int, seatID int) error { +func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad, seatID int) error { function := func(arg interface{}) interface{} { + bl = arg.(batchLoad) // var grpErr error // check if we've canceled the request while this was in the queue. if sm.ctx.Err() != nil { return sm.ctx.Err() } - numSigs := batchVerifier.GetNumberOfEnqueuedSignatures() + numSigs := bl.batchVerifier.GetNumberOfEnqueuedSignatures() failed := make([]bool, numSigs, numSigs) - err := batchVerifier.VerifyWithFeedback(failed) + err := bl.batchVerifier.VerifyWithFeedback(failed) if err != nil && err != crypto.ErrBatchHasFailedSigs { // something bad happened // TODO: report error and discard the batch } - verifiedTxnGroups := make([][]transactions.SignedTxn, len(txnGroups)) - verifiedGroupCtxs := make([]*GroupContext, len(groupCtxs)) + verifiedTxnGroups := make([][]transactions.SignedTxn, len(bl.txnGroups)) + verifiedGroupCtxs := make([]*GroupContext, len(bl.groupCtxs)) failedSigIdx := 0 - for txgIdx := range txnGroups { + for txgIdx := range bl.txnGroups { txGroupSigFailed := false // if err == nil, means all sigs are verified, no need to check for the failed - for err != nil && failedSigIdx < messagesForTxn[txgIdx] { + for err != nil && failedSigIdx < bl.messagesForTxn[txgIdx] { if failed[failedSigIdx] { // if there is a failed sig check, then no need to check the rest of the // sigs for this txnGroup - failedSigIdx = messagesForTxn[txgIdx] + failedSigIdx = bl.messagesForTxn[txgIdx] txGroupSigFailed = true } else { // proceed to check the next sig belonging to this txnGroup @@ -606,15 +597,15 @@ func (sm *streamManager) addVerificationTaskToThePool( } var vr VerificationResult if !txGroupSigFailed { - verifiedTxnGroups = append(verifiedTxnGroups, txnGroups[txgIdx]) - verifiedGroupCtxs = append(verifiedGroupCtxs, groupCtxs[txgIdx]) + verifiedTxnGroups = append(verifiedTxnGroups, bl.txnGroups[txgIdx]) + verifiedGroupCtxs = append(verifiedGroupCtxs, bl.groupCtxs[txgIdx]) vr = VerificationResult{ - txnGroup: txnGroups[txgIdx], + txnGroup: bl.txnGroups[txgIdx], verified: true, } } else { vr = VerificationResult{ - txnGroup: txnGroups[txgIdx], + txnGroup: bl.txnGroups[txgIdx], verified: false, } } @@ -634,9 +625,9 @@ func (sm *streamManager) addVerificationTaskToThePool( } // loading them all at once to lock the cache once sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) - + // return the seat return nil } - err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, nil, sm.returnChans[seatID]) + err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, bl, sm.seatReturnChan) return err } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 5b9e15feed..a1d8896273 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -549,7 +549,7 @@ func TestStreamVerifier(t *testing.T) { Stream(ctx, cache, nil, stxnChan, resultChan, verificationPool) badTxnGroups := make(map[crypto.Signature]struct{}) - + /* for tgi := range txnGroups { if rand.Float32() > 0.7 { // make a bad sig @@ -558,24 +558,27 @@ func TestStreamVerifier(t *testing.T) { badTxnGroups[txnGroups[tgi][0].Sig] = struct{}{} } } - +*/ errChan := make(chan error) go func() { defer close(errChan) // process the thecked signatures for x := 0; x < numOfTxnGroups; x++ { - fmt.Printf("receiving x=%d\n", x) + fmt.Printf("receiving x=%d/%d\n", x, numOfTxnGroups) select { case result := <-resultChan: + if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { - fmt.Printf("%d ", len(badTxnGroups)) delete(badTxnGroups, result.txnGroup[0].Sig) - fmt.Printf("%dn ", len(badTxnGroups)) - err := fmt.Errorf("%dth transaction varified with a bad sig", x) - errChan <- err + if result.verified { + err := fmt.Errorf("%dth transaction varified with a bad sig", x) + errChan <- err + } } else { - err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) - errChan <- err + if !result.verified { + err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) + errChan <- err + } } case <-ctx.Done(): break From 15a8898c570166b5a79744e588e8e6947a6ec88c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 1 Oct 2022 21:56:30 -0400 Subject: [PATCH 021/156] working draft --- data/transactions/verify/txn.go | 38 +++++++++++++++------------- data/transactions/verify/txn_test.go | 11 ++++---- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 8cffc46238..b335008fcc 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -462,7 +462,6 @@ type VerificationResult struct { type streamManager struct { seatReturnChan chan interface{} - poolSeats chan int resultChan chan<- VerificationResult verificationPool execpool.BacklogPool ctx context.Context @@ -496,10 +495,9 @@ const singelTxnValidationTime = 100 * time.Millisecond func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { - numberOfExecPoolSeats := 4 + numberOfExecPoolSeats := 128 sm := streamManager{ - seatReturnChan: make(chan interface{}), - poolSeats: make(chan int, numberOfExecPoolSeats), + seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), resultChan: resultChan, verificationPool: verificationPool, ctx: ctx, @@ -507,11 +505,12 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } for x := 0; x < numberOfExecPoolSeats; x++ { - sm.poolSeats <- x + sm.seatReturnChan <- struct{}{} } - timer := time.NewTicker(singelTxnValidationTime / 2) go func() { bl := makeBatchLoad() + timer := time.NewTicker(singelTxnValidationTime / 2) + var added bool for { select { case stx := <-stxnChan: @@ -526,9 +525,11 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le bl.groupCtxs = append(bl.groupCtxs, groupCtx) bl.txnGroups = append(bl.txnGroups, stx.txnGroup) bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) - if len(bl.groupCtxs) == txnPerWorksetThreshold { - timer = sm.processBatch(bl) - bl = makeBatchLoad() + if len(bl.groupCtxs) >= txnPerWorksetThreshold { + timer, added = sm.processBatch(bl) + if added { + bl = makeBatchLoad() + } } case <-timer.C: if len(bl.groupCtxs) == 0 { @@ -536,8 +537,10 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le timer = time.NewTicker(singelTxnValidationTime / 2) continue } - timer = sm.processBatch(bl) - bl = makeBatchLoad() + timer, added = sm.processBatch(bl) + if added { + bl = makeBatchLoad() + } case <-ctx.Done(): return //TODO: report the error ctx.Err() } @@ -546,12 +549,13 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } -func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker) { +func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added bool) { select { // try to pick a seat in the pool - case seatID := <-sm.poolSeats: - sm.addVerificationTaskToThePool(bl, seatID) + case <-sm.seatReturnChan: + sm.addVerificationTaskToThePool(bl) timer = time.NewTicker(singelTxnValidationTime / 2) + added = true // TODO: queue to the pool. // fmt.Println(err) default: @@ -561,7 +565,7 @@ func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker) { return } -func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad, seatID int) error { +func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { function := func(arg interface{}) interface{} { bl = arg.(batchLoad) @@ -612,7 +616,6 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad, seatID int) // send the txn result out the pipe select { case sm.resultChan <- vr: - fmt.Println("sent!") /* // if the channel is not accepting, should not block here // report dropped txn. caching is fine, if it comes back in the block @@ -625,8 +628,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad, seatID int) } // loading them all at once to lock the cache once sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) - // return the seat - return nil + return struct{}{} } err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, bl, sm.seatReturnChan) return err diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index a1d8896273..b9f67bc023 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -520,8 +520,8 @@ func TestTxnGroupCacheUpdate(t *testing.T) { func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxnGroups := 10000 - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxnGroups, 20, 50) + numOfTxns := 10000 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) blkHdr := bookkeeping.BlockHeader{ Round: 50, GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), @@ -539,7 +539,7 @@ func TestStreamVerifier(t *testing.T) { defer verificationPool.Shutdown() txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) - + numOfTxnGroups := len(txnGroups) ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) stxnChan := make(chan VerificationElement, 3) @@ -549,7 +549,7 @@ func TestStreamVerifier(t *testing.T) { Stream(ctx, cache, nil, stxnChan, resultChan, verificationPool) badTxnGroups := make(map[crypto.Signature]struct{}) - /* + for tgi := range txnGroups { if rand.Float32() > 0.7 { // make a bad sig @@ -558,13 +558,12 @@ func TestStreamVerifier(t *testing.T) { badTxnGroups[txnGroups[tgi][0].Sig] = struct{}{} } } -*/ + errChan := make(chan error) go func() { defer close(errChan) // process the thecked signatures for x := 0; x < numOfTxnGroups; x++ { - fmt.Printf("receiving x=%d/%d\n", x, numOfTxnGroups) select { case result := <-resultChan: From 8c392f0eb7956114daa72e732d4d003017794232 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sun, 2 Oct 2022 12:05:04 -0400 Subject: [PATCH 022/156] block wait when no seats and batch is full --- data/transactions/verify/txn.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index b335008fcc..4b58ce4680 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -491,11 +491,11 @@ func makeBatchLoad() batchLoad { // This gives us: // [wait time] <= [validation time of a single txn] / 2 const singelTxnValidationTime = 100 * time.Millisecond - +const numberOfExecPoolSeats = 8 +//const txnPerWorksetThreshold = 32 func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { - numberOfExecPoolSeats := 128 sm := streamManager{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), resultChan: resultChan, @@ -526,6 +526,7 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le bl.txnGroups = append(bl.txnGroups, stx.txnGroup) bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) if len(bl.groupCtxs) >= txnPerWorksetThreshold { + // TODO: the limit of 32 should not pass timer, added = sm.processBatch(bl) if added { bl = makeBatchLoad() @@ -550,8 +551,19 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added bool) { + if bl.batchVerifier.GetNumberOfEnqueuedSignatures() >= txnPerWorksetThreshold { + // Should not allow addition of more txns to the batch + // the varifier might be saturated. + // block and wait for a free seat + <-sm.seatReturnChan + sm.addVerificationTaskToThePool(bl) + timer = time.NewTicker(singelTxnValidationTime / 2) + added = true + return + } + // Otherwise, if cannot find a seat, can go back and collect + // more signatures instead of waiting here select { - // try to pick a seat in the pool case <-sm.seatReturnChan: sm.addVerificationTaskToThePool(bl) timer = time.NewTicker(singelTxnValidationTime / 2) From ae18181c2f9470a76924fb5d07d459a2d6e0e8f6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sun, 2 Oct 2022 12:09:10 -0400 Subject: [PATCH 023/156] tiny simplification --- data/transactions/verify/txn.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4b58ce4680..73eb661358 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -611,19 +611,15 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { failedSigIdx++ } } - var vr VerificationResult + verified := false if !txGroupSigFailed { verifiedTxnGroups = append(verifiedTxnGroups, bl.txnGroups[txgIdx]) verifiedGroupCtxs = append(verifiedGroupCtxs, bl.groupCtxs[txgIdx]) - vr = VerificationResult{ - txnGroup: bl.txnGroups[txgIdx], - verified: true, - } - } else { - vr = VerificationResult{ - txnGroup: bl.txnGroups[txgIdx], - verified: false, - } + verified = true + } + vr := VerificationResult{ + txnGroup: bl.txnGroups[txgIdx], + verified: verified, } // send the txn result out the pipe select { From fe97bfe2af9b9398c956acba0cd22bd16a0ac90d Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 3 Oct 2022 12:22:54 -0400 Subject: [PATCH 024/156] CR: fix comment --- crypto/batchverifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index f3e8d7ebec..d417a74a9d 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -140,7 +140,7 @@ func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners -// otherwise, returns an error, and sets the indexes of the failed sigs in failed +// otherwise, returns false, and sets the indexes of the failed sigs in failed func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, failed []bool) bool { numberOfSignatures := len(messages) From 50933d66046aaacf05c832e84ef6d3c07d1126be Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 3 Oct 2022 13:30:36 -0400 Subject: [PATCH 025/156] CR: refactor the new interface --- crypto/batchverifier.go | 24 ++++++--------- crypto/batchverifier_test.go | 59 +++++++++++++++++------------------- 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index f3e8d7ebec..a5f327dbeb 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -53,7 +53,6 @@ const minBatchVerifierAlloc = 16 // Batch verifications errors var ( ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification") - errInvalidFailedSlice = errors.New("failed slice size is not equal to the number of enqueued signatures") ) //export ed25519_randombytes_unsafe @@ -110,32 +109,29 @@ func (b *BatchVerifier) getNumberOfEnqueuedSignatures() int { } // Verify verifies that all the signatures are valid. in that case nil is returned -// if the batch is zero an appropriate error is return. func (b *BatchVerifier) Verify() error { - return b.VerifyWithFeedback(nil) + _, err := b.VerifyWithFeedback() + return err } // VerifyWithFeedback verifies that all the signatures are valid. -// failed slice should have len equal to getNumberOfEnqueuedSignatures -// if all sigs are valid, nil will be returned -// if some txns are invalid, true will be set at the appropriate index in failed -func (b *BatchVerifier) VerifyWithFeedback(failed []bool) error { - if failed != nil && len(failed) != b.getNumberOfEnqueuedSignatures() { - return errInvalidFailedSlice - } - +// if all sigs are valid, nil will be returned for err (failed will have all false) +// if some txns are invalid, true will be set in failed at the corresponding indexes, and +// ErrBatchVerificationFailed for err +func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { if b.getNumberOfEnqueuedSignatures() == 0 { - return nil + return nil, nil } + failed = make([]bool, b.getNumberOfEnqueuedSignatures(), b.getNumberOfEnqueuedSignatures()) var messages = make([][]byte, b.getNumberOfEnqueuedSignatures()) for i, m := range b.messages { messages[i] = HashRep(m) } if batchVerificationImpl(messages, b.publicKeys, b.signatures, failed) { - return nil + return failed, nil } - return ErrBatchVerificationFailed + return failed, ErrBatchVerificationFailed } // batchVerificationImpl invokes the ed25519 batch verification algorithm. diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 3ca36b1186..fe2e0fa0eb 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -126,36 +126,6 @@ func TestEmpty(t *testing.T) { require.NoError(t, bv.Verify()) } -// TestBatchVerifierBadFailedArray tests that VerifyWithFeedback returns -// the expected error if the failed array size is not correct -func TestBatchVerifierBadFailedArray(t *testing.T) { - partitiontest.PartitionTest(t) - - var s Seed - bv := MakeBatchVerifierWithHint(4) - for i := 0; i < 4; i++ { - msg := randString() - RandBytes(s[:]) - sigSecrets := GenerateSignatureSecrets(s) - sig := sigSecrets.Sign(msg) - bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) - } - require.Equal(t, 4, bv.getNumberOfEnqueuedSignatures()) - failed := make([]bool, 4, 6) - err := bv.VerifyWithFeedback(failed) - require.NoError(t, err) - - failed = make([]bool, 3, 4) - err = bv.VerifyWithFeedback(failed) - require.Error(t, err) - require.Equal(t, errInvalidFailedSlice, err) - - failed = make([]bool, 5, 5) - err = bv.VerifyWithFeedback(failed) - require.Error(t, err) - require.Equal(t, errInvalidFailedSlice, err) -} - // TestBatchVerifierIndividualResults tests that VerifyWithFeedback // returns the correct failed signature indexes func TestBatchVerifierIndividualResults(t *testing.T) { @@ -166,7 +136,6 @@ func TestBatchVerifierIndividualResults(t *testing.T) { bv := MakeBatchVerifierWithHint(n) var s Seed badSigs := make([]bool, n, n) - failed := make([]bool, n, n) hasBadSig := false for i := 0; i < n; i++ { msg := randString() @@ -182,7 +151,7 @@ func TestBatchVerifierIndividualResults(t *testing.T) { bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) } require.Equal(t, n, bv.getNumberOfEnqueuedSignatures()) - err := bv.VerifyWithFeedback(failed) + failed, err := bv.VerifyWithFeedback() if hasBadSig { require.Error(t, err) } else { @@ -193,3 +162,29 @@ func TestBatchVerifierIndividualResults(t *testing.T) { } } } + +// TestBatchVerifierIndividualResultsAllValid tests that VerifyWithFeedback +// returns the correct failed signature indexes when all are valid +func TestBatchVerifierIndividualResultsAllValid(t *testing.T) { + partitiontest.PartitionTest(t) + + for i := 1; i < 64*2+3; i++ { + n := i + bv := MakeBatchVerifierWithHint(n) + var s Seed + for i := 0; i < n; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, n, bv.getNumberOfEnqueuedSignatures()) + failed, err := bv.VerifyWithFeedback() + require.NoError(t, err) + require.Equal(t, bv.getNumberOfEnqueuedSignatures(), len(failed)) + for _, f := range failed { + require.False(t, f) + } + } +} From 7bc7e8fbb9a3b1bc79d3815efa9cb9b0d301c751 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 3 Oct 2022 13:36:32 -0400 Subject: [PATCH 026/156] formatting --- crypto/batchverifier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index a5f327dbeb..9247e7e360 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -116,13 +116,13 @@ func (b *BatchVerifier) Verify() error { // VerifyWithFeedback verifies that all the signatures are valid. // if all sigs are valid, nil will be returned for err (failed will have all false) -// if some txns are invalid, true will be set in failed at the corresponding indexes, and +// if some txns are invalid, true will be set in failed at the corresponding indexes, and // ErrBatchVerificationFailed for err func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { if b.getNumberOfEnqueuedSignatures() == 0 { return nil, nil } - failed = make([]bool, b.getNumberOfEnqueuedSignatures(), b.getNumberOfEnqueuedSignatures()) + failed = make([]bool, b.getNumberOfEnqueuedSignatures()) var messages = make([][]byte, b.getNumberOfEnqueuedSignatures()) for i, m := range b.messages { From 13d4701b71178ac4bcbd7ebea6054a13f652ab61 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 3 Oct 2022 14:03:34 -0400 Subject: [PATCH 027/156] more refactoring --- crypto/batchverifier.go | 19 +++++++++---------- crypto/onetimesig.go | 6 +++--- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index fc8c676537..2892a2e747 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -122,13 +122,12 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { if b.getNumberOfEnqueuedSignatures() == 0 { return nil, nil } - failed = make([]bool, b.getNumberOfEnqueuedSignatures()) - var messages = make([][]byte, b.getNumberOfEnqueuedSignatures()) for i, m := range b.messages { messages[i] = HashRep(m) } - if batchVerificationImpl(messages, b.publicKeys, b.signatures, failed) { + allValid, failed := batchVerificationImpl(messages, b.publicKeys, b.signatures) + if allValid { return failed, nil } return failed, ErrBatchVerificationFailed @@ -137,7 +136,7 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners // otherwise, returns false, and sets the indexes of the failed sigs in failed -func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature, failed []bool) bool { +func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { numberOfSignatures := len(messages) @@ -172,11 +171,11 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si (**C.uchar)(unsafe.Pointer(signaturesAllocation)), C.size_t(len(messages)), (*C.int)(unsafe.Pointer(valid))) - if allValid != 0 && failed != nil { - for i := 0; i < numberOfSignatures; i++ { - cint := *(*C.int)(unsafe.Pointer(uintptr(valid) + uintptr(i*C.sizeof_int))) - failed[i] = (cint == 0) - } + + failed = make([]bool, numberOfSignatures) + for i := 0; i < numberOfSignatures; i++ { + cint := *(*C.int)(unsafe.Pointer(uintptr(valid) + uintptr(i*C.sizeof_int))) + failed[i] = (cint == 0) } - return allValid == 0 + return allValid == 0, failed } diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index c028a7c895..4026df1308 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -319,12 +319,12 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message Batch: id.Batch, } - return batchVerificationImpl( + allValid, _ := batchVerificationImpl( [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, - []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, nil, + []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, ) - + return allValid } // DeleteBeforeFineGrained deletes ephemeral keys before (but not including) the given id. From 4ad80d9a54594e33df522a00adfa50882e9679de Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 3 Oct 2022 14:13:01 -0400 Subject: [PATCH 028/156] gofmt --- crypto/onetimesig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index 4026df1308..2aaa58bc0c 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -319,7 +319,7 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message Batch: id.Batch, } - allValid, _ := batchVerificationImpl( + allValid, _ := batchVerificationImpl( [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, From 09bf3c2a849504fb4e074ee5c836ba367602a6d6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 4 Oct 2022 16:01:31 -0400 Subject: [PATCH 029/156] fix merge conflicts --- crypto/batchverifier.go | 2 +- crypto/batchverifier_test.go | 59 +++++++++++++++------------------ crypto/onetimesig.go | 2 +- data/transactions/verify/txn.go | 4 +-- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index a2ff82143a..3c3d798b63 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -119,7 +119,7 @@ func (b *BatchVerifier) Verify() error { // if some txns are invalid, true will be set in failed at the corresponding indexes, and // ErrBatchVerificationFailed for err func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { - if b.getNumberOfEnqueuedSignatures() == 0 { + if b.GetNumberOfEnqueuedSignatures() == 0 { return nil, nil } diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index ea5e88b4bd..18e02a3ef3 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -126,36 +126,6 @@ func TestEmpty(t *testing.T) { require.NoError(t, bv.Verify()) } -// TestBatchVerifierBadFailedArray tests that VerifyWithFeedback returns -// the expected error if the failed array size is not correct -func TestBatchVerifierBadFailedArray(t *testing.T) { - partitiontest.PartitionTest(t) - - var s Seed - bv := MakeBatchVerifierWithHint(4) - for i := 0; i < 4; i++ { - msg := randString() - RandBytes(s[:]) - sigSecrets := GenerateSignatureSecrets(s) - sig := sigSecrets.Sign(msg) - bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) - } - require.Equal(t, 4, bv.GetNumberOfEnqueuedSignatures()) - failed := make([]bool, 4, 6) - err := bv.VerifyWithFeedback(failed) - require.NoError(t, err) - - failed = make([]bool, 3, 4) - err = bv.VerifyWithFeedback(failed) - require.Error(t, err) - require.Equal(t, errInvalidFailedSlice, err) - - failed = make([]bool, 5, 5) - err = bv.VerifyWithFeedback(failed) - require.Error(t, err) - require.Equal(t, errInvalidFailedSlice, err) -} - // TestBatchVerifierIndividualResults tests that VerifyWithFeedback // returns the correct failed signature indexes func TestBatchVerifierIndividualResults(t *testing.T) { @@ -166,7 +136,6 @@ func TestBatchVerifierIndividualResults(t *testing.T) { bv := MakeBatchVerifierWithHint(n) var s Seed badSigs := make([]bool, n, n) - failed := make([]bool, n, n) hasBadSig := false for i := 0; i < n; i++ { msg := randString() @@ -182,7 +151,7 @@ func TestBatchVerifierIndividualResults(t *testing.T) { bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) } require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) - err := bv.VerifyWithFeedback(failed) + failed, err := bv.VerifyWithFeedback() if hasBadSig { require.Error(t, err) } else { @@ -193,3 +162,29 @@ func TestBatchVerifierIndividualResults(t *testing.T) { } } } + +// TestBatchVerifierIndividualResultsAllValid tests that VerifyWithFeedback +// returns the correct failed signature indexes when all are valid +func TestBatchVerifierIndividualResultsAllValid(t *testing.T) { + partitiontest.PartitionTest(t) + + for i := 1; i < 64*2+3; i++ { + n := i + bv := MakeBatchVerifierWithHint(n) + var s Seed + for i := 0; i < n; i++ { + msg := randString() + RandBytes(s[:]) + sigSecrets := GenerateSignatureSecrets(s) + sig := sigSecrets.Sign(msg) + bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) + } + require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) + failed, err := bv.VerifyWithFeedback() + require.NoError(t, err) + require.Equal(t, bv.GetNumberOfEnqueuedSignatures(), len(failed)) + for _, f := range failed { + require.False(t, f) + } + } +} diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index 9e9a2fd49f..2aaa58bc0c 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -322,7 +322,7 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message allValid, _ := batchVerificationImpl( [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, - []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, nil, + []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, ) return allValid } diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 73eb661358..eea0f5a368 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -586,9 +586,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { if sm.ctx.Err() != nil { return sm.ctx.Err() } - numSigs := bl.batchVerifier.GetNumberOfEnqueuedSignatures() - failed := make([]bool, numSigs, numSigs) - err := bl.batchVerifier.VerifyWithFeedback(failed) + failed, err := bl.batchVerifier.VerifyWithFeedback() if err != nil && err != crypto.ErrBatchHasFailedSigs { // something bad happened // TODO: report error and discard the batch From 6081a6eacfff34a878d379e5bcc3f81768d06894 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 5 Oct 2022 19:20:20 -0400 Subject: [PATCH 030/156] partial refactoring for adapting to asyncVerify model --- data/transactions/verify/txn.go | 62 +++++++++++++++++++++------------ data/txHandler.go | 31 +++++++++++++++-- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index eea0f5a368..0983b744fa 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -451,13 +451,14 @@ func (w *worksetBuilder) completed() bool { } type VerificationElement struct { - txnGroup []transactions.SignedTxn - contextHdr bookkeeping.BlockHeader + txnGroup []transactions.SignedTxn + context interface{} } type VerificationResult struct { txnGroup []transactions.SignedTxn - verified bool + context interface{} + err error } type streamManager struct { @@ -472,6 +473,7 @@ type batchLoad struct { batchVerifier *crypto.BatchVerifier txnGroups [][]transactions.SignedTxn groupCtxs []*GroupContext + elementContext []interface{} messagesForTxn []int } @@ -492,9 +494,17 @@ func makeBatchLoad() batchLoad { // [wait time] <= [validation time of a single txn] / 2 const singelTxnValidationTime = 100 * time.Millisecond const numberOfExecPoolSeats = 8 + +// internalBufferSize is the size of the chan that will hold the arriving stxns before they get pre-processed +const internalBufferSize = 25000 + //const txnPerWorksetThreshold = 32 -func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.LedgerForSignature, - stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult, verificationPool execpool.BacklogPool) { + +func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( + stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult) { + + stxnChan = make(<-chan VerificationElement, internalBufferSize) + resultChan = make(chan<- VerificationResult) sm := streamManager{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), @@ -504,10 +514,11 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le cache: cache, } - for x := 0; x < numberOfExecPoolSeats; x++ { - sm.seatReturnChan <- struct{}{} - } go func() { + for x := 0; x < numberOfExecPoolSeats; x++ { + sm.seatReturnChan <- struct{}{} + } + bl := makeBatchLoad() timer := time.NewTicker(singelTxnValidationTime / 2) var added bool @@ -516,7 +527,7 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le case stx := <-stxnChan: timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchPrep(stx.txnGroup, stx.contextHdr, ledger, bl.batchVerifier) + groupCtx, err := txnGroupBatchPrep(stx.txnGroup, bookkeeping.BlockHeader{}, ledger, bl.batchVerifier) //TODO: report the error ctx.Err() if err != nil { @@ -524,6 +535,7 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } bl.groupCtxs = append(bl.groupCtxs, groupCtx) bl.txnGroups = append(bl.txnGroups, stx.txnGroup) + bl.elementContext = append(bl.elementContext, stx.context) bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) if len(bl.groupCtxs) >= txnPerWorksetThreshold { // TODO: the limit of 32 should not pass @@ -547,7 +559,7 @@ func Stream(ctx context.Context, cache VerifiedTransactionCache, ledger logic.Le } } }() - + return } func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added bool) { @@ -577,6 +589,21 @@ func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added b return } +// send the result out the chan +func (sm *streamManager) sendOut(vr VerificationResult) { + // send the txn result out the pipe + select { + case sm.resultChan <- vr: + /* + // if the channel is not accepting, should not block here + // report dropped txn. caching is fine, if it comes back in the block + default: + fmt.Println("skipped!!") + //TODO: report this + */ + } +} + func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { function := func(arg interface{}) interface{} { @@ -617,20 +644,11 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { } vr := VerificationResult{ txnGroup: bl.txnGroups[txgIdx], - verified: verified, - } - // send the txn result out the pipe - select { - case sm.resultChan <- vr: - /* - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: - fmt.Println("skipped!!") - //TODO: report this - */ + context: bl.elementContext[txgIdx], + err: nil, } + sm.snedOut(vr) } // loading them all at once to lock the cache once sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) diff --git a/data/txHandler.go b/data/txHandler.go index cd4c25c8e0..75b5ce9eb0 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -66,6 +66,7 @@ type TxHandler struct { net network.GossipNode ctx context.Context ctxCancel context.CancelFunc + streamVerifierChan chan []transactions.SignedTxn } // MakeTxHandler makes a new handler for transaction messages @@ -90,9 +91,32 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go backlogQueue: make(chan *txBacklogMsg, txBacklogSize), postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, + streamVerifierChan: make(chan verify.VerificationElement), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + var outChan chan<- VerificationResult + handler.streamVerifierChan, outChan = verifier.MakeStream(ctx, handler.ledger, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache) + + // This goroutine will listen to the results of the handled txn verification and pass them to postVerificationQueue + go func() { + for { + select { + case <-cts.Done(): + return + case result := <-outChan: + txBLMsg := result.context.(*txBacklogMsg) + txBLMsg.verificationErr = result.err + select { + case handler.postVerificationQueue <- tx: + default: + // we failed to write to the output queue, since the queue was full. + // adding the metric here allows us to monitor how frequently it happens. + transactionMessagesDroppedFromPool.Inc(nil) + } + } + } + }() return handler } @@ -147,8 +171,9 @@ func (handler *TxHandler) backlogWorker() { continue } - // enqueue the task to the verification pool. - handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, wi, nil) + handler.streamVerifierChan <- verify.VerificationElement{txnGroup: wi.unverifiedTxGroup, context: wi} + // // enqueue the task to the verification pool. + // handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, wi, nil) case wi, ok := <-handler.postVerificationQueue: if !ok { @@ -193,6 +218,8 @@ func (handler *TxHandler) postprocessCheckedTxn(wi *txBacklogMsg) { handler.net.Relay(handler.ctx, protocol.TxnTag, reencode(verifiedTxGroup), false, wi.rawmsg.Sender) } +//func batchVerifySignature( + // asyncVerifySignature verifies that the given transaction group is valid, and update the txBacklogMsg data structure accordingly. func (handler *TxHandler) asyncVerifySignature(arg interface{}) interface{} { tx := arg.(*txBacklogMsg) From 01700bb696ce6497259831575f2e28701b5af0a4 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 13:00:36 -0400 Subject: [PATCH 031/156] add new block watcher --- data/transactions/verify/txn.go | 52 ++++++++++++++++++++++------ data/transactions/verify/txn_test.go | 15 ++++---- data/txHandler.go | 12 ++++++- 3 files changed, 61 insertions(+), 18 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 0983b744fa..e8e3c86d99 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -23,12 +23,15 @@ import ( "fmt" "time" + "github.com/algorand/go-deadlock" + "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/execpool" "github.com/algorand/go-algorand/util/metrics" @@ -38,6 +41,8 @@ var logicGoodTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_ var logicRejTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_logic_rej", Description: "Total transaction scripts executed and rejected"}) var logicErrTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_logic_err", Description: "Total transaction scripts executed and errored"}) +var InvalidSignature = errors.New("At least one signature didn't pass verification") + // The PaysetGroups is taking large set of transaction groups and attempt to verify their validity using multiple go-routines. // When doing so, it attempts to break these into smaller "worksets" where each workset takes about 2ms of execution time in order // to avoid context switching overhead while providing good validation cancelation responsiveness. Each one of these worksets is @@ -79,6 +84,7 @@ func PrepareGroupContext(group []transactions.SignedTxn, contextHdr bookkeeping. consensusParams, ok := config.Consensus[contextHdr.CurrentProtocol] if !ok { return nil, protocol.Error(contextHdr.CurrentProtocol) + // return nil, fmt.Errorf("Unsupported protocol: %w", protocol.Error(contextHdr.CurrentProtocol)) } return &GroupContext{ specAddrs: transactions.SpecialAddresses{ @@ -477,6 +483,31 @@ type batchLoad struct { messagesForTxn []int } +type NewBlockWatcher struct { + blkHeader bookkeeping.BlockHeader + mu deadlock.RWMutex +} + +func MakeNewBlockWatcher(blkHdr bookkeeping.BlockHeader) (nbw NewBlockWatcher) { + nbw = NewBlockWatcher{ + blkHeader: blkHdr, + } + return nbw +} +func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { + if nbw.blkHeader.Round >= block.BlockHeader.Round { + return + } + nbw.mu.Lock() + defer nbw.mu.Unlock() + nbw.blkHeader = block.BlockHeader +} +func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { + nbw.mu.RLock() + defer nbw.mu.RUnlock() + return nbw.blkHeader +} + func makeBatchLoad() batchLoad { bl := batchLoad{} bl.batchVerifier = crypto.MakeBatchVerifier() @@ -500,11 +531,11 @@ const internalBufferSize = 25000 //const txnPerWorksetThreshold = 32 -func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( - stxnChan <-chan VerificationElement, resultChan chan<- VerificationResult) { +func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( + stxnInput chan<- VerificationElement, resultOtput <-chan VerificationResult) { - stxnChan = make(<-chan VerificationElement, internalBufferSize) - resultChan = make(chan<- VerificationResult) + stxnChan := make(chan VerificationElement, internalBufferSize) + resultChan := make(chan VerificationResult) sm := streamManager{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), @@ -527,7 +558,7 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, verificati case stx := <-stxnChan: timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchPrep(stx.txnGroup, bookkeeping.BlockHeader{}, ledger, bl.batchVerifier) + groupCtx, err := txnGroupBatchPrep(stx.txnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) //TODO: report the error ctx.Err() if err != nil { @@ -559,7 +590,7 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, verificati } } }() - return + return stxnChan, resultChan } func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added bool) { @@ -636,19 +667,20 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { failedSigIdx++ } } - verified := false + var result error if !txGroupSigFailed { verifiedTxnGroups = append(verifiedTxnGroups, bl.txnGroups[txgIdx]) verifiedGroupCtxs = append(verifiedGroupCtxs, bl.groupCtxs[txgIdx]) - verified = true + } else { + result = InvalidSignature } vr := VerificationResult{ txnGroup: bl.txnGroups[txgIdx], context: bl.elementContext[txgIdx], - err: nil, + err: result, } - sm.snedOut(vr) + sm.sendOut(vr) } // loading them all at once to lock the cache once sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index b9f67bc023..b9f7278840 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -542,11 +542,12 @@ func TestStreamVerifier(t *testing.T) { numOfTxnGroups := len(txnGroups) ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) - stxnChan := make(chan VerificationElement, 3) - resultChan := make(chan VerificationResult, 3) defer cancel() - Stream(ctx, cache, nil, stxnChan, resultChan, verificationPool) + + nbw := MakeNewBlockWatcher(blkHdr) + + stxnChan, resultChan := MakeStream(ctx, nil, nbw, verificationPool, cache) badTxnGroups := make(map[crypto.Signature]struct{}) @@ -566,15 +567,15 @@ func TestStreamVerifier(t *testing.T) { for x := 0; x < numOfTxnGroups; x++ { select { case result := <-resultChan: - + if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { delete(badTxnGroups, result.txnGroup[0].Sig) - if result.verified { + if result.err == nil { err := fmt.Errorf("%dth transaction varified with a bad sig", x) errChan <- err } } else { - if !result.verified { + if result.err != nil { err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) errChan <- err } @@ -592,7 +593,7 @@ func TestStreamVerifier(t *testing.T) { case <-ctx.Done(): break default: - stxnChan <- VerificationElement{txnGroup: tg, contextHdr: blkHdr} + stxnChan <- VerificationElement{txnGroup: tg, context: nil} } } }() diff --git a/data/txHandler.go b/data/txHandler.go index 75b5ce9eb0..d1eea7666c 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -96,7 +96,17 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) var outChan chan<- VerificationResult - handler.streamVerifierChan, outChan = verifier.MakeStream(ctx, handler.ledger, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache) + + + latest := handler.ledger.Latest() + latestHdr, err := handler.ledger.BlockHdr(latest) + if err != nil { + logging.Base().Warnf("Could not get header for previous block %v: %v", latest, err) + return network.OutgoingMessage{}, true + } + nbw := verifier.MakeNewBlockWatcher(latestHdr) + handler.ledger.RegisterBlockListeners([]BlockListener{&nbw}) + handler.streamVerifierChan, outChan = verifier.MakeStream(ctx, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache) // This goroutine will listen to the results of the handled txn verification and pass them to postVerificationQueue go func() { From 1bb577197850f433927f45ba356db29b04206280 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 15:18:26 -0400 Subject: [PATCH 032/156] more on txHandler --- data/transactions/verify/txn.go | 22 +++++++++++----------- data/txHandler.go | 25 +++++++++++++------------ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index e8e3c86d99..6912f660a4 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -457,14 +457,14 @@ func (w *worksetBuilder) completed() bool { } type VerificationElement struct { - txnGroup []transactions.SignedTxn - context interface{} + TxnGroup []transactions.SignedTxn + Context interface{} } type VerificationResult struct { - txnGroup []transactions.SignedTxn - context interface{} - err error + TxnGroup []transactions.SignedTxn + Context interface{} + Err error } type streamManager struct { @@ -558,15 +558,15 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw NewBlo case stx := <-stxnChan: timer = time.NewTicker(singelTxnValidationTime / 2) // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchPrep(stx.txnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) + groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) //TODO: report the error ctx.Err() if err != nil { continue } bl.groupCtxs = append(bl.groupCtxs, groupCtx) - bl.txnGroups = append(bl.txnGroups, stx.txnGroup) - bl.elementContext = append(bl.elementContext, stx.context) + bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) + bl.elementContext = append(bl.elementContext, stx.Context) bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) if len(bl.groupCtxs) >= txnPerWorksetThreshold { // TODO: the limit of 32 should not pass @@ -675,9 +675,9 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { result = InvalidSignature } vr := VerificationResult{ - txnGroup: bl.txnGroups[txgIdx], - context: bl.elementContext[txgIdx], - err: result, + TxnGroup: bl.txnGroups[txgIdx], + Context: bl.elementContext[txgIdx], + Err: result, } sm.sendOut(vr) diff --git a/data/txHandler.go b/data/txHandler.go index d1eea7666c..dea9674e89 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -29,6 +29,7 @@ import ( "github.com/algorand/go-algorand/data/pools" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/verify" + ledgerpkg "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" @@ -66,7 +67,7 @@ type TxHandler struct { net network.GossipNode ctx context.Context ctxCancel context.CancelFunc - streamVerifierChan chan []transactions.SignedTxn + streamVerifierChan chan<- verify.VerificationElement } // MakeTxHandler makes a new handler for transaction messages @@ -95,30 +96,30 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - var outChan chan<- VerificationResult + var outChan <-chan verify.VerificationResult latest := handler.ledger.Latest() latestHdr, err := handler.ledger.BlockHdr(latest) if err != nil { - logging.Base().Warnf("Could not get header for previous block %v: %v", latest, err) - return network.OutgoingMessage{}, true + logging.Base().Fatal("MakeTxHandler: Could not get header for previous block") + return nil } - nbw := verifier.MakeNewBlockWatcher(latestHdr) - handler.ledger.RegisterBlockListeners([]BlockListener{&nbw}) - handler.streamVerifierChan, outChan = verifier.MakeStream(ctx, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache) + nbw := verify.MakeNewBlockWatcher(latestHdr) + handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{&nbw}) + handler.streamVerifierChan, outChan = verify.MakeStream(handler.ctx, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) // This goroutine will listen to the results of the handled txn verification and pass them to postVerificationQueue go func() { for { select { - case <-cts.Done(): + case <-handler.ctx.Done(): return case result := <-outChan: - txBLMsg := result.context.(*txBacklogMsg) - txBLMsg.verificationErr = result.err + txBLMsg := result.Context.(*txBacklogMsg) + txBLMsg.verificationErr = result.Err select { - case handler.postVerificationQueue <- tx: + case handler.postVerificationQueue <- txBLMsg: default: // we failed to write to the output queue, since the queue was full. // adding the metric here allows us to monitor how frequently it happens. @@ -181,7 +182,7 @@ func (handler *TxHandler) backlogWorker() { continue } - handler.streamVerifierChan <- verify.VerificationElement{txnGroup: wi.unverifiedTxGroup, context: wi} + handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} // // enqueue the task to the verification pool. // handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, wi, nil) From d1d3df30c260072bc1e59aaf7349de25378559b5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 16:50:48 -0400 Subject: [PATCH 033/156] fix the test --- data/transactions/verify/txn.go | 1 - data/transactions/verify/txn_test.go | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 6912f660a4..2031689a02 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -679,7 +679,6 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { Context: bl.elementContext[txgIdx], Err: result, } - sm.sendOut(vr) } // loading them all at once to lock the cache once diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index b9f7278840..a6b6bbf627 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -568,14 +568,14 @@ func TestStreamVerifier(t *testing.T) { select { case result := <-resultChan: - if _, has := badTxnGroups[result.txnGroup[0].Sig]; has { - delete(badTxnGroups, result.txnGroup[0].Sig) - if result.err == nil { + if _, has := badTxnGroups[result.TxnGroup[0].Sig]; has { + delete(badTxnGroups, result.TxnGroup[0].Sig) + if result.Err == nil { err := fmt.Errorf("%dth transaction varified with a bad sig", x) errChan <- err } } else { - if result.err != nil { + if result.Err != nil { err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) errChan <- err } @@ -593,7 +593,7 @@ func TestStreamVerifier(t *testing.T) { case <-ctx.Done(): break default: - stxnChan <- VerificationElement{txnGroup: tg, context: nil} + stxnChan <- VerificationElement{TxnGroup: tg, Context: nil} } } }() From 9b13063ca888f1396ff072358dc252b931b60540 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 19:15:48 -0400 Subject: [PATCH 034/156] add comments and fix lint issues --- data/transactions/verify/txn.go | 68 ++++++++++++++++++++++++--------- data/txHandler.go | 3 +- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 2031689a02..7df8573380 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -41,7 +41,8 @@ var logicGoodTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_ var logicRejTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_logic_rej", Description: "Total transaction scripts executed and rejected"}) var logicErrTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_logic_err", Description: "Total transaction scripts executed and errored"}) -var InvalidSignature = errors.New("At least one signature didn't pass verification") +// ErrInvalidSignature is the error returned to report that at least one signature is invalid +var ErrInvalidSignature = errors.New("At least one signature didn't pass verification") // The PaysetGroups is taking large set of transaction groups and attempt to verify their validity using multiple go-routines. // When doing so, it attempts to break these into smaller "worksets" where each workset takes about 2ms of execution time in order @@ -456,11 +457,17 @@ func (w *worksetBuilder) completed() bool { return w.idx >= len(w.payset) } +// VerificationElement is the element passed the Stream verifier +// Context is a reference associated with the txn group which is passed +// with the result type VerificationElement struct { TxnGroup []transactions.SignedTxn Context interface{} } +// VerificationResult is the result of the txn group verification +// Context is a reference associated with the txn group which was +// initially passed to the stream verifier type VerificationResult struct { TxnGroup []transactions.SignedTxn Context interface{} @@ -483,17 +490,22 @@ type batchLoad struct { messagesForTxn []int } +// NewBlockWatcher is a struct used to provide a new block header to the +// stream verifier type NewBlockWatcher struct { blkHeader bookkeeping.BlockHeader mu deadlock.RWMutex } -func MakeNewBlockWatcher(blkHdr bookkeeping.BlockHeader) (nbw NewBlockWatcher) { - nbw = NewBlockWatcher{ +// MakeNewBlockWatcher construct a new block watcher with the initial blkHdr +func MakeNewBlockWatcher(blkHdr bookkeeping.BlockHeader) (nbw *NewBlockWatcher) { + nbw = &NewBlockWatcher{ blkHeader: blkHdr, } return nbw } + +// OnNewBlock implements the interface to subscribe to new block notifications from the ledger func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { if nbw.blkHeader.Round >= block.BlockHeader.Round { return @@ -502,6 +514,7 @@ func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore defer nbw.mu.Unlock() nbw.blkHeader = block.BlockHeader } + func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { nbw.mu.RLock() defer nbw.mu.RUnlock() @@ -531,7 +544,10 @@ const internalBufferSize = 25000 //const txnPerWorksetThreshold = 32 -func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( +// MakeStream creates a new stream verifier and returns the chans used to send txn groups +// to it and obtain the txn signature verification result from +func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, + verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( stxnInput chan<- VerificationElement, resultOtput <-chan VerificationResult) { stxnChan := make(chan VerificationElement, internalBufferSize) @@ -599,7 +615,11 @@ func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added b // the varifier might be saturated. // block and wait for a free seat <-sm.seatReturnChan - sm.addVerificationTaskToThePool(bl) + err := sm.addVerificationTaskToThePool(bl) + if err != nil { + // TODO: report the error + fmt.Println(err) + } timer = time.NewTicker(singelTxnValidationTime / 2) added = true return @@ -608,9 +628,13 @@ func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added b // more signatures instead of waiting here select { case <-sm.seatReturnChan: - sm.addVerificationTaskToThePool(bl) + err := sm.addVerificationTaskToThePool(bl) timer = time.NewTicker(singelTxnValidationTime / 2) added = true + if err != nil { + // TODO: report the error + fmt.Println(err) + } // TODO: queue to the pool. // fmt.Println(err) default: @@ -623,16 +647,19 @@ func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added b // send the result out the chan func (sm *streamManager) sendOut(vr VerificationResult) { // send the txn result out the pipe - select { - case sm.resultChan <- vr: - /* - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: - fmt.Println("skipped!!") - //TODO: report this - */ - } + sm.resultChan <- vr + /* + select { + case sm.resultChan <- vr: + + // if the channel is not accepting, should not block here + // report dropped txn. caching is fine, if it comes back in the block + default: + fmt.Println("skipped!!") + //TODO: report this + + } + */ } func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { @@ -646,6 +673,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { } failed, err := bl.batchVerifier.VerifyWithFeedback() if err != nil && err != crypto.ErrBatchHasFailedSigs { + fmt.Println(err) // something bad happened // TODO: report error and discard the batch } @@ -672,7 +700,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { verifiedTxnGroups = append(verifiedTxnGroups, bl.txnGroups[txgIdx]) verifiedGroupCtxs = append(verifiedGroupCtxs, bl.groupCtxs[txgIdx]) } else { - result = InvalidSignature + result = ErrInvalidSignature } vr := VerificationResult{ TxnGroup: bl.txnGroups[txgIdx], @@ -682,7 +710,11 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { sm.sendOut(vr) } // loading them all at once to lock the cache once - sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) + err = sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) + if err != nil { + // TODO: handle the error + fmt.Println(err) + } return struct{}{} } err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, bl, sm.seatReturnChan) diff --git a/data/txHandler.go b/data/txHandler.go index dea9674e89..5735133b5c 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -98,7 +98,6 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) var outChan <-chan verify.VerificationResult - latest := handler.ledger.Latest() latestHdr, err := handler.ledger.BlockHdr(latest) if err != nil { @@ -106,7 +105,7 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go return nil } nbw := verify.MakeNewBlockWatcher(latestHdr) - handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{&nbw}) + handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) handler.streamVerifierChan, outChan = verify.MakeStream(handler.ctx, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) // This goroutine will listen to the results of the handled txn verification and pass them to postVerificationQueue From ad0887be9996d39a9bf32bf3e6e71f380fbfa8dc Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 19:46:12 -0400 Subject: [PATCH 035/156] cleanup --- data/transactions/verify/txn.go | 1 - 1 file changed, 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 7df8573380..28c01432f7 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -85,7 +85,6 @@ func PrepareGroupContext(group []transactions.SignedTxn, contextHdr bookkeeping. consensusParams, ok := config.Consensus[contextHdr.CurrentProtocol] if !ok { return nil, protocol.Error(contextHdr.CurrentProtocol) - // return nil, fmt.Errorf("Unsupported protocol: %w", protocol.Error(contextHdr.CurrentProtocol)) } return &GroupContext{ specAddrs: transactions.SpecialAddresses{ From 04a3fe83a0fed02c2a8cbd1beceb550cde39cca3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 19:50:39 -0400 Subject: [PATCH 036/156] merge conflict fix --- data/transactions/verify/txn_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 530c980de7..879607a21f 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -577,7 +577,7 @@ func TestTxnGroupCacheUpdate(t *testing.T) { restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchVerificationFailed.Error()) + verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) } // TestTxnGroupCacheUpdateMultiSig makes sure that a payment transaction signed with multisig @@ -599,7 +599,7 @@ func TestTxnGroupCacheUpdateMultiSig(t *testing.T) { restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Msig.Subsigs[0].Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchVerificationFailed.Error()) + verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) } // TestTxnGroupCacheUpdateFailLogic test makes sure that a payment transaction contains a logic (and no signature) @@ -682,7 +682,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Lsig.Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchVerificationFailed.Error()) + verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) // signature is correct and logic fails breakSignatureFunc = func(txn *transactions.SignedTxn) { @@ -766,7 +766,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txn.Lsig.Msig.Subsigs[0].Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchVerificationFailed.Error()) + verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) // signature is correct and logic fails breakSignatureFunc = func(txn *transactions.SignedTxn) { txn.Lsig.Args[0][0]++ From 94a9d616b4dfdc59a3d61c021314c13cc6a58ef6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 20:06:52 -0400 Subject: [PATCH 037/156] merge fallouts --- crypto/batchverifier_test.go | 65 +----------------------------------- 1 file changed, 1 insertion(+), 64 deletions(-) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 98e8d05066..c1f081115c 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -154,7 +154,7 @@ func TestBatchVerifierIndividualResults(t *testing.T) { } bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) } - require.Equal(t, n, bv.getNumberOfEnqueuedSignatures()) + require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) failed, err := bv.VerifyWithFeedback() if hasBadSig { require.ErrorIs(t, err, ErrBatchVerificationFailed) @@ -168,69 +168,6 @@ func TestBatchVerifierIndividualResults(t *testing.T) { } } -// TestBatchVerifierIndividualResultsAllValid tests that VerifyWithFeedback -// returns the correct failed signature indexes when all are valid -func TestBatchVerifierIndividualResultsAllValid(t *testing.T) { - partitiontest.PartitionTest(t) - - for i := 1; i < 64*2+3; i++ { - n := i - bv := MakeBatchVerifierWithHint(n) - var s Seed - for i := 0; i < n; i++ { - msg := randString() - RandBytes(s[:]) - sigSecrets := GenerateSignatureSecrets(s) - sig := sigSecrets.Sign(msg) - bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) - } - require.Equal(t, n, bv.getNumberOfEnqueuedSignatures()) - failed, err := bv.VerifyWithFeedback() - require.NoError(t, err) - require.Equal(t, bv.getNumberOfEnqueuedSignatures(), len(failed)) - for _, f := range failed { - require.False(t, f) - } - } -} - -// TestBatchVerifierIndividualResults tests that VerifyWithFeedback -// returns the correct failed signature indexes -func TestBatchVerifierIndividualResults(t *testing.T) { - partitiontest.PartitionTest(t) - - for i := 1; i < 64*2+3; i++ { - n := i - bv := MakeBatchVerifierWithHint(n) - var s Seed - badSigs := make([]bool, n, n) - hasBadSig := false - for i := 0; i < n; i++ { - msg := randString() - RandBytes(s[:]) - sigSecrets := GenerateSignatureSecrets(s) - sig := sigSecrets.Sign(msg) - if rand.Float32() > 0.5 { - // make a bad sig - sig[0] = sig[0] + 1 - badSigs[i] = true - hasBadSig = true - } - bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig) - } - require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) - failed, err := bv.VerifyWithFeedback() - if hasBadSig { - require.Error(t, err) - } else { - require.NoError(t, err) - } - for i := range badSigs { - require.Equal(t, badSigs[i], failed[i]) - } - } -} - // TestBatchVerifierIndividualResultsAllValid tests that VerifyWithFeedback // returns the correct failed signature indexes when all are valid func TestBatchVerifierIndividualResultsAllValid(t *testing.T) { From ed49a2c5c3277963c1221ee07c99ad4d1745ca7d Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 20:35:55 -0400 Subject: [PATCH 038/156] refactoring in txHandler --- data/txHandler.go | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/data/txHandler.go b/data/txHandler.go index f06a829a9c..ac73bcbaed 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -96,8 +96,9 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - var outChan <-chan verify.VerificationResult + // prepare the transaction stream verifer + var outChan <-chan verify.VerificationResult latest := handler.ledger.Latest() latestHdr, err := handler.ledger.BlockHdr(latest) if err != nil { @@ -106,28 +107,30 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } nbw := verify.MakeNewBlockWatcher(latestHdr) handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - handler.streamVerifierChan, outChan = verify.MakeStream(handler.ctx, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) + handler.streamVerifierChan, outChan = verify.MakeStream(handler.ctx, + handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) + go processTxnStreamVerResults(handler.ctx, outChan, handler.postVerificationQueue) + return handler +} - // This goroutine will listen to the results of the handled txn verification and pass them to postVerificationQueue - go func() { - for { +func processTxnStreamVerResults(ctx context.Context, outChan <-chan verify.VerificationResult, + postVerificationQueue chan *txBacklogMsg) { + for { + select { + case <-ctx.Done(): + return + case result := <-outChan: + txBLMsg := result.Context.(*txBacklogMsg) + txBLMsg.verificationErr = result.Err select { - case <-handler.ctx.Done(): - return - case result := <-outChan: - txBLMsg := result.Context.(*txBacklogMsg) - txBLMsg.verificationErr = result.Err - select { - case handler.postVerificationQueue <- txBLMsg: - default: - // we failed to write to the output queue, since the queue was full. - // adding the metric here allows us to monitor how frequently it happens. - transactionMessagesDroppedFromPool.Inc(nil) - } + case postVerificationQueue <- txBLMsg: + default: + // we failed to write to the output queue, since the queue was full. + // adding the metric here allows us to monitor how frequently it happens. + transactionMessagesDroppedFromPool.Inc(nil) } } - }() - return handler + } } // Start enables the processing of incoming messages at the transaction handler @@ -182,8 +185,6 @@ func (handler *TxHandler) backlogWorker() { } handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} - // // enqueue the task to the verification pool. - // handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, wi, nil) case wi, ok := <-handler.postVerificationQueue: if !ok { From 1be683ec2f06dcb8c2a4824ce44418bd26a5a655 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 6 Oct 2022 20:41:03 -0400 Subject: [PATCH 039/156] fix merge fallout --- crypto/batchverifier_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index c1f081115c..8d47857a73 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -157,7 +157,7 @@ func TestBatchVerifierIndividualResults(t *testing.T) { require.Equal(t, n, bv.GetNumberOfEnqueuedSignatures()) failed, err := bv.VerifyWithFeedback() if hasBadSig { - require.ErrorIs(t, err, ErrBatchVerificationFailed) + require.ErrorIs(t, err, ErrBatchHasFailedSigs) } else { require.NoError(t, err) } From 852b3bf7271b76a10223d4ae68429bd81c2f9ebc Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 7 Oct 2022 23:51:27 -0400 Subject: [PATCH 040/156] improved timing management and refactoring --- data/transactions/verify/txn.go | 88 ++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 28c01432f7..b44ef93206 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -535,8 +535,9 @@ func makeBatchLoad() batchLoad { // [validation time added to the group by one more txn] = [validation time of a single txn] / 2 // This gives us: // [wait time] <= [validation time of a single txn] / 2 -const singelTxnValidationTime = 100 * time.Millisecond -const numberOfExecPoolSeats = 8 +const waitForNextTxnDuration = 50 * time.Millisecond +const waitForFirstTxnDuration = 2000 * time.Millisecond +const numberOfExecPoolSeats = 64 // internalBufferSize is the size of the chan that will hold the arriving stxns before they get pre-processed const internalBufferSize = 25000 @@ -566,39 +567,74 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBl } bl := makeBatchLoad() - timer := time.NewTicker(singelTxnValidationTime / 2) + timer := time.NewTicker(waitForFirstTxnDuration) var added bool for { select { case stx := <-stxnChan: - timer = time.NewTicker(singelTxnValidationTime / 2) + isFirstInBatch := bl.batchVerifier.GetNumberOfEnqueuedSignatures() == 0 // TODO: separate operations here, and get the sig verification inside LogicSig outside groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) //TODO: report the error ctx.Err() if err != nil { + // verification failed. can return here + vr := VerificationResult{ + TxnGroup: stx.TxnGroup, + Context: stx.Context, + Err: err, + } + sm.sendOut(vr) + continue + } + + numEnqueued := bl.batchVerifier.GetNumberOfEnqueuedSignatures() + // if nothing is added to the batch, then the txngroup can be returned here + if numEnqueued == 0 || + (len(bl.messagesForTxn) > 0 && numEnqueued == bl.messagesForTxn[len(bl.messagesForTxn)-1]) { + vr := VerificationResult{ + TxnGroup: stx.TxnGroup, + Context: stx.Context, + Err: err, + } + sm.sendOut(vr) continue } + bl.groupCtxs = append(bl.groupCtxs, groupCtx) bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) bl.elementContext = append(bl.elementContext, stx.Context) - bl.messagesForTxn = append(bl.messagesForTxn, bl.batchVerifier.GetNumberOfEnqueuedSignatures()) + bl.messagesForTxn = append(bl.messagesForTxn, numEnqueued) if len(bl.groupCtxs) >= txnPerWorksetThreshold { // TODO: the limit of 32 should not pass - timer, added = sm.processBatch(bl) - if added { - bl = makeBatchLoad() + err := sm.processFullBatch(bl) + if err != nil { + fmt.Println(err) + } + bl = makeBatchLoad() + // starting a new batch. Can wait long, since nothing is blocked + timer.Reset(waitForFirstTxnDuration) + } else { + // the batch is not full, can wait for some more + if isFirstInBatch { + // reset the timer only when a new batch started + timer.Reset(waitForNextTxnDuration) } } case <-timer.C: + // timer ticked. it is time to send the batch even if it is not full if len(bl.groupCtxs) == 0 { - // nothing yet... wait some more - timer = time.NewTicker(singelTxnValidationTime / 2) + // nothing batched yet... wait some more + timer.Reset(waitForFirstTxnDuration) continue } - timer, added = sm.processBatch(bl) + added = sm.processBatch(bl) if added { bl = makeBatchLoad() + timer.Reset(waitForFirstTxnDuration) + } else { + // was not added because no available seats. wait for some more txns + timer.Reset(waitForNextTxnDuration) } case <-ctx.Done(): return //TODO: report the error ctx.Err() @@ -608,37 +644,29 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBl return stxnChan, resultChan } -func (sm *streamManager) processBatch(bl batchLoad) (timer *time.Ticker, added bool) { - if bl.batchVerifier.GetNumberOfEnqueuedSignatures() >= txnPerWorksetThreshold { - // Should not allow addition of more txns to the batch - // the varifier might be saturated. - // block and wait for a free seat - <-sm.seatReturnChan - err := sm.addVerificationTaskToThePool(bl) - if err != nil { - // TODO: report the error - fmt.Println(err) - } - timer = time.NewTicker(singelTxnValidationTime / 2) - added = true - return - } - // Otherwise, if cannot find a seat, can go back and collect +func (sm *streamManager) processFullBatch(bl batchLoad) (err error) { + // This is a full batch, no additions are allowed + // block and wait for a free seat + <-sm.seatReturnChan + err = sm.addVerificationTaskToThePool(bl) + return +} + +func (sm *streamManager) processBatch(bl batchLoad) (added bool) { + // if cannot find a seat, can go back and collect // more signatures instead of waiting here select { case <-sm.seatReturnChan: err := sm.addVerificationTaskToThePool(bl) - timer = time.NewTicker(singelTxnValidationTime / 2) - added = true if err != nil { // TODO: report the error fmt.Println(err) } + added = true // TODO: queue to the pool. // fmt.Println(err) default: // if no free seats, wait some more for more txns - timer = time.NewTicker(singelTxnValidationTime / 2) } return } From 948c316740c6399c572e3ad3a1e7a1a20eea718e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 12 Oct 2022 19:18:56 -0400 Subject: [PATCH 041/156] draft of perf test --- data/txHandler_test.go | 155 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 14a5495eb2..a5a434d704 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -17,6 +17,7 @@ package data import ( + "encoding/binary" "fmt" "io" "math/rand" @@ -129,6 +130,160 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { }) } +func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { + const numUsers = 100 + log := logging.TestingLog(b) + log.SetLevel(logging.Warn) + secrets := make([]*crypto.SignatureSecrets, numUsers) + addresses := make([]basics.Address, numUsers) + + genesis := make(map[basics.Address]basics.AccountData) + for i := 0; i < numUsers; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + genesis[addr] = basics.AccountData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, + } + } + + genesis[poolAddr] = basics.AccountData{ + Status: basics.NotParticipating, + MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, + } + + require.Equal(b, len(genesis), numUsers+1) + genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) + require.NoError(b, err) + + l := ledger + + cfg.TxPoolSize = 75000 + cfg.EnableProcessBlockStats = false + tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) + txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + + var outChan chan *txBacklogMsg + + // Make a test backlog worker, which is simiar to backlogWorker, but sends the results + // through the outChan instead of passing it to postprocessCheckedTxn + go func() { + defer handler.backlogWg.Done() + for { + // prioritize the postVerificationQueue + select { + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + outChan <- wi + // restart the loop so that we could empty out the post verification queue. + continue + default: + } + + // we have no more post verification items. wait for either backlog queue item or post verification item. + select { + case wi, ok := <-handler.backlogQueue: + if !ok { + return + } + if handler.checkAlreadyCommitted(wi) { + continue + } + + handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + outChan <- wi + + case <-handler.ctx.Done(): + return + } + } + }() + + goodTxnGroups := make(map[Uint64], interface{}) + badTxnGroups := make(map[Uint64], interface{}) + makeSignedTxnGroups := func(N int) [][]transactions.SignedTxn { + maxGrpSize := proto.MaxTxGroupSize + ret := make([][]transactions.SignedTxn, 0, N) + for u := 0; u < N; u++ { + grpSize := rand.Intn(maxGrpSize) + var txGroup transactions.TxGroup + txns := make([]transactions.Transaction, grpSize) + for g := 0; g < grpSize; g++ { + // generate transactions + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, u) + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: addresses[(u+g)%numUsers], + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, + FirstValid: 0, + LastValid: basics.Round(proto.MaxTxnLife), + Note: noteField, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: addresses[(u+g+1)%numUsers], + Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, + }, + } + txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.HashObj(tx)) + txns = append(txns, tx) + } + groupHash := crypto.HashObj(txGroup) + signedTxGroup = make([]transactions.SignedTxn, 0, grpSize) + for g, txn := range txns { + txn.Group = groupHash + signedTx := txn.Sign(secrets[(u+g)%numUsers]) + signedTxGroup = append(signedTxGroup, signedTx) + } + // randomly make bad signatures + if rand.Float32() < 0.3 { + signedTxGroup[rand.Intn(grpSize)].Sig[0] = signedTxGroup[rand.Intn(grpSize)].Sig[0] + 1 + badTxnGroups[u] = struct{}{} + } else { + goodTxnGroups[u] = struct{}{} + } + ret = append(ret, signedTxGroup) + } + return ret + } + + b.Run("handleIncomingTxns", func(b *testing.B) { + signedTransactionGroups := makeSignedTxnGroups(b.N) + encodedSignedTransactionGroups := make([]IncomingMessage, b.N) + for _, stxngrp := range signedTransactionGroups { + encodedSignedTransactionGroups = + append(encodedSignedTransactionGroups, IncomingMessage{Data: protocol.Encode(&stxngrp)}) + } + b.ResetTimer() + for i := range encodedSignedTransactionGroups { + txHandler.processIncomingTxn(encodedSignedTransactionGroups[i]) + } + }) + b.Run("verifyIncomingTxns", func(b *testing.B) { + signedTransactionGroups := makeSignedTxnGroups(b.N) + b.ResetTimer() + for wi := range outChan { + + } + }) +} + // vtCache is a noop VerifiedTransactionCache type vtCache struct{} From ecbab6d78842ac00d2f648cddfc0e9f7b877dbd8 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 13 Oct 2022 12:45:34 -0400 Subject: [PATCH 042/156] progress on perf --- data/txHandler_test.go | 328 +++++++++++++++++++++------------------- ledger/internal/eval.go | 1 - 2 files changed, 174 insertions(+), 155 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index a5a434d704..38c2c8979c 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -130,160 +130,6 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { }) } -func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { - const numUsers = 100 - log := logging.TestingLog(b) - log.SetLevel(logging.Warn) - secrets := make([]*crypto.SignatureSecrets, numUsers) - addresses := make([]basics.Address, numUsers) - - genesis := make(map[basics.Address]basics.AccountData) - for i := 0; i < numUsers; i++ { - secret := keypair() - addr := basics.Address(secret.SignatureVerifier) - secrets[i] = secret - addresses[i] = addr - genesis[addr] = basics.AccountData{ - Status: basics.Online, - MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, - } - } - - genesis[poolAddr] = basics.AccountData{ - Status: basics.NotParticipating, - MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, - } - - require.Equal(b, len(genesis), numUsers+1) - genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) - ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) - const inMem = true - cfg := config.GetDefaultLocal() - cfg.Archival = true - ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) - require.NoError(b, err) - - l := ledger - - cfg.TxPoolSize = 75000 - cfg.EnableProcessBlockStats = false - tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) - - var outChan chan *txBacklogMsg - - // Make a test backlog worker, which is simiar to backlogWorker, but sends the results - // through the outChan instead of passing it to postprocessCheckedTxn - go func() { - defer handler.backlogWg.Done() - for { - // prioritize the postVerificationQueue - select { - case wi, ok := <-handler.postVerificationQueue: - if !ok { - return - } - outChan <- wi - // restart the loop so that we could empty out the post verification queue. - continue - default: - } - - // we have no more post verification items. wait for either backlog queue item or post verification item. - select { - case wi, ok := <-handler.backlogQueue: - if !ok { - return - } - if handler.checkAlreadyCommitted(wi) { - continue - } - - handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} - - case wi, ok := <-handler.postVerificationQueue: - if !ok { - return - } - outChan <- wi - - case <-handler.ctx.Done(): - return - } - } - }() - - goodTxnGroups := make(map[Uint64], interface{}) - badTxnGroups := make(map[Uint64], interface{}) - makeSignedTxnGroups := func(N int) [][]transactions.SignedTxn { - maxGrpSize := proto.MaxTxGroupSize - ret := make([][]transactions.SignedTxn, 0, N) - for u := 0; u < N; u++ { - grpSize := rand.Intn(maxGrpSize) - var txGroup transactions.TxGroup - txns := make([]transactions.Transaction, grpSize) - for g := 0; g < grpSize; g++ { - // generate transactions - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, u) - tx := transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: addresses[(u+g)%numUsers], - Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, - FirstValid: 0, - LastValid: basics.Round(proto.MaxTxnLife), - Note: noteField, - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: addresses[(u+g+1)%numUsers], - Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, - }, - } - txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.HashObj(tx)) - txns = append(txns, tx) - } - groupHash := crypto.HashObj(txGroup) - signedTxGroup = make([]transactions.SignedTxn, 0, grpSize) - for g, txn := range txns { - txn.Group = groupHash - signedTx := txn.Sign(secrets[(u+g)%numUsers]) - signedTxGroup = append(signedTxGroup, signedTx) - } - // randomly make bad signatures - if rand.Float32() < 0.3 { - signedTxGroup[rand.Intn(grpSize)].Sig[0] = signedTxGroup[rand.Intn(grpSize)].Sig[0] + 1 - badTxnGroups[u] = struct{}{} - } else { - goodTxnGroups[u] = struct{}{} - } - ret = append(ret, signedTxGroup) - } - return ret - } - - b.Run("handleIncomingTxns", func(b *testing.B) { - signedTransactionGroups := makeSignedTxnGroups(b.N) - encodedSignedTransactionGroups := make([]IncomingMessage, b.N) - for _, stxngrp := range signedTransactionGroups { - encodedSignedTransactionGroups = - append(encodedSignedTransactionGroups, IncomingMessage{Data: protocol.Encode(&stxngrp)}) - } - b.ResetTimer() - for i := range encodedSignedTransactionGroups { - txHandler.processIncomingTxn(encodedSignedTransactionGroups[i]) - } - }) - b.Run("verifyIncomingTxns", func(b *testing.B) { - signedTransactionGroups := makeSignedTxnGroups(b.N) - b.ResetTimer() - for wi := range outChan { - - } - }) -} - // vtCache is a noop VerifiedTransactionCache type vtCache struct{} @@ -403,3 +249,177 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { require.Equal(b, benchTxnNum, idx) } } + + +func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { + const numUsers = 100 + log := logging.TestingLog(b) + log.SetLevel(logging.Warn) + secrets := make([]*crypto.SignatureSecrets, numUsers) + addresses := make([]basics.Address, numUsers) + + genesis := make(map[basics.Address]basics.AccountData) + for i := 0; i < numUsers; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + genesis[addr] = basics.AccountData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, + } + } + + genesis[poolAddr] = basics.AccountData{ + Status: basics.NotParticipating, + MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, + } + + require.Equal(b, len(genesis), numUsers+1) + genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) + require.NoError(b, err) + + l := ledger + + cfg.TxPoolSize = 75000 + cfg.EnableProcessBlockStats = false + tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) + txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + + outChan := make(chan *txBacklogMsg, 10) + + // Make a test backlog worker, which is simiar to backlogWorker, but sends the results + // through the outChan instead of passing it to postprocessCheckedTxn + go func() { + for { + // prioritize the postVerificationQueue + select { + case wi, ok := <-txHandler.postVerificationQueue: + if !ok { + return + } + outChan <- wi + // restart the loop so that we could empty out the post verification queue. + continue + default: + } + + // we have no more post verification items. wait for either backlog queue item or post verification item. + select { + case wi, ok := <-txHandler.backlogQueue: + if !ok { + return + } + if txHandler.checkAlreadyCommitted(wi) { + continue + } + + txHandler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + + case wi, ok := <-txHandler.postVerificationQueue: + if !ok { + return + } + outChan <- wi + + case <-txHandler.ctx.Done(): + return + } + } + }() + + goodTxnGroups := make(map[uint64]interface{}) + badTxnGroups := make(map[uint64]interface{}) + makeSignedTxnGroups := func(N int) [][]transactions.SignedTxn { + maxGrpSize := proto.MaxTxGroupSize + ret := make([][]transactions.SignedTxn, 0, N) + for u := 0; u < N; u++ { + grpSize := rand.Intn(maxGrpSize-1) + 1 + var txGroup transactions.TxGroup + txns := make([]transactions.Transaction, 0, grpSize) + for g := 0; g < grpSize; g++ { + // generate transactions + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(u)) + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: addresses[(u+g)%numUsers], + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, + FirstValid: 0, + LastValid: basics.Round(proto.MaxTxnLife), + GenesisHash: genesisHash, + Note: noteField, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: addresses[(u+g+1)%numUsers], + Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, + }, + } + txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) + txns = append(txns, tx) + } + groupHash := crypto.HashObj(txGroup) + signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) + for g, txn := range txns { + txn.Group = groupHash + signedTx := txn.Sign(secrets[(u+g)%numUsers]) + signedTx.Txn = txn + signedTxGroup = append(signedTxGroup, signedTx) + } + // randomly make bad signatures + if rand.Float32() < 0.3 { + tinGrp := rand.Intn(grpSize) + signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 + badTxnGroups[uint64(u)] = struct{}{} + } else { + goodTxnGroups[uint64(u)] = struct{}{} + } + ret = append(ret, signedTxGroup) + } + return ret + } + + signedTransactionGroups := makeSignedTxnGroups(b.N) + encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) + for _, stxngrp := range signedTransactionGroups { + data := make([]byte, 0) + for _, stxn := range stxngrp { + data = append(data, protocol.Encode(&stxn)...) + } + encodedSignedTransactionGroups = + append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) + } + + for _, tg := range encodedSignedTransactionGroups { + txHandler.processIncomingTxn(tg) + } + + go func() { + defer close(txHandler.postVerificationQueue) + counter := 0 + b.ResetTimer() + for wi := range outChan { + counter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + if wi.verificationErr == nil { + if _, found := goodTxnGroups[u]; !found { + fmt.Printf("wrong!") + } + } else { + if _, found := badTxnGroups[u]; !found { + fmt.Printf("wrong!") + } + } + if counter == b.N { + break + } + } + }() +} diff --git a/ledger/internal/eval.go b/ledger/internal/eval.go index f2750d8a08..3f02eb5e10 100644 --- a/ledger/internal/eval.go +++ b/ledger/internal/eval.go @@ -850,7 +850,6 @@ func (eval *BlockEvaluator) TestTransactionGroup(txgroup []transactions.SignedTx if !txn.Txn.Group.IsZero() { txWithoutGroup := txn.Txn txWithoutGroup.Group = crypto.Digest{} - group.TxGroupHashes = append(group.TxGroupHashes, crypto.Digest(txWithoutGroup.ID())) } else if len(txgroup) > 1 { return fmt.Errorf("transactionGroup: [%d] had zero Group but was submitted in a group of %d", gi, len(txgroup)) From 1339a1d46c4d616115653cb18484679289ed372b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 13 Oct 2022 12:46:05 -0400 Subject: [PATCH 043/156] gofmt --- data/txHandler_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 38c2c8979c..7e30e44e4d 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -250,7 +250,6 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } } - func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { const numUsers = 100 log := logging.TestingLog(b) From 82ba3d51452b14d58105b0881a3a4f56ab0f9929 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 14 Oct 2022 00:42:25 -0400 Subject: [PATCH 044/156] more pregress on the benchmark --- data/transactions/verify/txn.go | 15 +-- data/txHandler.go | 45 +++------ data/txHandler_test.go | 160 +++++++++++++++++++------------- 3 files changed, 115 insertions(+), 105 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index b44ef93206..691f4d40eb 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -540,17 +540,17 @@ const waitForFirstTxnDuration = 2000 * time.Millisecond const numberOfExecPoolSeats = 64 // internalBufferSize is the size of the chan that will hold the arriving stxns before they get pre-processed -const internalBufferSize = 25000 +const internalBufferSize = 0 //const txnPerWorksetThreshold = 32 // MakeStream creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, +func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, + ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( - stxnInput chan<- VerificationElement, resultOtput <-chan VerificationResult) { + resultOtput <-chan VerificationResult) { - stxnChan := make(chan VerificationElement, internalBufferSize) resultChan := make(chan VerificationResult) sm := streamManager{ @@ -571,7 +571,7 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBl var added bool for { select { - case stx := <-stxnChan: + case stx, ok := <-stxnChan: isFirstInBatch := bl.batchVerifier.GetNumberOfEnqueuedSignatures() == 0 // TODO: separate operations here, and get the sig verification inside LogicSig outside groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) @@ -605,7 +605,8 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBl bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) bl.elementContext = append(bl.elementContext, stx.Context) bl.messagesForTxn = append(bl.messagesForTxn, numEnqueued) - if len(bl.groupCtxs) >= txnPerWorksetThreshold { + // if not expecting any more transactions (!ok) or if the batch is full + if !ok || len(bl.groupCtxs) >= txnPerWorksetThreshold { // TODO: the limit of 32 should not pass err := sm.processFullBatch(bl) if err != nil { @@ -641,7 +642,7 @@ func MakeStream(ctx context.Context, ledger logic.LedgerForSignature, nbw *NewBl } } }() - return stxnChan, resultChan + return resultChan } func (sm *streamManager) processFullBatch(bl batchLoad) (err error) { diff --git a/data/txHandler.go b/data/txHandler.go index ac73bcbaed..d8271ac55e 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -67,7 +67,7 @@ type TxHandler struct { net network.GossipNode ctx context.Context ctxCancel context.CancelFunc - streamVerifierChan chan<- verify.VerificationElement + streamVerifierChan chan verify.VerificationElement } // MakeTxHandler makes a new handler for transaction messages @@ -107,23 +107,26 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } nbw := verify.MakeNewBlockWatcher(latestHdr) handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - handler.streamVerifierChan, outChan = verify.MakeStream(handler.ctx, + outChan = verify.MakeStream(handler.ctx, handler.streamVerifierChan, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) - go processTxnStreamVerResults(handler.ctx, outChan, handler.postVerificationQueue) + go handler.processTxnStreamVerResults(outChan) return handler } -func processTxnStreamVerResults(ctx context.Context, outChan <-chan verify.VerificationResult, - postVerificationQueue chan *txBacklogMsg) { +func (handler *TxHandler) processTxnStreamVerResults(outChan <-chan verify.VerificationResult) { for { select { - case <-ctx.Done(): + case <-handler.ctx.Done(): return - case result := <-outChan: + case result, ok := <-outChan: + if !ok { + close(handler.postVerificationQueue) + return + } txBLMsg := result.Context.(*txBacklogMsg) txBLMsg.verificationErr = result.Err select { - case postVerificationQueue <- txBLMsg: + case handler.postVerificationQueue <- txBLMsg: default: // we failed to write to the output queue, since the queue was full. // adding the metric here allows us to monitor how frequently it happens. @@ -178,6 +181,7 @@ func (handler *TxHandler) backlogWorker() { select { case wi, ok := <-handler.backlogQueue: if !ok { + close(handler.streamVerifierChan) return } if handler.checkAlreadyCommitted(wi) { @@ -229,31 +233,6 @@ func (handler *TxHandler) postprocessCheckedTxn(wi *txBacklogMsg) { handler.net.Relay(handler.ctx, protocol.TxnTag, reencode(verifiedTxGroup), false, wi.rawmsg.Sender) } -// asyncVerifySignature verifies that the given transaction group is valid, and update the txBacklogMsg data structure accordingly. -func (handler *TxHandler) asyncVerifySignature(arg interface{}) interface{} { - tx := arg.(*txBacklogMsg) - - // build the transaction verification context - latest := handler.ledger.Latest() - latestHdr, err := handler.ledger.BlockHdr(latest) - if err != nil { - tx.verificationErr = fmt.Errorf("Could not get header for previous block %d: %w", latest, err) - logging.Base().Warnf("Could not get header for previous block %d: %v", latest, err) - } else { - // we can't use PaysetGroups here since it's using a execpool like this go-routine and we don't want to deadlock. - _, tx.verificationErr = verify.TxnGroup(tx.unverifiedTxGroup, latestHdr, handler.ledger.VerifiedTransactionCache(), handler.ledger) - } - - select { - case handler.postVerificationQueue <- tx: - default: - // we failed to write to the output queue, since the queue was full. - // adding the metric here allows us to monitor how frequently it happens. - transactionMessagesDroppedFromPool.Inc(nil) - } - return nil -} - func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) network.OutgoingMessage { dec := protocol.NewMsgpDecoderBytes(rawmsg.Data) ntx := 0 diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 7e30e44e4d..1d88b7440e 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "math/rand" + "sync" "testing" "time" @@ -254,9 +255,10 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { const numUsers = 100 log := logging.TestingLog(b) log.SetLevel(logging.Warn) - secrets := make([]*crypto.SignatureSecrets, numUsers) addresses := make([]basics.Address, numUsers) - + secrets := make([]*crypto.SignatureSecrets, numUsers) + + // prepare the accounts genesis := make(map[basics.Address]basics.AccountData) for i := 0; i < numUsers; i++ { secret := keypair() @@ -269,6 +271,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { } } + genesis[poolAddr] = basics.AccountData{ Status: basics.NotParticipating, MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, @@ -290,12 +293,18 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + defer txHandler.ctxCancel() outChan := make(chan *txBacklogMsg, 10) + wg := sync.WaitGroup{} + + wg.Add(1) // Make a test backlog worker, which is simiar to backlogWorker, but sends the results // through the outChan instead of passing it to postprocessCheckedTxn go func() { + defer wg.Done() + defer close(outChan) for { // prioritize the postVerificationQueue select { @@ -313,12 +322,16 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { select { case wi, ok := <-txHandler.backlogQueue: if !ok { - return + close(txHandler.streamVerifierChan) + // unlike the production backlog, do not return here, so that all pending txns getting verified + // will get the chance to get pushed out + // return + continue } if txHandler.checkAlreadyCommitted(wi) { + // this is not expected during the test continue } - txHandler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} case wi, ok := <-txHandler.postVerificationQueue: @@ -333,59 +346,8 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { } }() - goodTxnGroups := make(map[uint64]interface{}) - badTxnGroups := make(map[uint64]interface{}) - makeSignedTxnGroups := func(N int) [][]transactions.SignedTxn { - maxGrpSize := proto.MaxTxGroupSize - ret := make([][]transactions.SignedTxn, 0, N) - for u := 0; u < N; u++ { - grpSize := rand.Intn(maxGrpSize-1) + 1 - var txGroup transactions.TxGroup - txns := make([]transactions.Transaction, 0, grpSize) - for g := 0; g < grpSize; g++ { - // generate transactions - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(u)) - tx := transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: addresses[(u+g)%numUsers], - Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, - FirstValid: 0, - LastValid: basics.Round(proto.MaxTxnLife), - GenesisHash: genesisHash, - Note: noteField, - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: addresses[(u+g+1)%numUsers], - Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, - }, - } - txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) - txns = append(txns, tx) - } - groupHash := crypto.HashObj(txGroup) - signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) - for g, txn := range txns { - txn.Group = groupHash - signedTx := txn.Sign(secrets[(u+g)%numUsers]) - signedTx.Txn = txn - signedTxGroup = append(signedTxGroup, signedTx) - } - // randomly make bad signatures - if rand.Float32() < 0.3 { - tinGrp := rand.Intn(grpSize) - signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 - badTxnGroups[uint64(u)] = struct{}{} - } else { - goodTxnGroups[uint64(u)] = struct{}{} - } - ret = append(ret, signedTxGroup) - } - return ret - } - - signedTransactionGroups := makeSignedTxnGroups(b.N) + // Prepare the transactions + signedTransactionGroups, goodTxnGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) for _, stxngrp := range signedTransactionGroups { data := make([]byte, 0) @@ -396,16 +358,19 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) } - for _, tg := range encodedSignedTransactionGroups { - txHandler.processIncomingTxn(tg) - } - + // Process the results and make sure they are correct + wg.Add(1) go func() { - defer close(txHandler.postVerificationQueue) + defer wg.Done() counter := 0 + defer func() { + fmt.Printf("processed %d txns\n", counter) + }() + defer close(txHandler.postVerificationQueue) b.ResetTimer() for wi := range outChan { counter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) if wi.verificationErr == nil { if _, found := goodTxnGroups[u]; !found { @@ -416,9 +381,74 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { fmt.Printf("wrong!") } } - if counter == b.N { - break - } } }() + + // Send the transactions + for _, tg := range encodedSignedTransactionGroups { + // time the streaming of the txns to at most 6000 tps + txHandler.processIncomingTxn(tg) + time.Sleep(160*time.Microsecond) + time.Sleep(160*time.Microsecond) + } + // time.Sleep(3*time.Second) + close(txHandler.backlogQueue) + wg.Wait() +} + +func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, + secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, + goodTxnGroups, badTxnGroups map[uint64]interface{}) { + + goodTxnGroups = make(map[uint64]interface{}) + badTxnGroups = make(map[uint64]interface{}) + + maxGrpSize := proto.MaxTxGroupSize + ret = make([][]transactions.SignedTxn, 0, N) + for u := 0; u < N; u++ { + grpSize := rand.Intn(maxGrpSize-1) + 1 + grpSize = 1 + var txGroup transactions.TxGroup + txns := make([]transactions.Transaction, 0, grpSize) + for g := 0; g < grpSize; g++ { + // generate transactions + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(u)) + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: addresses[(u+g)%numUsers], + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, + FirstValid: 0, + LastValid: basics.Round(proto.MaxTxnLife), + GenesisHash: genesisHash, + Note: noteField, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: addresses[(u+g+1)%numUsers], + Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, + }, + } + txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) + txns = append(txns, tx) + } + groupHash := crypto.HashObj(txGroup) + signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) + for g, txn := range txns { + txn.Group = groupHash + signedTx := txn.Sign(secrets[(u+g)%numUsers]) + signedTx.Txn = txn + signedTxGroup = append(signedTxGroup, signedTx) + } + // randomly make bad signatures + if rand.Float32() < 0.3 { + tinGrp := rand.Intn(grpSize) + signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 + badTxnGroups[uint64(u)] = struct{}{} + } else { + goodTxnGroups[uint64(u)] = struct{}{} + } + ret = append(ret, signedTxGroup) + } + return } From 5ab00b9ac60002304072e243961f6a1522750a16 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 14 Oct 2022 13:40:29 -0400 Subject: [PATCH 045/156] benchmark fixed --- data/transactions/verify/txn.go | 32 ++++++++++++++++++++--- data/txHandler_test.go | 46 ++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 691f4d40eb..46f897975a 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "sync" "time" "github.com/algorand/go-deadlock" @@ -479,6 +480,7 @@ type streamManager struct { verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache + execPoolWg sync.WaitGroup } type batchLoad struct { @@ -572,6 +574,22 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, for { select { case stx, ok := <-stxnChan: + // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer + if !ok { + err := sm.processFullBatch(bl) + if err != nil { + vr := VerificationResult{ + TxnGroup: stx.TxnGroup, + Context: stx.Context, + Err: err,// TODO: maybe this error in internal, and should not go out + } + sm.sendOut(vr) + } + // for for all pending tasks out, then close the result chan + sm.execPoolWg.Wait() + close(resultChan) + return + } isFirstInBatch := bl.batchVerifier.GetNumberOfEnqueuedSignatures() == 0 // TODO: separate operations here, and get the sig verification inside LogicSig outside groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) @@ -605,12 +623,17 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) bl.elementContext = append(bl.elementContext, stx.Context) bl.messagesForTxn = append(bl.messagesForTxn, numEnqueued) - // if not expecting any more transactions (!ok) or if the batch is full - if !ok || len(bl.groupCtxs) >= txnPerWorksetThreshold { + if len(bl.groupCtxs) >= txnPerWorksetThreshold { // TODO: the limit of 32 should not pass err := sm.processFullBatch(bl) if err != nil { - fmt.Println(err) + vr := VerificationResult{ + TxnGroup: stx.TxnGroup, + Context: stx.Context, + Err: err,// TODO: maybe this error in internal, and should not go out + } + sm.sendOut(vr) + continue } bl = makeBatchLoad() // starting a new batch. Can wait long, since nothing is blocked @@ -691,8 +714,9 @@ func (sm *streamManager) sendOut(vr VerificationResult) { } func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { - + sm.execPoolWg.Add(1) function := func(arg interface{}) interface{} { + defer sm.execPoolWg.Done() bl = arg.(batchLoad) // var grpErr error // check if we've canceled the request while this was in the queue. diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 1d88b7440e..dc2e92eaef 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "math/rand" + "strings" "sync" "testing" "time" @@ -40,6 +41,7 @@ import ( "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/execpool" + "github.com/algorand/go-algorand/util/metrics" ) func BenchmarkTxHandlerProcessing(b *testing.B) { @@ -292,8 +294,8 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { cfg.EnableProcessBlockStats = false tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) - defer txHandler.ctxCancel() + handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + defer handler.ctxCancel() outChan := make(chan *txBacklogMsg, 10) @@ -308,7 +310,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { for { // prioritize the postVerificationQueue select { - case wi, ok := <-txHandler.postVerificationQueue: + case wi, ok := <-handler.postVerificationQueue: if !ok { return } @@ -320,27 +322,29 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { // we have no more post verification items. wait for either backlog queue item or post verification item. select { - case wi, ok := <-txHandler.backlogQueue: + case wi, ok := <-handler.backlogQueue: if !ok { - close(txHandler.streamVerifierChan) - // unlike the production backlog, do not return here, so that all pending txns getting verified - // will get the chance to get pushed out - // return - continue + close(handler.streamVerifierChan) + // wait until all the pending responses are obtained. + // this is not in backlogWorker, maybe should be + for wi := range handler.postVerificationQueue { + outChan <- wi + } + return } - if txHandler.checkAlreadyCommitted(wi) { + if handler.checkAlreadyCommitted(wi) { // this is not expected during the test continue } - txHandler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} - case wi, ok := <-txHandler.postVerificationQueue: + case wi, ok := <-handler.postVerificationQueue: if !ok { return } outChan <- wi - case <-txHandler.ctx.Done(): + case <-handler.ctx.Done(): return } } @@ -366,7 +370,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { defer func() { fmt.Printf("processed %d txns\n", counter) }() - defer close(txHandler.postVerificationQueue) b.ResetTimer() for wi := range outChan { counter++ @@ -387,13 +390,20 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { // Send the transactions for _, tg := range encodedSignedTransactionGroups { // time the streaming of the txns to at most 6000 tps - txHandler.processIncomingTxn(tg) - time.Sleep(160*time.Microsecond) + handler.processIncomingTxn(tg) time.Sleep(160*time.Microsecond) } - // time.Sleep(3*time.Second) - close(txHandler.backlogQueue) + close(handler.backlogQueue) wg.Wait() + + var buf strings.Builder + metrics.DefaultRegistry().WriteMetrics(&buf, "") + str := buf.String() + x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") + str = str[x+44:x+44+strings.Index(str[x+44:], "\n")] + str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) + fmt.Printf("Dropped from backlog: %s\n", str) + } func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, From 103113c3a1c7a60f153e667135cbea2894bc494a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 14 Oct 2022 16:23:09 -0400 Subject: [PATCH 046/156] benchmark and other improvements --- data/transactions/verify/txn.go | 8 ++++---- data/txHandler.go | 4 +++- data/txHandler_test.go | 28 +++++++++++----------------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 46f897975a..a9a47d23ac 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -539,7 +539,6 @@ func makeBatchLoad() batchLoad { // [wait time] <= [validation time of a single txn] / 2 const waitForNextTxnDuration = 50 * time.Millisecond const waitForFirstTxnDuration = 2000 * time.Millisecond -const numberOfExecPoolSeats = 64 // internalBufferSize is the size of the chan that will hold the arriving stxns before they get pre-processed const internalBufferSize = 0 @@ -554,6 +553,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, resultOtput <-chan VerificationResult) { resultChan := make(chan VerificationResult) + numberOfExecPoolSeats := verificationPool.GetParallelism() sm := streamManager{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), @@ -581,7 +581,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, vr := VerificationResult{ TxnGroup: stx.TxnGroup, Context: stx.Context, - Err: err,// TODO: maybe this error in internal, and should not go out + Err: err, // TODO: maybe this error in internal, and should not go out } sm.sendOut(vr) } @@ -623,14 +623,14 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) bl.elementContext = append(bl.elementContext, stx.Context) bl.messagesForTxn = append(bl.messagesForTxn, numEnqueued) - if len(bl.groupCtxs) >= txnPerWorksetThreshold { + if len(bl.groupCtxs) >= txnPerWorksetThreshold/4 { // TODO: the limit of 32 should not pass err := sm.processFullBatch(bl) if err != nil { vr := VerificationResult{ TxnGroup: stx.TxnGroup, Context: stx.Context, - Err: err,// TODO: maybe this error in internal, and should not go out + Err: err, // TODO: maybe this error in internal, and should not go out } sm.sendOut(vr) continue diff --git a/data/txHandler.go b/data/txHandler.go index d8271ac55e..d80f65937e 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -46,6 +46,8 @@ var txBacklogSize = config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnByt var transactionMessagesHandled = metrics.MakeCounter(metrics.TransactionMessagesHandled) var transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) var transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) +// verifierStreamBufferSize is the number of txn that coult be accumulated before the verifier stream consumes them +var verifierStreamBufferSize = 1000 // The txBacklogMsg structure used to track a single incoming transaction from the gossip network, type txBacklogMsg struct { @@ -92,7 +94,7 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go backlogQueue: make(chan *txBacklogMsg, txBacklogSize), postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, - streamVerifierChan: make(chan verify.VerificationElement), + streamVerifierChan: make(chan verify.VerificationElement, verifierStreamBufferSize), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index dc2e92eaef..31ab50f3e0 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -259,7 +259,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { log.SetLevel(logging.Warn) addresses := make([]basics.Address, numUsers) secrets := make([]*crypto.SignatureSecrets, numUsers) - + // prepare the accounts genesis := make(map[basics.Address]basics.AccountData) for i := 0; i < numUsers; i++ { @@ -273,7 +273,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { } } - genesis[poolAddr] = basics.AccountData{ Status: basics.NotParticipating, MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, @@ -351,7 +350,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { }() // Prepare the transactions - signedTransactionGroups, goodTxnGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) for _, stxngrp := range signedTransactionGroups { data := make([]byte, 0) @@ -375,14 +374,11 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { counter++ u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] if wi.verificationErr == nil { - if _, found := goodTxnGroups[u]; !found { - fmt.Printf("wrong!") - } + require.False(b, inBad, "No error for bad signature") } else { - if _, found := badTxnGroups[u]; !found { - fmt.Printf("wrong!") - } + require.True(b, inBad, "Error for good signature") } } }() @@ -391,16 +387,17 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { for _, tg := range encodedSignedTransactionGroups { // time the streaming of the txns to at most 6000 tps handler.processIncomingTxn(tg) - time.Sleep(160*time.Microsecond) + randduration := time.Duration(uint64(((1 + rand.Float32()) * 3))) + time.Sleep(randduration * time.Microsecond) } close(handler.backlogQueue) wg.Wait() var buf strings.Builder metrics.DefaultRegistry().WriteMetrics(&buf, "") - str := buf.String() + str := buf.String() x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") - str = str[x+44:x+44+strings.Index(str[x+44:], "\n")] + str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) fmt.Printf("Dropped from backlog: %s\n", str) @@ -408,9 +405,8 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, - goodTxnGroups, badTxnGroups map[uint64]interface{}) { + badTxnGroups map[uint64]interface{}) { - goodTxnGroups = make(map[uint64]interface{}) badTxnGroups = make(map[uint64]interface{}) maxGrpSize := proto.MaxTxGroupSize @@ -451,12 +447,10 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 0.3 { + if rand.Float32() < 0.1 { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} - } else { - goodTxnGroups[uint64(u)] = struct{}{} } ret = append(ret, signedTxGroup) } From 702d2466b4fbddf0559971e6d3cd779357d86521 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 14 Oct 2022 19:51:37 -0400 Subject: [PATCH 047/156] comments --- data/txHandler_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 31ab50f3e0..3ca41712a9 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -253,6 +253,8 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } } +// BenchmarkIncomingTxHandlerProcessing sends singed transactions to be handled and verified +// It reports the number of dropped transactions func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { const numUsers = 100 log := logging.TestingLog(b) @@ -288,9 +290,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { require.NoError(b, err) l := ledger - - cfg.TxPoolSize = 75000 - cfg.EnableProcessBlockStats = false tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) @@ -383,7 +382,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { } }() - // Send the transactions + // Send the transactions to the verifier for _, tg := range encodedSignedTransactionGroups { // time the streaming of the txns to at most 6000 tps handler.processIncomingTxn(tg) @@ -393,6 +392,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { close(handler.backlogQueue) wg.Wait() + // Report the number of transactions dropped because the backlog was busy var buf strings.Builder metrics.DefaultRegistry().WriteMetrics(&buf, "") str := buf.String() @@ -400,9 +400,9 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) fmt.Printf("Dropped from backlog: %s\n", str) - } +// Prepare N transaction groups of random sizes with randomly invalid signatures func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { @@ -413,7 +413,6 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, ret = make([][]transactions.SignedTxn, 0, N) for u := 0; u < N; u++ { grpSize := rand.Intn(maxGrpSize-1) + 1 - grpSize = 1 var txGroup transactions.TxGroup txns := make([]transactions.Transaction, 0, grpSize) for g := 0; g < grpSize; g++ { From 66dd43ee624f2b4d8c5804928e4ebc854fab2b38 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 15 Oct 2022 11:24:57 -0400 Subject: [PATCH 048/156] cleanup before pushing check to execpool --- data/transactions/verify/txn.go | 83 +++++++++++----------------- data/transactions/verify/txn_test.go | 4 +- data/txHandler_test.go | 7 +-- 3 files changed, 35 insertions(+), 59 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a9a47d23ac..210327b87d 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -547,9 +547,8 @@ const internalBufferSize = 0 // MakeStream creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, - ledger logic.LedgerForSignature, nbw *NewBlockWatcher, - verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( +func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, ledger logic.LedgerForSignature, + nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( resultOtput <-chan VerificationResult) { resultChan := make(chan VerificationResult) @@ -578,31 +577,22 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, if !ok { err := sm.processFullBatch(bl) if err != nil { - vr := VerificationResult{ - TxnGroup: stx.TxnGroup, - Context: stx.Context, - Err: err, // TODO: maybe this error in internal, and should not go out - } - sm.sendOut(vr) + sm.sendResult(stx.TxnGroup, stx.Context, err)// TODO: maybe this error in internal, and should not go out } // for for all pending tasks out, then close the result chan sm.execPoolWg.Wait() close(resultChan) return } + isFirstInBatch := bl.batchVerifier.GetNumberOfEnqueuedSignatures() == 0 + // TODO: separate operations here, and get the sig verification inside LogicSig outside groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) //TODO: report the error ctx.Err() - if err != nil { // verification failed. can return here - vr := VerificationResult{ - TxnGroup: stx.TxnGroup, - Context: stx.Context, - Err: err, - } - sm.sendOut(vr) + sm.sendResult(stx.TxnGroup, stx.Context, err) continue } @@ -610,12 +600,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, // if nothing is added to the batch, then the txngroup can be returned here if numEnqueued == 0 || (len(bl.messagesForTxn) > 0 && numEnqueued == bl.messagesForTxn[len(bl.messagesForTxn)-1]) { - vr := VerificationResult{ - TxnGroup: stx.TxnGroup, - Context: stx.Context, - Err: err, - } - sm.sendOut(vr) + sm.sendResult(stx.TxnGroup, stx.Context, err) continue } @@ -627,12 +612,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, // TODO: the limit of 32 should not pass err := sm.processFullBatch(bl) if err != nil { - vr := VerificationResult{ - TxnGroup: stx.TxnGroup, - Context: stx.Context, - Err: err, // TODO: maybe this error in internal, and should not go out - } - sm.sendOut(vr) + sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out continue } bl = makeBatchLoad() @@ -668,6 +648,28 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, return resultChan } +func (sm *streamManager) sendResult(veTxnGroup []transactions.SignedTxn, veContext interface{}, err error) { + vr := VerificationResult{ + TxnGroup: veTxnGroup, + Context: veContext, + Err: err, + } + // send the txn result out the pipe + sm.resultChan <- vr + /* + select { + case sm.resultChan <- vr: + + // if the channel is not accepting, should not block here + // report dropped txn. caching is fine, if it comes back in the block + default: + fmt.Println("skipped!!") + //TODO: report this + + } + */ +} + func (sm *streamManager) processFullBatch(bl batchLoad) (err error) { // This is a full batch, no additions are allowed // block and wait for a free seat @@ -695,24 +697,6 @@ func (sm *streamManager) processBatch(bl batchLoad) (added bool) { return } -// send the result out the chan -func (sm *streamManager) sendOut(vr VerificationResult) { - // send the txn result out the pipe - sm.resultChan <- vr - /* - select { - case sm.resultChan <- vr: - - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: - fmt.Println("skipped!!") - //TODO: report this - - } - */ -} - func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { sm.execPoolWg.Add(1) function := func(arg interface{}) interface{} { @@ -754,12 +738,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { } else { result = ErrInvalidSignature } - vr := VerificationResult{ - TxnGroup: bl.txnGroups[txgIdx], - Context: bl.elementContext[txgIdx], - Err: result, - } - sm.sendOut(vr) + sm.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) } // loading them all at once to lock the cache once err = sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 879607a21f..26499c01d8 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -915,8 +915,8 @@ func TestStreamVerifier(t *testing.T) { defer cancel() nbw := MakeNewBlockWatcher(blkHdr) - - stxnChan, resultChan := MakeStream(ctx, nil, nbw, verificationPool, cache) + stxnChan := make(chan VerificationElement) + resultChan := MakeStream(ctx, stxnChan, nil, nbw, verificationPool, cache) badTxnGroups := make(map[crypto.Signature]struct{}) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 3ca41712a9..2aac403fce 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -274,7 +274,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, } } - genesis[poolAddr] = basics.AccountData{ Status: basics.NotParticipating, MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, @@ -296,9 +295,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { defer handler.ctxCancel() outChan := make(chan *txBacklogMsg, 10) - wg := sync.WaitGroup{} - wg.Add(1) // Make a test backlog worker, which is simiar to backlogWorker, but sends the results // through the outChan instead of passing it to postprocessCheckedTxn @@ -369,9 +366,9 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { fmt.Printf("processed %d txns\n", counter) }() b.ResetTimer() + tt := time.Now() for wi := range outChan { counter++ - u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { @@ -380,6 +377,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { require.True(b, inBad, "Error for good signature") } } + fmt.Printf("TPS: %d\n", uint64(counter)*1000000000/uint64(time.Since(tt))) }() // Send the transactions to the verifier @@ -406,7 +404,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { - badTxnGroups = make(map[uint64]interface{}) maxGrpSize := proto.MaxTxGroupSize From 4f085b7a9cc3a40fb077876e75de1af9b04a435b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 15 Oct 2022 11:33:58 -0400 Subject: [PATCH 049/156] gofmt --- data/transactions/verify/txn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 210327b87d..9a95538bd1 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -577,7 +577,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, ledger if !ok { err := sm.processFullBatch(bl) if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err)// TODO: maybe this error in internal, and should not go out + sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out } // for for all pending tasks out, then close the result chan sm.execPoolWg.Wait() From 67674582b5de482e98e2b73dc90c06e32b2aa11f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sun, 16 Oct 2022 23:07:53 -0400 Subject: [PATCH 050/156] move the checks to inside exec pool task --- data/transactions/verify/txn.go | 201 +++++++++++++++++++-------- data/transactions/verify/txn_test.go | 4 +- data/txHandler.go | 6 +- data/txHandler_test.go | 4 +- 4 files changed, 152 insertions(+), 63 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 9a95538bd1..bfe798d6f4 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -457,10 +457,10 @@ func (w *worksetBuilder) completed() bool { return w.idx >= len(w.payset) } -// VerificationElement is the element passed the Stream verifier +// UnverifiedElement is the element passed the Stream verifier // Context is a reference associated with the txn group which is passed // with the result -type VerificationElement struct { +type UnverifiedElement struct { TxnGroup []transactions.SignedTxn Context interface{} } @@ -483,14 +483,6 @@ type streamManager struct { execPoolWg sync.WaitGroup } -type batchLoad struct { - batchVerifier *crypto.BatchVerifier - txnGroups [][]transactions.SignedTxn - groupCtxs []*GroupContext - elementContext []interface{} - messagesForTxn []int -} - // NewBlockWatcher is a struct used to provide a new block header to the // stream verifier type NewBlockWatcher struct { @@ -522,15 +514,49 @@ func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { return nbw.blkHeader } -func makeBatchLoad() batchLoad { - bl := batchLoad{} - bl.batchVerifier = crypto.MakeBatchVerifier() - bl.groupCtxs = make([]*GroupContext, 0) +type unverifiedElementList struct { + elementList []UnverifiedElement + nbw *NewBlockWatcher + ledger logic.LedgerForSignature +} + +func makeUnverifiedElementList(nbw *NewBlockWatcher, ledger logic.LedgerForSignature) (uel unverifiedElementList) { + uel.nbw = nbw + uel.ledger = ledger + uel.elementList = make([]UnverifiedElement, 0) + return +} + +func makeSingleUnverifiedElement(nbw *NewBlockWatcher, ledger logic.LedgerForSignature, ue UnverifiedElement) (uel unverifiedElementList) { + uel.nbw = nbw + uel.ledger = ledger + uel.elementList = []UnverifiedElement{ue} + return +} + +type batchLoad struct { + txnGroups [][]transactions.SignedTxn + groupCtxs []*GroupContext + elementContext []interface{} + messagesForTxn []int +} + +func makeBatchLoad() (bl batchLoad) { bl.txnGroups = make([][]transactions.SignedTxn, 0) + bl.groupCtxs = make([]*GroupContext, 0) + bl.elementContext = make([]interface{}, 0) bl.messagesForTxn = make([]int, 0) return bl } +func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext, eltctx interface{}, numBatchableSigs int) { + bl.txnGroups = append(bl.txnGroups, txngrp) + bl.groupCtxs = append(bl.groupCtxs, gctx) + bl.elementContext = append(bl.elementContext, eltctx) + bl.messagesForTxn = append(bl.messagesForTxn, numBatchableSigs) + +} + // wait time for another txn should satisfy the following inequality: // [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] // since these are difficult to estimate, the simplified version could be to assume: @@ -547,7 +573,7 @@ const internalBufferSize = 0 // MakeStream creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, ledger logic.LedgerForSignature, +func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( resultOtput <-chan VerificationResult) { @@ -567,17 +593,20 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, ledger sm.seatReturnChan <- struct{}{} } - bl := makeBatchLoad() timer := time.NewTicker(waitForFirstTxnDuration) var added bool + var numberOfSigsInCurrent uint64 + uel := makeUnverifiedElementList(nbw, ledger) for { select { case stx, ok := <-stxnChan: // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer if !ok { - err := sm.processFullBatch(bl) - if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out + if numberOfSigsInCurrent > 0 { + err := sm.processFullBatch(uel) + if err != nil { + sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out + } } // for for all pending tasks out, then close the result chan sm.execPoolWg.Wait() @@ -585,59 +614,54 @@ func MakeStream(ctx context.Context, stxnChan <-chan VerificationElement, ledger return } - isFirstInBatch := bl.batchVerifier.GetNumberOfEnqueuedSignatures() == 0 - - // TODO: separate operations here, and get the sig verification inside LogicSig outside - groupCtx, err := txnGroupBatchPrep(stx.TxnGroup, nbw.getBlockHeader(), ledger, bl.batchVerifier) - //TODO: report the error ctx.Err() + isFirstInBatch := numberOfSigsInCurrent == 0 + numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) if err != nil { - // verification failed. can return here sm.sendResult(stx.TxnGroup, stx.Context, err) continue } - numEnqueued := bl.batchVerifier.GetNumberOfEnqueuedSignatures() - // if nothing is added to the batch, then the txngroup can be returned here - if numEnqueued == 0 || - (len(bl.messagesForTxn) > 0 && numEnqueued == bl.messagesForTxn[len(bl.messagesForTxn)-1]) { - sm.sendResult(stx.TxnGroup, stx.Context, err) + // if no batchable signatures here, send this as a task of its own + if numberOfBatchableSigsInGroup == 0 { + err := sm.processFullBatch(makeSingleUnverifiedElement(nbw, ledger, stx)) + if err != nil { + sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out + } continue } - bl.groupCtxs = append(bl.groupCtxs, groupCtx) - bl.txnGroups = append(bl.txnGroups, stx.TxnGroup) - bl.elementContext = append(bl.elementContext, stx.Context) - bl.messagesForTxn = append(bl.messagesForTxn, numEnqueued) - if len(bl.groupCtxs) >= txnPerWorksetThreshold/4 { - // TODO: the limit of 32 should not pass - err := sm.processFullBatch(bl) + // add this txngrp to the list of batchable txn groups + numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup + uel.elementList = append(uel.elementList, stx) + if numberOfSigsInCurrent >= txnPerWorksetThreshold { + err := sm.processFullBatch(uel) if err != nil { sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out continue } - bl = makeBatchLoad() + uel = makeUnverifiedElementList(nbw, ledger) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // the batch is not full, can wait for some more if isFirstInBatch { - // reset the timer only when a new batch started + // an element is added and is waiting. shorten the waiting time timer.Reset(waitForNextTxnDuration) } } case <-timer.C: // timer ticked. it is time to send the batch even if it is not full - if len(bl.groupCtxs) == 0 { + if numberOfSigsInCurrent == 0 { // nothing batched yet... wait some more timer.Reset(waitForFirstTxnDuration) continue } - added = sm.processBatch(bl) + added = sm.processBatch(uel) if added { - bl = makeBatchLoad() + uel = makeUnverifiedElementList(nbw, ledger) + // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // was not added because no available seats. wait for some more txns + // was not added because no-available-seats. wait for some more txns timer.Reset(waitForNextTxnDuration) } case <-ctx.Done(): @@ -670,20 +694,20 @@ func (sm *streamManager) sendResult(veTxnGroup []transactions.SignedTxn, veConte */ } -func (sm *streamManager) processFullBatch(bl batchLoad) (err error) { +func (sm *streamManager) processFullBatch(uel unverifiedElementList) (err error) { // This is a full batch, no additions are allowed // block and wait for a free seat <-sm.seatReturnChan - err = sm.addVerificationTaskToThePool(bl) + err = sm.addVerificationTaskToThePool(uel) return } -func (sm *streamManager) processBatch(bl batchLoad) (added bool) { +func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { // if cannot find a seat, can go back and collect // more signatures instead of waiting here select { case <-sm.seatReturnChan: - err := sm.addVerificationTaskToThePool(bl) + err := sm.addVerificationTaskToThePool(uel) if err != nil { // TODO: report the error fmt.Println(err) @@ -697,17 +721,35 @@ func (sm *streamManager) processBatch(bl batchLoad) (added bool) { return } -func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { +func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) error { sm.execPoolWg.Add(1) function := func(arg interface{}) interface{} { defer sm.execPoolWg.Done() - bl = arg.(batchLoad) - // var grpErr error - // check if we've canceled the request while this was in the queue. + if sm.ctx.Err() != nil { return sm.ctx.Err() } - failed, err := bl.batchVerifier.VerifyWithFeedback() + + uel := arg.(*unverifiedElementList) + batchVerifier := crypto.MakeBatchVerifier() + + bl := makeBatchLoad() + previousTotal := 0 + // TODO: separate operations here, and get the sig verification inside LogicSig outside + for _, ue := range uel.elementList { + groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) + if err != nil { + // verification failed, cannot go to the batch. return here. + sm.sendResult(ue.TxnGroup, ue.Context, err) + continue + } + totalBatchCount := batchVerifier.GetNumberOfEnqueuedSignatures() + currentCount := totalBatchCount - previousTotal + previousTotal = totalBatchCount + bl.addLoad(ue.TxnGroup, groupCtx, ue.Context, currentCount) + } + + failed, err := batchVerifier.VerifyWithFeedback() if err != nil && err != crypto.ErrBatchHasFailedSigs { fmt.Println(err) // something bad happened @@ -719,7 +761,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { failedSigIdx := 0 for txgIdx := range bl.txnGroups { txGroupSigFailed := false - // if err == nil, means all sigs are verified, no need to check for the failed + // if err == nil, then all sigs are verified, no need to check for the failed for err != nil && failedSigIdx < bl.messagesForTxn[txgIdx] { if failed[failedSigIdx] { // if there is a failed sig check, then no need to check the rest of the @@ -740,7 +782,7 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { } sm.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) } - // loading them all at once to lock the cache once + // loading them all at once by locking the cache once err = sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) if err != nil { // TODO: handle the error @@ -748,6 +790,53 @@ func (sm *streamManager) addVerificationTaskToThePool(bl batchLoad) error { } return struct{}{} } - err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, bl, sm.seatReturnChan) + err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, &uel, sm.seatReturnChan) return err } + +func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs uint64, err error) { + batchSigs = 0 + for _, stx := range stxs { + count, err := getNumberOfBatchableSigsInTxn(stx) + if err != nil { + return 0, err + } + batchSigs = batchSigs + count + } + return +} + +func getNumberOfBatchableSigsInTxn(stx transactions.SignedTxn) (batchSigs uint64, err error) { + var hasSig, hasMsig bool + numSigs := 0 + if stx.Sig != (crypto.Signature{}) { + numSigs++ + hasSig = true + } + if !stx.Msig.Blank() { + numSigs++ + hasMsig = true + } + if !stx.Lsig.Blank() { + numSigs++ + } + + if numSigs == 0 { + return 0, errors.New("signedtxn has no sig") + } + if numSigs != 1 { + return 0, errors.New("signedtxn should only have one of Sig or Msig or LogicSig") + } + if hasSig { + return 1, nil + } + if hasMsig { + sig := stx.Msig + for _, subsigi := range sig.Subsigs { + if (subsigi.Sig != crypto.Signature{}) { + batchSigs++ + } + } + } + return +} diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 26499c01d8..09ab0a15a3 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -915,7 +915,7 @@ func TestStreamVerifier(t *testing.T) { defer cancel() nbw := MakeNewBlockWatcher(blkHdr) - stxnChan := make(chan VerificationElement) + stxnChan := make(chan UnverifiedElement) resultChan := MakeStream(ctx, stxnChan, nil, nbw, verificationPool, cache) badTxnGroups := make(map[crypto.Signature]struct{}) @@ -962,7 +962,7 @@ func TestStreamVerifier(t *testing.T) { case <-ctx.Done(): break default: - stxnChan <- VerificationElement{TxnGroup: tg, Context: nil} + stxnChan <- UnverifiedElement{TxnGroup: tg, Context: nil} } } }() diff --git a/data/txHandler.go b/data/txHandler.go index d80f65937e..88b42d8dec 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -69,7 +69,7 @@ type TxHandler struct { net network.GossipNode ctx context.Context ctxCancel context.CancelFunc - streamVerifierChan chan verify.VerificationElement + streamVerifierChan chan verify.UnverifiedElement } // MakeTxHandler makes a new handler for transaction messages @@ -94,7 +94,7 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go backlogQueue: make(chan *txBacklogMsg, txBacklogSize), postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, - streamVerifierChan: make(chan verify.VerificationElement, verifierStreamBufferSize), + streamVerifierChan: make(chan verify.UnverifiedElement, verifierStreamBufferSize), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) @@ -190,7 +190,7 @@ func (handler *TxHandler) backlogWorker() { continue } - handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 2aac403fce..3f76fbc28b 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -331,7 +331,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { // this is not expected during the test continue } - handler.streamVerifierChan <- verify.VerificationElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { @@ -443,7 +443,7 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 0.1 { + if rand.Float32() < 20.1 { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} From febee128b283b675ee738b6bdb0a0b9be914ae68 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 17 Oct 2022 23:14:50 -0400 Subject: [PATCH 051/156] a fix --- data/transactions/verify/txn.go | 10 +--------- data/txHandler_test.go | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index bfe798d6f4..acd3c01d65 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -566,11 +566,6 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext const waitForNextTxnDuration = 50 * time.Millisecond const waitForFirstTxnDuration = 2000 * time.Millisecond -// internalBufferSize is the size of the chan that will hold the arriving stxns before they get pre-processed -const internalBufferSize = 0 - -//const txnPerWorksetThreshold = 32 - // MakeStream creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger logic.LedgerForSignature, @@ -734,7 +729,6 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) batchVerifier := crypto.MakeBatchVerifier() bl := makeBatchLoad() - previousTotal := 0 // TODO: separate operations here, and get the sig verification inside LogicSig outside for _, ue := range uel.elementList { groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) @@ -744,9 +738,7 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) continue } totalBatchCount := batchVerifier.GetNumberOfEnqueuedSignatures() - currentCount := totalBatchCount - previousTotal - previousTotal = totalBatchCount - bl.addLoad(ue.TxnGroup, groupCtx, ue.Context, currentCount) + bl.addLoad(ue.TxnGroup, groupCtx, ue.Context, totalBatchCount) } failed, err := batchVerifier.VerifyWithFeedback() diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 3f76fbc28b..800ce0d41b 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -361,23 +361,28 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { wg.Add(1) go func() { defer wg.Done() - counter := 0 + groupCounter := 0 + txnCounter := 0 + invalidCounter := 0 defer func() { - fmt.Printf("processed %d txns\n", counter) + fmt.Printf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) }() b.ResetTimer() tt := time.Now() for wi := range outChan { - counter++ + txnCounter = txnCounter + len(wi.unverifiedTxGroup) + groupCounter++ u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { - require.False(b, inBad, "No error for bad signature") + require.False(b, inBad, "No error for invalid signature") } else { + invalidCounter++ require.True(b, inBad, "Error for good signature") } } - fmt.Printf("TPS: %d\n", uint64(counter)*1000000000/uint64(time.Since(tt))) + fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + fmt.Printf("Txn groups with invalid sigs: %d\n", invalidCounter) }() // Send the transactions to the verifier @@ -397,7 +402,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) - fmt.Printf("Dropped from backlog: %s\n", str) + fmt.Printf("dropped %s txn gropus\n", str) } // Prepare N transaction groups of random sizes with randomly invalid signatures @@ -443,7 +448,7 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 20.1 { + if rand.Float32() < 0.1 { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} From 1c88acddb9d5a66478d1ac1280d91e5f25b12190 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 18 Oct 2022 00:37:50 -0400 Subject: [PATCH 052/156] fix bug --- data/transactions/verify/txn.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index acd3c01d65..d2927dab4b 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -563,7 +563,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext // [validation time added to the group by one more txn] = [validation time of a single txn] / 2 // This gives us: // [wait time] <= [validation time of a single txn] / 2 -const waitForNextTxnDuration = 50 * time.Millisecond +const waitForNextTxnDuration = 5 * time.Millisecond const waitForFirstTxnDuration = 2000 * time.Millisecond // MakeStream creates a new stream verifier and returns the chans used to send txn groups @@ -634,6 +634,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out continue } + numberOfSigsInCurrent = 0 uel = makeUnverifiedElementList(nbw, ledger) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) @@ -652,6 +653,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l } added = sm.processBatch(uel) if added { + numberOfSigsInCurrent = 0 uel = makeUnverifiedElementList(nbw, ledger) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) From 6f2794f053c70e9d77f2980547141619886e593c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 18 Oct 2022 23:53:23 -0400 Subject: [PATCH 053/156] some more progress --- crypto/batchverifier_test.go | 22 +++++++++++ data/transactions/verify/txn.go | 65 ++++++++++++++++----------------- data/txHandler_test.go | 3 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 8d47857a73..c1d9e28dfa 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -17,6 +17,7 @@ package crypto import ( + "fmt" "math/rand" "testing" @@ -120,6 +121,27 @@ func BenchmarkBatchVerifier(b *testing.B) { require.NoError(b, bv.Verify()) } +func BenchmarkBatchVerifierBig(b *testing.B) { + c := makeCurve25519Secret() + for batchSize := 1; batchSize <= 48; batchSize++ { + bv := MakeBatchVerifierWithHint(batchSize) + for i := 0; i < batchSize; i++ { + str := randString() + bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str)) + } + b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) { + totalTransactions := b.N + count := totalTransactions / batchSize + if count * batchSize < totalTransactions { + count++ + } + for x := 0; x < count; x++ { + require.NoError(b, bv.Verify()) + } + }) + } +} + func TestEmpty(t *testing.T) { partitiontest.PartitionTest(t) bv := MakeBatchVerifier() diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index d2927dab4b..a2dc29ac9e 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -480,7 +480,7 @@ type streamManager struct { verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache - execPoolWg sync.WaitGroup + pendingTasksWg sync.WaitGroup } // NewBlockWatcher is a struct used to provide a new block header to the @@ -598,13 +598,11 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer if !ok { if numberOfSigsInCurrent > 0 { - err := sm.processFullBatch(uel) - if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out - } + sm.addVerificationTaskToThePool(uel) + // ignore the error since we are done here } - // for for all pending tasks out, then close the result chan - sm.execPoolWg.Wait() + // wait for the pending tasks, then close the result chan + sm.pendingTasksWg.Wait() close(resultChan) return } @@ -618,9 +616,10 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // if no batchable signatures here, send this as a task of its own if numberOfBatchableSigsInGroup == 0 { - err := sm.processFullBatch(makeSingleUnverifiedElement(nbw, ledger, stx)) + // no point in blocking and waiting for a seat here, let it wait in the exec pool + err := sm.addVerificationTaskToThePool(makeSingleUnverifiedElement(nbw, ledger, stx)) if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out + sm.sendResult(stx.TxnGroup, stx.Context, err) } continue } @@ -628,16 +627,18 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // add this txngrp to the list of batchable txn groups numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup uel.elementList = append(uel.elementList, stx) - if numberOfSigsInCurrent >= txnPerWorksetThreshold { - err := sm.processFullBatch(uel) - if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) // TODO: maybe this error in internal, and should not go out - continue + if numberOfSigsInCurrent > txnPerWorksetThreshold { + // enough transaction in the batch to efficiently verify + added = sm.processBatch(uel) + if added { + numberOfSigsInCurrent = 0 + uel = makeUnverifiedElementList(nbw, ledger) + // starting a new batch. Can wait long, since nothing is blocked + timer.Reset(waitForFirstTxnDuration) + } else { + // was not added because no-available-seats. wait for some more txns + timer.Reset(waitForNextTxnDuration) } - numberOfSigsInCurrent = 0 - uel = makeUnverifiedElementList(nbw, ledger) - // starting a new batch. Can wait long, since nothing is blocked - timer.Reset(waitForFirstTxnDuration) } else { if isFirstInBatch { // an element is added and is waiting. shorten the waiting time @@ -691,37 +692,29 @@ func (sm *streamManager) sendResult(veTxnGroup []transactions.SignedTxn, veConte */ } -func (sm *streamManager) processFullBatch(uel unverifiedElementList) (err error) { - // This is a full batch, no additions are allowed - // block and wait for a free seat - <-sm.seatReturnChan - err = sm.addVerificationTaskToThePool(uel) - return -} - func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { // if cannot find a seat, can go back and collect // more signatures instead of waiting here + // more signatures to the batch do not harm performance (see crypto.BenchmarkBatchVerifierBig) select { case <-sm.seatReturnChan: err := sm.addVerificationTaskToThePool(uel) if err != nil { - // TODO: report the error - fmt.Println(err) + // An error is returned when the context of the pool expires + // No need to report this + return false } - added = true - // TODO: queue to the pool. - // fmt.Println(err) + return true default: // if no free seats, wait some more for more txns + return false } - return } func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) error { - sm.execPoolWg.Add(1) + sm.pendingTasksWg.Add(1) function := func(arg interface{}) interface{} { - defer sm.execPoolWg.Done() + defer sm.pendingTasksWg.Done() if sm.ctx.Err() != nil { return sm.ctx.Err() @@ -733,6 +726,9 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) bl := makeBatchLoad() // TODO: separate operations here, and get the sig verification inside LogicSig outside for _, ue := range uel.elementList { + if ue.TxnGroup[len(ue.TxnGroup)-1].Sig[0] == 0 && ue.TxnGroup[len(ue.TxnGroup)-1].Sig[2] == 0 && ue.TxnGroup[len(ue.TxnGroup)-1].Sig[4] == 0 { + fmt.Println("boom1") + } groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) if err != nil { // verification failed, cannot go to the batch. return here. @@ -784,6 +780,7 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) } return struct{}{} } + // EnqueueBacklog returns an error when the context is canceled err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, &uel, sm.seatReturnChan) return err } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 800ce0d41b..3e4fc8302a 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -366,6 +366,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { invalidCounter := 0 defer func() { fmt.Printf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) + fmt.Printf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) }() b.ResetTimer() tt := time.Now() @@ -378,7 +379,7 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { require.False(b, inBad, "No error for invalid signature") } else { invalidCounter++ - require.True(b, inBad, "Error for good signature") + require.True(b, inBad, fmt.Sprintf("Error for good signature: %s", wi.verificationErr)) } } fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) From e4d337489130457310360eba6c0201d62bc026fa Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 19 Oct 2022 00:29:22 -0400 Subject: [PATCH 054/156] try higher error rate --- data/txHandler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 3e4fc8302a..f04f595b2b 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -449,7 +449,7 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 0.1 { + if rand.Float32() < 0.7 { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} From a4de8cab461aef0a2bdf8d16f9706a4fe6f675c6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 20 Oct 2022 00:23:36 -0400 Subject: [PATCH 055/156] add benchmarks to evaluate the batching performance gains as the batchsize increases and as invalid sigs are introduced --- crypto/batchverifier_test.go | 42 +++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index c1d9e28dfa..07cea37735 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -121,9 +121,11 @@ func BenchmarkBatchVerifier(b *testing.B) { require.NoError(b, bv.Verify()) } +// BenchmarkBatchVerifierBig with b.N over 1000 will report the expected performance +// gain as the batchsize increases. All sigs are valid. func BenchmarkBatchVerifierBig(b *testing.B) { c := makeCurve25519Secret() - for batchSize := 1; batchSize <= 48; batchSize++ { + for batchSize := 1; batchSize <= 96; batchSize++ { bv := MakeBatchVerifierWithHint(batchSize) for i := 0; i < batchSize; i++ { str := randString() @@ -142,6 +144,44 @@ func BenchmarkBatchVerifierBig(b *testing.B) { } } +// BenchmarkBatchVerifierBigWithInvalid builds over BenchmarkBatchVerifierBig by introducing +// invalid sigs to even numbered batch sizes. This shows the impact of invalid sigs on the +// performance. Basically, all the gains from batching disappear. +func BenchmarkBatchVerifierBigWithInvalid(b *testing.B) { + c := makeCurve25519Secret() + badSig := Signature{} + for batchSize := 1; batchSize <= 96; batchSize++ { + bv := MakeBatchVerifierWithHint(batchSize) + for i := 0; i < batchSize; i++ { + str := randString() + if batchSize%2 == 0 && (i == 0 || rand.Float32() < 0.1) { + bv.EnqueueSignature(c.SignatureVerifier, str, badSig) + } else { + bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str)) + } + } + b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) { + totalTransactions := b.N + count := totalTransactions / batchSize + if count * batchSize < totalTransactions { + count++ + } + for x := 0; x < count; x++ { + failed, err := bv.VerifyWithFeedback() + if err != nil { + for i, f := range failed { + if bv.signatures[i] == badSig { + require.True(b, f) + } else { + require.False(b, f) + } + } + } + } + }) + } +} + func TestEmpty(t *testing.T) { partitiontest.PartitionTest(t) bv := MakeBatchVerifier() From 5265923bc3bc9d6ea43e89aa4828e9deeb80bd74 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 20 Oct 2022 00:23:51 -0400 Subject: [PATCH 056/156] gofmt --- crypto/batchverifier.go | 9 ++++++++- crypto/batchverifier_test.go | 6 +++--- data/txHandler_test.go | 11 +++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 991b66eb56..9ec61c2ccb 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -38,6 +38,7 @@ package crypto import "C" import ( "errors" + // "runtime" "unsafe" ) @@ -133,11 +134,13 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { return failed, ErrBatchHasFailedSigs } +var Counter int + // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners // otherwise, returns false, and sets the indexes of the failed sigs in failed func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { - + Counter++ numberOfSignatures := len(messages) messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) @@ -153,6 +156,10 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si C.free(publicKeysAllocation) C.free(signaturesAllocation) C.free(valid) + + // runtime.KeepAlive(messages) + // runtime.KeepAlive(publicKeys) + // runtime.KeepAlive(signatures) }() // load all the data pointers into the array pointers. diff --git a/crypto/batchverifier_test.go b/crypto/batchverifier_test.go index 07cea37735..ccf1afeec8 100644 --- a/crypto/batchverifier_test.go +++ b/crypto/batchverifier_test.go @@ -134,7 +134,7 @@ func BenchmarkBatchVerifierBig(b *testing.B) { b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) { totalTransactions := b.N count := totalTransactions / batchSize - if count * batchSize < totalTransactions { + if count*batchSize < totalTransactions { count++ } for x := 0; x < count; x++ { @@ -146,7 +146,7 @@ func BenchmarkBatchVerifierBig(b *testing.B) { // BenchmarkBatchVerifierBigWithInvalid builds over BenchmarkBatchVerifierBig by introducing // invalid sigs to even numbered batch sizes. This shows the impact of invalid sigs on the -// performance. Basically, all the gains from batching disappear. +// performance. Basically, all the gains from batching disappear. func BenchmarkBatchVerifierBigWithInvalid(b *testing.B) { c := makeCurve25519Secret() badSig := Signature{} @@ -163,7 +163,7 @@ func BenchmarkBatchVerifierBigWithInvalid(b *testing.B) { b.Run(fmt.Sprintf("running batchsize %d", batchSize), func(b *testing.B) { totalTransactions := b.N count := totalTransactions / batchSize - if count * batchSize < totalTransactions { + if count*batchSize < totalTransactions { count++ } for x := 0; x < count; x++ { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index f04f595b2b..6117c523f0 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -256,6 +256,7 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { // BenchmarkIncomingTxHandlerProcessing sends singed transactions to be handled and verified // It reports the number of dropped transactions func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { + crypto.Counter = 0 const numUsers = 100 log := logging.TestingLog(b) log.SetLevel(logging.Warn) @@ -376,10 +377,16 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { - require.False(b, inBad, "No error for invalid signature") + if inBad { + fmt.Printf("No error for invalid signature XXXXXX ***********************\n") + } + // require.False(b, inBad, "No error for invalid signature") } else { invalidCounter++ - require.True(b, inBad, fmt.Sprintf("Error for good signature: %s", wi.verificationErr)) + if !inBad{ + // fmt.Printf("Error for good signature: %s\n", wi.verificationErr) + } + // require.True(b, inBad, fmt.Sprintf("Error for good signature: %s", wi.verificationErr)) } } fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) From 96449823ce1f5bbf7fd08499a2deca4e57a7ca7a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 20 Oct 2022 22:29:22 -0400 Subject: [PATCH 057/156] benchmark without backlog --- data/transactions/verify/txn.go | 4 +- data/txHandler_test.go | 110 ++++++++++++++++++++++++++++---- 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a2dc29ac9e..69398cfbbc 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -50,7 +50,7 @@ var ErrInvalidSignature = errors.New("At least one signature didn't pass verific // to avoid context switching overhead while providing good validation cancelation responsiveness. Each one of these worksets is // "populated" with roughly txnPerWorksetThreshold transactions. ( note that the real evaluation time is unknown, but benchmarks // show that these are realistic numbers ) -const txnPerWorksetThreshold = 32 +const txnPerWorksetThreshold = 64 // When the PaysetGroups is generating worksets, it enqueues up to concurrentWorksets entries to the execution pool. This serves several // purposes : @@ -573,7 +573,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l resultOtput <-chan VerificationResult) { resultChan := make(chan VerificationResult) - numberOfExecPoolSeats := verificationPool.GetParallelism() + numberOfExecPoolSeats := verificationPool.GetParallelism() * 10 sm := streamManager{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 6117c523f0..d9cc3074df 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -254,7 +254,8 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } // BenchmarkIncomingTxHandlerProcessing sends singed transactions to be handled and verified -// It reports the number of dropped transactions +// It reports the number of dropped transactions. This includes the decoding of the +// messages, and emulating the backlog processing. func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { crypto.Counter = 0 const numUsers = 100 @@ -377,16 +378,10 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { - if inBad { - fmt.Printf("No error for invalid signature XXXXXX ***********************\n") - } - // require.False(b, inBad, "No error for invalid signature") + require.False(b, inBad, "No error for invalid signature") } else { invalidCounter++ - if !inBad{ - // fmt.Printf("Error for good signature: %s\n", wi.verificationErr) - } - // require.True(b, inBad, fmt.Sprintf("Error for good signature: %s", wi.verificationErr)) + require.True(b, inBad, "Error for good signature") } } fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) @@ -395,7 +390,6 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { // Send the transactions to the verifier for _, tg := range encodedSignedTransactionGroups { - // time the streaming of the txns to at most 6000 tps handler.processIncomingTxn(tg) randduration := time.Duration(uint64(((1 + rand.Float32()) * 3))) time.Sleep(randduration * time.Microsecond) @@ -456,7 +450,7 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 0.7 { + if rand.Float32() < 0.0001 { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} @@ -465,3 +459,97 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, } return } + +// BenchmarkHandler sends singed transactions to be handled and verified +func BenchmarkHandler(b *testing.B) { + crypto.Counter = 0 + const numUsers = 100 + log := logging.TestingLog(b) + log.SetLevel(logging.Warn) + addresses := make([]basics.Address, numUsers) + secrets := make([]*crypto.SignatureSecrets, numUsers) + + // prepare the accounts + genesis := make(map[basics.Address]basics.AccountData) + for i := 0; i < numUsers; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + genesis[addr] = basics.AccountData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, + } + } + genesis[poolAddr] = basics.AccountData{ + Status: basics.NotParticipating, + MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, + } + + require.Equal(b, len(genesis), numUsers+1) + genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) + require.NoError(b, err) + + l := ledger + tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) + handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + defer handler.ctxCancel() + + // Prepare the transactions + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) + outChan := handler.postVerificationQueue + wg := sync.WaitGroup{} + + // Process the results and make sure they are correct + wg.Add(1) + go func() { + defer wg.Done() + groupCounter := 0 + txnCounter := 0 + invalidCounter := 0 + defer func() { + fmt.Printf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) + fmt.Printf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) + }() + b.ResetTimer() + tt := time.Now() + for wi := range outChan { + txnCounter = txnCounter + len(wi.unverifiedTxGroup) + groupCounter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.verificationErr == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + } + fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + fmt.Printf("Txn groups with invalid sigs: %d\n", invalidCounter) + }() + + for _, stxngrp := range signedTransactionGroups { + blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, Context: &blm} + } + // shut down to end the test + close(handler.streamVerifierChan) + close(handler.backlogQueue) + wg.Wait() + + // Report the number of transactions dropped because the backlog was busy + var buf strings.Builder + metrics.DefaultRegistry().WriteMetrics(&buf, "") + str := buf.String() + x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") + str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] + str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) + fmt.Printf("dropped %s txn gropus\n", str) +} From 0c168e99841c74258ec4d01b175fd9d8d70486a7 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 21 Oct 2022 16:20:31 -0400 Subject: [PATCH 058/156] organize the test and the benchmark --- data/transactions/verify/txn.go | 3 -- data/txHandler_test.go | 82 ++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 69398cfbbc..ba8d9ed628 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -726,9 +726,6 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) bl := makeBatchLoad() // TODO: separate operations here, and get the sig verification inside LogicSig outside for _, ue := range uel.elementList { - if ue.TxnGroup[len(ue.TxnGroup)-1].Sig[0] == 0 && ue.TxnGroup[len(ue.TxnGroup)-1].Sig[2] == 0 && ue.TxnGroup[len(ue.TxnGroup)-1].Sig[4] == 0 { - fmt.Println("boom1") - } groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) if err != nil { // verification failed, cannot go to the batch. return here. diff --git a/data/txHandler_test.go b/data/txHandler_test.go index d9cc3074df..2ff5b5d10e 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -253,13 +253,24 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } } -// BenchmarkIncomingTxHandlerProcessing sends singed transactions to be handled and verified +func TestIncomingTxHandle(t *testing.T) { + incomingTxHandlerProcessing(1, t) +} + +func TestIncomingTxGroupHandle(t *testing.T) { + incomingTxHandlerProcessing(proto.MaxTxGroupSize, t) +} + +// incomingTxHandlerProcessing is comprehensive transaction handling test, emulating +// the production code as much as possible. +// sends singed transactions to be handled and verified // It reports the number of dropped transactions. This includes the decoding of the // messages, and emulating the backlog processing. -func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { +func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { crypto.Counter = 0 const numUsers = 100 - log := logging.TestingLog(b) + numberOfTransactionGroups := 1000 + log := logging.TestingLog(t) log.SetLevel(logging.Warn) addresses := make([]basics.Address, numUsers) secrets := make([]*crypto.SignatureSecrets, numUsers) @@ -281,14 +292,14 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, } - require.Equal(b, len(genesis), numUsers+1) + require.Equal(t, len(genesis), numUsers+1) genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) - ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + ledgerName := fmt.Sprintf("%s-mem-%d", t.Name(), numberOfTransactionGroups) const inMem = true cfg := config.GetDefaultLocal() cfg.Archival = true ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) - require.NoError(b, err) + require.NoError(t, err) l := ledger tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) @@ -348,8 +359,9 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { }() // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) - encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) + signedTransactionGroups, badTxnGroups := + makeSignedTxnGroups(numberOfTransactionGroups, numUsers, maxGroupSize, addresses, secrets) + encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, numberOfTransactionGroups) for _, stxngrp := range signedTransactionGroups { data := make([]byte, 0) for _, stxn := range stxngrp { @@ -367,10 +379,11 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { txnCounter := 0 invalidCounter := 0 defer func() { - fmt.Printf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) - fmt.Printf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) + t.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) + if crypto.Counter > 0 { + t.Logf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) + } }() - b.ResetTimer() tt := time.Now() for wi := range outChan { txnCounter = txnCounter + len(wi.unverifiedTxGroup) @@ -378,14 +391,14 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { - require.False(b, inBad, "No error for invalid signature") + require.False(t, inBad, "No error for invalid signature") } else { invalidCounter++ - require.True(b, inBad, "Error for good signature") + require.True(t, inBad, "Error for good signature") } } - fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - fmt.Printf("Txn groups with invalid sigs: %d\n", invalidCounter) + t.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + t.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) }() // Send the transactions to the verifier @@ -404,19 +417,22 @@ func BenchmarkIncomingTxHandlerProcessing(b *testing.B) { x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) - fmt.Printf("dropped %s txn gropus\n", str) + t.Logf("dropped %s txn gropus\n", str) } // Prepare N transaction groups of random sizes with randomly invalid signatures -func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, +func makeSignedTxnGroups(N, numUsers, maxGroupSize int, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { badTxnGroups = make(map[uint64]interface{}) - maxGrpSize := proto.MaxTxGroupSize + protoMaxGrpSize := proto.MaxTxGroupSize ret = make([][]transactions.SignedTxn, 0, N) for u := 0; u < N; u++ { - grpSize := rand.Intn(maxGrpSize-1) + 1 + grpSize := rand.Intn(protoMaxGrpSize-1) + 1 + if grpSize > maxGroupSize { + grpSize = maxGroupSize + } var txGroup transactions.TxGroup txns := make([]transactions.Transaction, 0, grpSize) for g := 0; g < grpSize; g++ { @@ -461,7 +477,17 @@ func makeSignedTxnGroups(N, numUsers int, addresses []basics.Address, } // BenchmarkHandler sends singed transactions to be handled and verified -func BenchmarkHandler(b *testing.B) { +func BenchmarkHandleTxns(b *testing.B) { + b.N = b.N * proto.MaxTxGroupSize / 2 + runHandlerBenchmark(1, b) +} + +// BenchmarkHandler sends singed transaction groups to be handled and verified +func BenchmarkHandleTxnGroups(b *testing.B) { + runHandlerBenchmark(proto.MaxTxGroupSize, b) +} + +func runHandlerBenchmark(maxGroupSize int, b *testing.B) { crypto.Counter = 0 const numUsers = 100 log := logging.TestingLog(b) @@ -502,7 +528,7 @@ func BenchmarkHandler(b *testing.B) { defer handler.ctxCancel() // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, addresses, secrets) + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, addresses, secrets) outChan := handler.postVerificationQueue wg := sync.WaitGroup{} @@ -514,10 +540,11 @@ func BenchmarkHandler(b *testing.B) { txnCounter := 0 invalidCounter := 0 defer func() { - fmt.Printf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) - fmt.Printf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) + b.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) + if crypto.Counter > 0 { + b.Logf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) + } }() - b.ResetTimer() tt := time.Now() for wi := range outChan { txnCounter = txnCounter + len(wi.unverifiedTxGroup) @@ -531,10 +558,11 @@ func BenchmarkHandler(b *testing.B) { require.True(b, inBad, "Error for good signature") } } - fmt.Printf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - fmt.Printf("Txn groups with invalid sigs: %d\n", invalidCounter) + b.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + b.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) }() + b.ResetTimer() for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, Context: &blm} @@ -551,5 +579,5 @@ func BenchmarkHandler(b *testing.B) { x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) - fmt.Printf("dropped %s txn gropus\n", str) + b.Logf("dropped %s txn gropus\n", str) } From dc197003cb9b2a9a847ac86292d43c6761e07478 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 21 Oct 2022 22:28:48 -0400 Subject: [PATCH 059/156] reorganizing the benchmark into a test and a benchmark --- data/txHandler_test.go | 40 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 2ff5b5d10e..428e6bf51f 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -267,7 +267,6 @@ func TestIncomingTxGroupHandle(t *testing.T) { // It reports the number of dropped transactions. This includes the decoding of the // messages, and emulating the backlog processing. func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { - crypto.Counter = 0 const numUsers = 100 numberOfTransactionGroups := 1000 log := logging.TestingLog(t) @@ -360,7 +359,7 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { // Prepare the transactions signedTransactionGroups, badTxnGroups := - makeSignedTxnGroups(numberOfTransactionGroups, numUsers, maxGroupSize, addresses, secrets) + makeSignedTxnGroups(numberOfTransactionGroups, numUsers, maxGroupSize, 0.5, addresses, secrets) encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, numberOfTransactionGroups) for _, stxngrp := range signedTransactionGroups { data := make([]byte, 0) @@ -380,11 +379,7 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { invalidCounter := 0 defer func() { t.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) - if crypto.Counter > 0 { - t.Logf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) - } }() - tt := time.Now() for wi := range outChan { txnCounter = txnCounter + len(wi.unverifiedTxGroup) groupCounter++ @@ -397,7 +392,6 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { require.True(t, inBad, "Error for good signature") } } - t.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) t.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) }() @@ -421,7 +415,7 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { } // Prepare N transaction groups of random sizes with randomly invalid signatures -func makeSignedTxnGroups(N, numUsers, maxGroupSize int, addresses []basics.Address, +func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalid float32, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { badTxnGroups = make(map[uint64]interface{}) @@ -466,7 +460,7 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, addresses []basics.Addre signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < 0.0001 { + if rand.Float32() < invalid { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} @@ -488,7 +482,6 @@ func BenchmarkHandleTxnGroups(b *testing.B) { } func runHandlerBenchmark(maxGroupSize int, b *testing.B) { - crypto.Counter = 0 const numUsers = 100 log := logging.TestingLog(b) log.SetLevel(logging.Warn) @@ -528,26 +521,20 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { defer handler.ctxCancel() // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, addresses, secrets) + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) outChan := handler.postVerificationQueue wg := sync.WaitGroup{} + var tt time.Time // Process the results and make sure they are correct wg.Add(1) go func() { defer wg.Done() groupCounter := 0 - txnCounter := 0 + var txnCounter uint64 invalidCounter := 0 - defer func() { - b.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) - if crypto.Counter > 0 { - b.Logf("crypto calls %d avg txn/batch: %d\n", crypto.Counter, txnCounter/crypto.Counter) - } - }() - tt := time.Now() for wi := range outChan { - txnCounter = txnCounter + len(wi.unverifiedTxGroup) + txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) groupCounter++ u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] @@ -559,10 +546,12 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { } } b.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - b.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) + b.Logf("Time/txn: %d(microsec)\n", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("processed total: [%d groups (%d invalid)] [%d txns]\n", groupCounter, invalidCounter, txnCounter) }() b.ResetTimer() + tt = time.Now() for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, Context: &blm} @@ -571,13 +560,4 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { close(handler.streamVerifierChan) close(handler.backlogQueue) wg.Wait() - - // Report the number of transactions dropped because the backlog was busy - var buf strings.Builder - metrics.DefaultRegistry().WriteMetrics(&buf, "") - str := buf.String() - x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") - str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] - str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) - b.Logf("dropped %s txn gropus\n", str) } From 91bd7583f8a6a469423c77c3d77556e1ca9a5348 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 21 Oct 2022 22:44:22 -0400 Subject: [PATCH 060/156] cleanup --- crypto/batchverifier.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 2b7183bc23..d3f3b9bc9b 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -134,13 +134,11 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { return failed, ErrBatchHasFailedSigs } -var Counter int - // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners // otherwise, returns false, and sets the indexes of the failed sigs in failed func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { - Counter++ + numberOfSignatures := len(messages) messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) From e834b74d0bb6524d2e0a8e8ee6caa91af688cd05 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 22 Oct 2022 00:34:44 -0400 Subject: [PATCH 061/156] comments --- data/txHandler_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 428e6bf51f..d1e1139b71 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -261,11 +261,8 @@ func TestIncomingTxGroupHandle(t *testing.T) { incomingTxHandlerProcessing(proto.MaxTxGroupSize, t) } -// incomingTxHandlerProcessing is comprehensive transaction handling test, emulating -// the production code as much as possible. -// sends singed transactions to be handled and verified -// It reports the number of dropped transactions. This includes the decoding of the -// messages, and emulating the backlog processing. +// incomingTxHandlerProcessing is a comprehensive transaction handling test +// It handles the singed transactions by passing them to the backlog for verification func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { const numUsers = 100 numberOfTransactionGroups := 1000 @@ -414,8 +411,9 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { t.Logf("dropped %s txn gropus\n", str) } -// Prepare N transaction groups of random sizes with randomly invalid signatures -func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalid float32, addresses []basics.Address, +// makeSignedTxnGroups prepares N transaction groups of random (maxGroupSize) sizes with random +// invalid signatures of a given probability (invalidProb) +func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, addresses []basics.Address, secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { badTxnGroups = make(map[uint64]interface{}) @@ -460,7 +458,7 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalid float32, address signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures - if rand.Float32() < invalid { + if rand.Float32() < invalidProb { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 badTxnGroups[uint64(u)] = struct{}{} @@ -470,17 +468,19 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalid float32, address return } -// BenchmarkHandler sends singed transactions to be handled and verified +// BenchmarkHandler sends singed transactions the the verifier func BenchmarkHandleTxns(b *testing.B) { b.N = b.N * proto.MaxTxGroupSize / 2 runHandlerBenchmark(1, b) } -// BenchmarkHandler sends singed transaction groups to be handled and verified +// BenchmarkHandler sends singed transaction groups to the verifier func BenchmarkHandleTxnGroups(b *testing.B) { runHandlerBenchmark(proto.MaxTxGroupSize, b) } +// runHandlerBenchmark has a similar workflow to incomingTxHandlerProcessing, +// but bypasses the backlog, and sends the transactions directly to the verifier func runHandlerBenchmark(maxGroupSize int, b *testing.B) { const numUsers = 100 log := logging.TestingLog(b) From 5a98a953549b365c75c6fce3913d6e4237cdbb84 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 24 Oct 2022 16:14:18 -0400 Subject: [PATCH 062/156] test cleanup --- data/transactions/verify/txn_test.go | 102 +++++++-------------------- 1 file changed, 24 insertions(+), 78 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 09ab0a15a3..1c4a9ca1e2 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -66,17 +66,7 @@ type DummyLedgerForSignature struct { } func (d *DummyLedgerForSignature) BlockHdrCached(basics.Round) (bookkeeping.BlockHeader, error) { - return bookkeeping.BlockHeader{ - Round: 50, - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - UpgradeState: bookkeeping.UpgradeState{ - CurrentProtocol: protocol.ConsensusCurrentVersion, - }, - RewardsState: bookkeeping.RewardsState{ - FeeSink: feeSink, - RewardsPool: poolAddr, - }, - }, nil + return createDummyBlockHeader(), nil } func keypair() *crypto.SignatureSecrets { @@ -109,20 +99,7 @@ func generateMultiSigTxn(numTxs, numAccs int, blockRound basics.Round, t *testin exp = int(blockRound) + rand.Intn(30) } - txs[i] = transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: multiAddress[s], - Fee: basics.MicroAlgos{Raw: f}, - FirstValid: basics.Round(iss), - LastValid: basics.Round(exp), - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: multiAddress[r], - Amount: basics.MicroAlgos{Raw: uint64(a)}, - }, - } + txs[i] = createPayTransaction(f, iss, exp, a, multiAddress[s], multiAddress[r]) signed[i].Txn = txs[i] // create multi sig that 2 out of 3 has signed the txn @@ -193,20 +170,7 @@ func generateTestObjects(numTxs, numAccs int, blockRound basics.Round) ([]transa exp = int(blockRound) + rand.Intn(30) } - txs[i] = transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: addresses[s], - Fee: basics.MicroAlgos{Raw: f}, - FirstValid: basics.Round(iss), - LastValid: basics.Round(exp), - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: addresses[r], - Amount: basics.MicroAlgos{Raw: uint64(a)}, - }, - } + txs[i] = createPayTransaction(f, iss, exp, a, addresses[s], addresses[r]) signed[i] = txs[i].Sign(secrets[s]) u += 100 } @@ -713,20 +677,7 @@ func TestTxnGroupCacheUpdateLogicWithMultiSig(t *testing.T) { a := rand.Intn(1000) f := config.Consensus[protocol.ConsensusCurrentVersion].MinTxnFee + uint64(rand.Intn(10)) - signedTxn[i].Txn = transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: multiAddress[s], - Fee: basics.MicroAlgos{Raw: f}, - FirstValid: basics.Round(1), - LastValid: basics.Round(100), - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: multiAddress[r], - Amount: basics.MicroAlgos{Raw: uint64(a)}, - }, - } + signedTxn[i].Txn = createPayTransaction(f, 1, 100, a, multiAddress[s], multiAddress[r]) // add a simple logic that verifies this condition: // sha256(arg0) == base64decode(5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E=) op, err := logic.AssembleString(`arg 0 @@ -791,6 +742,23 @@ func createDummyBlockHeader() bookkeeping.BlockHeader { } } +func createPayTransaction(fee uint64, fv, lv, amount int, sender, receiver basics.Address) transactions.Transaction { + return transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: fee}, + FirstValid: basics.Round(fv), + LastValid: basics.Round(lv), + GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: uint64(amount)}, + }, + } +} + // verifyGroup uses TxnGroup to verify txns and add them to the // cache. Then makes sure that only the valid txns are verified and added to // the cache. @@ -859,19 +827,7 @@ func BenchmarkTxn(b *testing.B) { b.N = 2000 } _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 50) - blk := bookkeeping.Block{ - BlockHeader: bookkeeping.BlockHeader{ - Round: 50, - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - UpgradeState: bookkeeping.UpgradeState{ - CurrentProtocol: protocol.ConsensusCurrentVersion, - }, - RewardsState: bookkeeping.RewardsState{ - FeeSink: feeSink, - RewardsPool: poolAddr, - }, - }, - } + blk := bookkeeping.Block{BlockHeader: createDummyBlockHeader()} txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) b.ResetTimer() @@ -891,17 +847,7 @@ func TestStreamVerifier(t *testing.T) { numOfTxns := 10000 _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - blkHdr := bookkeeping.BlockHeader{ - Round: 50, - GenesisHash: crypto.Hash([]byte{1, 2, 3, 4, 5}), - UpgradeState: bookkeeping.UpgradeState{ - CurrentProtocol: protocol.ConsensusCurrentVersion, - }, - RewardsState: bookkeeping.RewardsState{ - FeeSink: feeSink, - RewardsPool: poolAddr, - }, - } + blkHdr := createDummyBlockHeader() execPool := execpool.MakePool(t) verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) @@ -932,7 +878,7 @@ func TestStreamVerifier(t *testing.T) { errChan := make(chan error) go func() { defer close(errChan) - // process the thecked signatures + // process the results for x := 0; x < numOfTxnGroups; x++ { select { case result := <-resultChan: From 64311b28e08e449b13c48a53e40eefe32a61ba95 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 24 Oct 2022 16:19:39 -0400 Subject: [PATCH 063/156] gofmt --- data/transactions/verify/txn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index ba8d9ed628..7810880f5b 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -480,7 +480,7 @@ type streamManager struct { verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache - pendingTasksWg sync.WaitGroup + pendingTasksWg sync.WaitGroup } // NewBlockWatcher is a struct used to provide a new block header to the From 9ea27d1cae8d939514f0ed8915813488901430da Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 24 Oct 2022 20:35:57 -0400 Subject: [PATCH 064/156] move the ctx check. check the error to avoid lint error --- data/transactions/verify/txn.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 7810880f5b..c074798807 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -594,12 +594,18 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l uel := makeUnverifiedElementList(nbw, ledger) for { select { + case <-ctx.Done(): + return //TODO: report the error ctx.Err() case stx, ok := <-stxnChan: // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer if !ok { if numberOfSigsInCurrent > 0 { - sm.addVerificationTaskToThePool(uel) - // ignore the error since we are done here + // send the current accumulated transactions to the pool + err := sm.addVerificationTaskToThePool(uel) + // if the poll is already terminated + if err != nil { + continue + } } // wait for the pending tasks, then close the result chan sm.pendingTasksWg.Wait() @@ -662,8 +668,6 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // was not added because no-available-seats. wait for some more txns timer.Reset(waitForNextTxnDuration) } - case <-ctx.Done(): - return //TODO: report the error ctx.Err() } } }() From d8fb642186d109b208b93937fcba38c2ba1164b5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 25 Oct 2022 01:21:06 -0400 Subject: [PATCH 065/156] fix stateproof handling: not error if no sigs for SP, no seats for SP --- data/transactions/verify/txn.go | 29 +++++++++++++++++++++-------- data/txHandler.go | 8 ++++---- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index c074798807..19093fc750 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -584,6 +584,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l } go func() { + defer close(resultChan) for x := 0; x < numberOfExecPoolSeats; x++ { sm.seatReturnChan <- struct{}{} } @@ -594,15 +595,13 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l uel := makeUnverifiedElementList(nbw, ledger) for { select { - case <-ctx.Done(): - return //TODO: report the error ctx.Err() case stx, ok := <-stxnChan: // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer if !ok { if numberOfSigsInCurrent > 0 { // send the current accumulated transactions to the pool - err := sm.addVerificationTaskToThePool(uel) - // if the poll is already terminated + err := sm.addVerificationTaskToThePool(uel, false) + // if the pool is already terminated if err != nil { continue } @@ -623,7 +622,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // if no batchable signatures here, send this as a task of its own if numberOfBatchableSigsInGroup == 0 { // no point in blocking and waiting for a seat here, let it wait in the exec pool - err := sm.addVerificationTaskToThePool(makeSingleUnverifiedElement(nbw, ledger, stx)) + err := sm.addVerificationTaskToThePool(makeSingleUnverifiedElement(nbw, ledger, stx), false) if err != nil { sm.sendResult(stx.TxnGroup, stx.Context, err) } @@ -668,6 +667,9 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l // was not added because no-available-seats. wait for some more txns timer.Reset(waitForNextTxnDuration) } + case <-ctx.Done(): + return //TODO: report the error ctx.Err() + } } }() @@ -702,7 +704,7 @@ func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { // more signatures to the batch do not harm performance (see crypto.BenchmarkBatchVerifierBig) select { case <-sm.seatReturnChan: - err := sm.addVerificationTaskToThePool(uel) + err := sm.addVerificationTaskToThePool(uel, true) if err != nil { // An error is returned when the context of the pool expires // No need to report this @@ -715,7 +717,7 @@ func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { } } -func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) error { +func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList, returnSeat bool) error { sm.pendingTasksWg.Add(1) function := func(arg interface{}) interface{} { defer sm.pendingTasksWg.Done() @@ -781,8 +783,12 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList) } return struct{}{} } + var retChan chan interface{} + if returnSeat { + retChan = sm.seatReturnChan + } // EnqueueBacklog returns an error when the context is canceled - err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, &uel, sm.seatReturnChan) + err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, &uel, retChan) return err } @@ -814,6 +820,13 @@ func getNumberOfBatchableSigsInTxn(stx transactions.SignedTxn) (batchSigs uint64 } if numSigs == 0 { + // Special case: special sender address can issue special transaction + // types (state proof txn) without any signature. The well-formed + // check ensures that this transaction cannot pay any fee, and + // cannot have any other interesting fields, except for the state proof payload. + if stx.Txn.Sender == transactions.StateProofSender && stx.Txn.Type == protocol.StateProofTx { + return 0, nil + } return 0, errors.New("signedtxn has no sig") } if numSigs != 1 { diff --git a/data/txHandler.go b/data/txHandler.go index 25a559d46f..3a3860c790 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -111,18 +111,18 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) outChan = verify.MakeStream(handler.ctx, handler.streamVerifierChan, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) - go handler.processTxnStreamVerResults(outChan) + go handler.processTxnStreamVerifiedResults(outChan) return handler } -func (handler *TxHandler) processTxnStreamVerResults(outChan <-chan verify.VerificationResult) { +func (handler *TxHandler) processTxnStreamVerifiedResults(outChan <-chan verify.VerificationResult) { + defer close(handler.postVerificationQueue) for { select { case <-handler.ctx.Done(): return case result, ok := <-outChan: if !ok { - close(handler.postVerificationQueue) return } txBLMsg := result.Context.(*txBacklogMsg) @@ -167,6 +167,7 @@ func (handler *TxHandler) backlogWorker() { // Note: TestIncomingTxHandle and TestIncomingTxGroupHandle emulate this function. // Changes to the behavior in this function should be reflected in the test. defer handler.backlogWg.Done() + defer close(handler.streamVerifierChan) for { // prioritize the postVerificationQueue select { @@ -185,7 +186,6 @@ func (handler *TxHandler) backlogWorker() { select { case wi, ok := <-handler.backlogQueue: if !ok { - close(handler.streamVerifierChan) return } if handler.checkAlreadyCommitted(wi) { From 5a7639ffe04e05ccb50a60beadfd571c3b3a7f7e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 25 Oct 2022 01:43:08 -0400 Subject: [PATCH 066/156] remove extra close chan --- data/transactions/verify/txn.go | 1 - 1 file changed, 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 19093fc750..fc3096ff0c 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -608,7 +608,6 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l } // wait for the pending tasks, then close the result chan sm.pendingTasksWg.Wait() - close(resultChan) return } From 4ce3b3253aba2345faa76fd02efd4bab2aaca45c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 25 Oct 2022 10:21:33 -0400 Subject: [PATCH 067/156] no way to close the verifier input channel --- data/transactions/verify/txn.go | 1 + data/txHandler.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index fc3096ff0c..ad2fe3e0a2 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -596,6 +596,7 @@ func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger l for { select { case stx, ok := <-stxnChan: + // TODO: stxnChan is never closed in production code. this is only used in tests. REMOVE // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer if !ok { if numberOfSigsInCurrent > 0 { diff --git a/data/txHandler.go b/data/txHandler.go index 3a3860c790..fcc5a2969b 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -46,6 +46,7 @@ var txBacklogSize = config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnByt var transactionMessagesHandled = metrics.MakeCounter(metrics.TransactionMessagesHandled) var transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) var transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + // verifierStreamBufferSize is the number of txn that coult be accumulated before the verifier stream consumes them var verifierStreamBufferSize = 1000 @@ -167,7 +168,6 @@ func (handler *TxHandler) backlogWorker() { // Note: TestIncomingTxHandle and TestIncomingTxGroupHandle emulate this function. // Changes to the behavior in this function should be reflected in the test. defer handler.backlogWg.Done() - defer close(handler.streamVerifierChan) for { // prioritize the postVerificationQueue select { From 81a49c5e64cf7321dd7b3d92c4d93a7ef6927a74 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 26 Oct 2022 00:31:52 -0400 Subject: [PATCH 068/156] support for stop/start --- data/transactions/verify/txn.go | 197 ++++++++++++++------------- data/transactions/verify/txn_test.go | 4 +- data/txHandler.go | 29 ++-- data/txHandler_test.go | 30 ++-- 4 files changed, 134 insertions(+), 126 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index ad2fe3e0a2..0fb1b7c1bd 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -474,13 +474,16 @@ type VerificationResult struct { Err error } -type streamManager struct { +type StreamVerifier struct { seatReturnChan chan interface{} resultChan chan<- VerificationResult + stxnChan <-chan UnverifiedElement verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache pendingTasksWg sync.WaitGroup + nbw *NewBlockWatcher + ledger logic.LedgerForSignature } // NewBlockWatcher is a struct used to provide a new block header to the @@ -568,125 +571,123 @@ const waitForFirstTxnDuration = 2000 * time.Millisecond // MakeStream creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStream(ctx context.Context, stxnChan <-chan UnverifiedElement, ledger logic.LedgerForSignature, - nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) ( - resultOtput <-chan VerificationResult) { +func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, + ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, + cache VerifiedTransactionCache) (sv *StreamVerifier) { - resultChan := make(chan VerificationResult) + // limit the number of tasks queued to the execution pool numberOfExecPoolSeats := verificationPool.GetParallelism() * 10 - sm := streamManager{ + sv = &StreamVerifier{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), resultChan: resultChan, + stxnChan: stxnChan, verificationPool: verificationPool, ctx: ctx, cache: cache, + nbw: nbw, + ledger: ledger, + } + for x := 0; x < numberOfExecPoolSeats; x++ { + sv.seatReturnChan <- struct{}{} } + return sv +} - go func() { - defer close(resultChan) - for x := 0; x < numberOfExecPoolSeats; x++ { - sm.seatReturnChan <- struct{}{} - } +func (sv *StreamVerifier) Start() { + go sv.batchingLoop() +} - timer := time.NewTicker(waitForFirstTxnDuration) - var added bool - var numberOfSigsInCurrent uint64 - uel := makeUnverifiedElementList(nbw, ledger) - for { - select { - case stx, ok := <-stxnChan: - // TODO: stxnChan is never closed in production code. this is only used in tests. REMOVE - // if not expecting any more transactions (!ok) then flush what is at hand instead of waiting for the timer - if !ok { - if numberOfSigsInCurrent > 0 { - // send the current accumulated transactions to the pool - err := sm.addVerificationTaskToThePool(uel, false) - // if the pool is already terminated - if err != nil { - continue - } - } - // wait for the pending tasks, then close the result chan - sm.pendingTasksWg.Wait() - return - } +func (sv *StreamVerifier) batchingLoop() { + timer := time.NewTicker(waitForFirstTxnDuration) + var added bool + var numberOfSigsInCurrent uint64 + uel := makeUnverifiedElementList(sv.nbw, sv.ledger) + for { + select { + case stx := <-sv.stxnChan: + isFirstInBatch := numberOfSigsInCurrent == 0 + numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) + if err != nil { + sv.sendResult(stx.TxnGroup, stx.Context, err) + continue + } - isFirstInBatch := numberOfSigsInCurrent == 0 - numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) + // if no batchable signatures here, send this as a task of its own + if numberOfBatchableSigsInGroup == 0 { + // no point in blocking and waiting for a seat here, let it wait in the exec pool + err := sv.addVerificationTaskToThePool(makeSingleUnverifiedElement(sv.nbw, sv.ledger, stx), false) if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) - continue - } - - // if no batchable signatures here, send this as a task of its own - if numberOfBatchableSigsInGroup == 0 { - // no point in blocking and waiting for a seat here, let it wait in the exec pool - err := sm.addVerificationTaskToThePool(makeSingleUnverifiedElement(nbw, ledger, stx), false) - if err != nil { - sm.sendResult(stx.TxnGroup, stx.Context, err) - } - continue + sv.sendResult(stx.TxnGroup, stx.Context, err) } + continue + } - // add this txngrp to the list of batchable txn groups - numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup - uel.elementList = append(uel.elementList, stx) - if numberOfSigsInCurrent > txnPerWorksetThreshold { - // enough transaction in the batch to efficiently verify - added = sm.processBatch(uel) - if added { - numberOfSigsInCurrent = 0 - uel = makeUnverifiedElementList(nbw, ledger) - // starting a new batch. Can wait long, since nothing is blocked - timer.Reset(waitForFirstTxnDuration) - } else { - // was not added because no-available-seats. wait for some more txns - timer.Reset(waitForNextTxnDuration) - } - } else { - if isFirstInBatch { - // an element is added and is waiting. shorten the waiting time - timer.Reset(waitForNextTxnDuration) - } - } - case <-timer.C: - // timer ticked. it is time to send the batch even if it is not full - if numberOfSigsInCurrent == 0 { - // nothing batched yet... wait some more - timer.Reset(waitForFirstTxnDuration) - continue - } - added = sm.processBatch(uel) + // add this txngrp to the list of batchable txn groups + numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup + uel.elementList = append(uel.elementList, stx) + if numberOfSigsInCurrent > txnPerWorksetThreshold { + // enough transaction in the batch to efficiently verify + added = sv.processBatch(uel) if added { numberOfSigsInCurrent = 0 - uel = makeUnverifiedElementList(nbw, ledger) + uel = makeUnverifiedElementList(sv.nbw, sv.ledger) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { // was not added because no-available-seats. wait for some more txns timer.Reset(waitForNextTxnDuration) } - case <-ctx.Done(): - return //TODO: report the error ctx.Err() - + } else { + if isFirstInBatch { + // an element is added and is waiting. shorten the waiting time + timer.Reset(waitForNextTxnDuration) + } } + case <-timer.C: + // timer ticked. it is time to send the batch even if it is not full + if numberOfSigsInCurrent == 0 { + // nothing batched yet... wait some more + timer.Reset(waitForFirstTxnDuration) + continue + } + added = sv.processBatch(uel) + if added { + numberOfSigsInCurrent = 0 + uel = makeUnverifiedElementList(sv.nbw, sv.ledger) + // starting a new batch. Can wait long, since nothing is blocked + timer.Reset(waitForFirstTxnDuration) + } else { + // was not added because no-available-seats. wait for some more txns + timer.Reset(waitForNextTxnDuration) + } + case <-sv.ctx.Done(): + if numberOfSigsInCurrent > 0 { + // send the current accumulated transactions to the pool + err := sv.addVerificationTaskToThePool(uel, false) + // if the pool is already terminated + if err != nil { + return + } + } + // wait for the pending tasks, then close the result chan + sv.pendingTasksWg.Wait() + return } - }() - return resultChan + } } -func (sm *streamManager) sendResult(veTxnGroup []transactions.SignedTxn, veContext interface{}, err error) { +func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veContext interface{}, err error) { vr := VerificationResult{ TxnGroup: veTxnGroup, Context: veContext, Err: err, } // send the txn result out the pipe - sm.resultChan <- vr + sv.resultChan <- vr /* select { - case sm.resultChan <- vr: + case sv.resultChan <- vr: // if the channel is not accepting, should not block here // report dropped txn. caching is fine, if it comes back in the block @@ -698,13 +699,13 @@ func (sm *streamManager) sendResult(veTxnGroup []transactions.SignedTxn, veConte */ } -func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { +func (sv *StreamVerifier) processBatch(uel unverifiedElementList) (added bool) { // if cannot find a seat, can go back and collect // more signatures instead of waiting here // more signatures to the batch do not harm performance (see crypto.BenchmarkBatchVerifierBig) select { - case <-sm.seatReturnChan: - err := sm.addVerificationTaskToThePool(uel, true) + case <-sv.seatReturnChan: + err := sv.addVerificationTaskToThePool(uel, true) if err != nil { // An error is returned when the context of the pool expires // No need to report this @@ -717,13 +718,13 @@ func (sm *streamManager) processBatch(uel unverifiedElementList) (added bool) { } } -func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList, returnSeat bool) error { - sm.pendingTasksWg.Add(1) +func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList, returnSeat bool) error { + sv.pendingTasksWg.Add(1) function := func(arg interface{}) interface{} { - defer sm.pendingTasksWg.Done() + defer sv.pendingTasksWg.Done() - if sm.ctx.Err() != nil { - return sm.ctx.Err() + if sv.ctx.Err() != nil { + return sv.ctx.Err() } uel := arg.(*unverifiedElementList) @@ -735,7 +736,7 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList, groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) if err != nil { // verification failed, cannot go to the batch. return here. - sm.sendResult(ue.TxnGroup, ue.Context, err) + sv.sendResult(ue.TxnGroup, ue.Context, err) continue } totalBatchCount := batchVerifier.GetNumberOfEnqueuedSignatures() @@ -773,10 +774,10 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList, } else { result = ErrInvalidSignature } - sm.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) + sv.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) } // loading them all at once by locking the cache once - err = sm.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) + err = sv.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) if err != nil { // TODO: handle the error fmt.Println(err) @@ -785,10 +786,10 @@ func (sm *streamManager) addVerificationTaskToThePool(uel unverifiedElementList, } var retChan chan interface{} if returnSeat { - retChan = sm.seatReturnChan + retChan = sv.seatReturnChan } // EnqueueBacklog returns an error when the context is canceled - err := sm.verificationPool.EnqueueBacklog(sm.ctx, function, &uel, retChan) + err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, &uel, retChan) return err } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 1c4a9ca1e2..c26318e395 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -862,7 +862,9 @@ func TestStreamVerifier(t *testing.T) { nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) - resultChan := MakeStream(ctx, stxnChan, nil, nbw, verificationPool, cache) + resultChan := make(chan VerificationResult) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, nil, nbw, verificationPool, cache) + sv.Start() badTxnGroups := make(map[crypto.Signature]struct{}) diff --git a/data/txHandler.go b/data/txHandler.go index fcc5a2969b..e25a0ee4fd 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -70,7 +70,9 @@ type TxHandler struct { net network.GossipNode ctx context.Context ctxCancel context.CancelFunc + streamVerifier *verify.StreamVerifier streamVerifierChan chan verify.UnverifiedElement + streamReturnChan chan verify.VerificationResult } // MakeTxHandler makes a new handler for transaction messages @@ -96,12 +98,12 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, streamVerifierChan: make(chan verify.UnverifiedElement, verifierStreamBufferSize), + streamReturnChan: make(chan verify.VerificationResult, verifierStreamBufferSize), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) // prepare the transaction stream verifer - var outChan <-chan verify.VerificationResult latest := handler.ledger.Latest() latestHdr, err := handler.ledger.BlockHdr(latest) if err != nil { @@ -110,22 +112,18 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } nbw := verify.MakeNewBlockWatcher(latestHdr) handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - outChan = verify.MakeStream(handler.ctx, handler.streamVerifierChan, + handler.streamVerifier = verify.MakeStreamVerifier(handler.ctx, handler.streamVerifierChan, handler.streamReturnChan, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) - go handler.processTxnStreamVerifiedResults(outChan) return handler } -func (handler *TxHandler) processTxnStreamVerifiedResults(outChan <-chan verify.VerificationResult) { - defer close(handler.postVerificationQueue) +// processTxnStreamVerifiedResults relays the results from the stream verifier to the postVerificationQueue +// this is needed so that the exec pool never gets blocked when it is pushing out the result +// if the postVerificationQueue is full, it will be reported to the transactionMessagesDroppedFromPool metric +func (handler *TxHandler) processTxnStreamVerifiedResults() { for { select { - case <-handler.ctx.Done(): - return - case result, ok := <-outChan: - if !ok { - return - } + case result := <-handler.streamReturnChan: txBLMsg := result.Context.(*txBacklogMsg) txBLMsg.verificationErr = result.Err select { @@ -135,6 +133,9 @@ func (handler *TxHandler) processTxnStreamVerifiedResults(outChan <-chan verify. // adding the metric here allows us to monitor how frequently it happens. transactionMessagesDroppedFromPool.Inc(nil) } + case <-handler.ctx.Done(): + return + } } } @@ -146,6 +147,8 @@ func (handler *TxHandler) Start() { }) handler.backlogWg.Add(1) go handler.backlogWorker() + go handler.processTxnStreamVerifiedResults() + handler.streamVerifier.Start() } // Stop suspends the processing of incoming messages at the transaction handler @@ -176,7 +179,6 @@ func (handler *TxHandler) backlogWorker() { return } handler.postprocessCheckedTxn(wi) - // restart the loop so that we could empty out the post verification queue. continue default: @@ -186,16 +188,17 @@ func (handler *TxHandler) backlogWorker() { select { case wi, ok := <-handler.backlogQueue: if !ok { + // this is never happening since handler.backlogQueue is never closed return } if handler.checkAlreadyCommitted(wi) { continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { + // this is never happening since handler.postVerificationQueue is never closed return } handler.postprocessCheckedTxn(wi) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index d1e1139b71..8557addf74 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -328,12 +328,6 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { select { case wi, ok := <-handler.backlogQueue: if !ok { - close(handler.streamVerifierChan) - // wait until all the pending responses are obtained. - // this is not in backlogWorker, maybe should be - for wi := range handler.postVerificationQueue { - outChan <- wi - } return } if handler.checkAlreadyCommitted(wi) { @@ -398,7 +392,7 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { randduration := time.Duration(uint64(((1 + rand.Float32()) * 3))) time.Sleep(randduration * time.Microsecond) } - close(handler.backlogQueue) + handler.Stop() wg.Wait() // Report the number of transactions dropped because the backlog was busy @@ -518,7 +512,9 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) - defer handler.ctxCancel() + // emulate handler.Start() without the backlog + go handler.processTxnStreamVerifiedResults() + handler.streamVerifier.Start() // Prepare the transactions signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) @@ -533,6 +529,13 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { groupCounter := 0 var txnCounter uint64 invalidCounter := 0 + defer func() { + if txnCounter > 0 { + b.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + b.Logf("Time/txn: %d(microsec)\n", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("processed total: [%d groups (%d invalid)] [%d txns]\n", groupCounter, invalidCounter, txnCounter) + } + }() for wi := range outChan { txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) groupCounter++ @@ -544,10 +547,11 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { invalidCounter++ require.True(b, inBad, "Error for good signature") } + if groupCounter == len(signedTransactionGroups) { + // all the benchmark txns processed + break + } } - b.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - b.Logf("Time/txn: %d(microsec)\n", uint64((time.Since(tt)/time.Microsecond))/txnCounter) - b.Logf("processed total: [%d groups (%d invalid)] [%d txns]\n", groupCounter, invalidCounter, txnCounter) }() b.ResetTimer() @@ -556,8 +560,6 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, Context: &blm} } - // shut down to end the test - close(handler.streamVerifierChan) - close(handler.backlogQueue) wg.Wait() + handler.Stop() // cancel the handler ctx } From b2267ca30d3083bb3f99aeacd075723861c48cf7 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 26 Oct 2022 00:48:01 -0400 Subject: [PATCH 069/156] lint/ comments --- data/transactions/verify/txn.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 0fb1b7c1bd..3db16916b3 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -474,6 +474,8 @@ type VerificationResult struct { Err error } +// StreamVerifier verifies txn groups received through the stxnChan channel, and returns the +// results through the resultChan type StreamVerifier struct { seatReturnChan chan interface{} resultChan chan<- VerificationResult @@ -569,7 +571,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext const waitForNextTxnDuration = 5 * time.Millisecond const waitForFirstTxnDuration = 2000 * time.Millisecond -// MakeStream creates a new stream verifier and returns the chans used to send txn groups +// MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, @@ -594,6 +596,8 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, return sv } +// Start is called when the verifier is created and whenever it needs to restart after +// the ctx is canceled func (sv *StreamVerifier) Start() { go sv.batchingLoop() } From f2787ab212a898b04b4c70e88818cf27f5bc3331 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 27 Oct 2022 00:33:18 -0400 Subject: [PATCH 070/156] fix and improve the test --- data/transactions/verify/txn.go | 19 ++++++- data/txHandler.go | 2 +- data/txHandler_test.go | 88 ++++++++++++++++++++++----------- util/metrics/counter.go | 5 ++ 4 files changed, 81 insertions(+), 33 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 3db16916b3..55bd3939e9 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -578,7 +578,9 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, cache VerifiedTransactionCache) (sv *StreamVerifier) { // limit the number of tasks queued to the execution pool - numberOfExecPoolSeats := verificationPool.GetParallelism() * 10 + // the purpose of this parameter is to create bigger batches + // instead of having many smaller batching waiting in the execution pool + numberOfExecPoolSeats := verificationPool.GetParallelism() * 2 sv = &StreamVerifier{ seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), @@ -632,7 +634,20 @@ func (sv *StreamVerifier) batchingLoop() { uel.elementList = append(uel.elementList, stx) if numberOfSigsInCurrent > txnPerWorksetThreshold { // enough transaction in the batch to efficiently verify - added = sv.processBatch(uel) + + if numberOfSigsInCurrent > 4*txnPerWorksetThreshold { + // do not consider adding more txns to this batch. + // bypass the seat count and block if the exec pool is busy + // this is to prevent creation of very large batches + err := sv.addVerificationTaskToThePool(uel, false) + // if the pool is already terminated + if err != nil { + return + } + added = true + } else { + added = sv.processBatch(uel) + } if added { numberOfSigsInCurrent = 0 uel = makeUnverifiedElementList(sv.nbw, sv.ledger) diff --git a/data/txHandler.go b/data/txHandler.go index e25a0ee4fd..4aa5df6872 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -48,7 +48,7 @@ var transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.Transact var transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) // verifierStreamBufferSize is the number of txn that coult be accumulated before the verifier stream consumes them -var verifierStreamBufferSize = 1000 +var verifierStreamBufferSize = 0 // The txBacklogMsg structure used to track a single incoming transaction from the gossip network, type txBacklogMsg struct { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 8557addf74..424c93c0cb 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "math/rand" - "strings" "sync" "testing" "time" @@ -254,18 +253,35 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } func TestIncomingTxHandle(t *testing.T) { - incomingTxHandlerProcessing(1, t) + numberOfTransactionGroups := 100 + incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) +} + +func TestIncomingTxHandleDrops(t *testing.T) { + // use smaller backlog size to test the message drops + origValue := txBacklogSize + defer func() { + txBacklogSize = origValue + }() + txBacklogSize = 10 + + numberOfTransactionGroups := 1000 + incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) } func TestIncomingTxGroupHandle(t *testing.T) { - incomingTxHandlerProcessing(proto.MaxTxGroupSize, t) + numberOfTransactionGroups := 1000 / proto.MaxTxGroupSize + incomingTxHandlerProcessing(proto.MaxTxGroupSize, numberOfTransactionGroups, t) } // incomingTxHandlerProcessing is a comprehensive transaction handling test // It handles the singed transactions by passing them to the backlog for verification -func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { +func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t *testing.T) { + // reset the counters + transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) + transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + const numUsers = 100 - numberOfTransactionGroups := 1000 log := logging.TestingLog(t) log.SetLevel(logging.Warn) addresses := make([]basics.Address, numUsers) @@ -301,6 +317,10 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + // emulate handler.Start() without the backlog + go handler.processTxnStreamVerifiedResults() + handler.streamVerifier.Start() + defer handler.ctxCancel() outChan := make(chan *txBacklogMsg, 10) @@ -310,7 +330,6 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { // through the outChan instead of passing it to postprocessCheckedTxn go func() { defer wg.Done() - defer close(outChan) for { // prioritize the postVerificationQueue select { @@ -335,7 +354,6 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { continue } handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} - case wi, ok := <-handler.postVerificationQueue: if !ok { return @@ -365,44 +383,54 @@ func incomingTxHandlerProcessing(maxGroupSize int, t *testing.T) { wg.Add(1) go func() { defer wg.Done() - groupCounter := 0 + var groupCounter uint64 txnCounter := 0 invalidCounter := 0 + var droppedBacklog, droppedPool uint64 defer func() { + t.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) + t.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) + // release the backlog worker t.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) + handler.Stop() // cancel the handler ctx }() - for wi := range outChan { - txnCounter = txnCounter + len(wi.unverifiedTxGroup) - groupCounter++ - u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) - _, inBad := badTxnGroups[u] - if wi.verificationErr == nil { - require.False(t, inBad, "No error for invalid signature") - } else { - invalidCounter++ - require.True(t, inBad, "Error for good signature") + timer := time.NewTicker(25 * time.Millisecond) + for { + select { + case wi := <-outChan: + txnCounter = txnCounter + len(wi.unverifiedTxGroup) + groupCounter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.verificationErr == nil { + require.False(t, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(t, inBad, "Error for good signature") + } + case <-timer.C: + droppedBacklog, droppedPool, err = getDropped() + require.NoError(t, err) + if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { + // all the benchmark txns processed + return + } + time.Sleep(25 * time.Millisecond) } } - t.Logf("Txn groups with invalid sigs: %d\n", invalidCounter) }() // Send the transactions to the verifier for _, tg := range encodedSignedTransactionGroups { handler.processIncomingTxn(tg) - randduration := time.Duration(uint64(((1 + rand.Float32()) * 3))) - time.Sleep(randduration * time.Microsecond) } - handler.Stop() wg.Wait() +} - // Report the number of transactions dropped because the backlog was busy - var buf strings.Builder - metrics.DefaultRegistry().WriteMetrics(&buf, "") - str := buf.String() - x := strings.Index(str, "\nalgod_transaction_messages_dropped_backlog") - str = str[x+44 : x+44+strings.Index(str[x+44:], "\n")] - str = strings.TrimSpace(strings.ReplaceAll(str, "}", " ")) - t.Logf("dropped %s txn gropus\n", str) +func getDropped() (droppedBacklog, droppedPool uint64, err error) { + droppedBacklog = transactionMessagesDroppedFromBacklog.GetValue() + droppedPool = transactionMessagesDroppedFromPool.GetValue() + return } // makeSignedTxnGroups prepares N transaction groups of random (maxGroupSize) sizes with random diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 06ea4b0c44..86df08af0c 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -111,6 +111,11 @@ func (counter *Counter) AddMicrosecondsSince(t time.Time, labels map[string]stri counter.AddUint64(uint64(time.Since(t).Microseconds()), labels) } +// GetValue returns the value of the counter. +func (counter *Counter) GetValue() (x uint64) { + return atomic.LoadUint64(&counter.intValue) +} + func (counter *Counter) fastAddUint64(x uint64) { if atomic.AddUint64(&counter.intValue, x) == x { // What we just added is the whole value, this From 4c69a2949e2ba4783b2af8b830c77b03bd34a0b3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 27 Oct 2022 16:12:40 -0400 Subject: [PATCH 071/156] error handling cleanup --- data/transactions/verify/txn.go | 58 +++++++------------- data/transactions/verify/txn_test.go | 5 ++ data/transactions/verify/verifiedTxnCache.go | 10 ++-- data/txHandler_test.go | 24 ++++---- 4 files changed, 44 insertions(+), 53 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 55bd3939e9..48b22eafd9 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -562,13 +562,12 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext } -// wait time for another txn should satisfy the following inequality: -// [validation time added to the group by one more txn] + [wait time] <= [validation time of a single txn] -// since these are difficult to estimate, the simplified version could be to assume: -// [validation time added to the group by one more txn] = [validation time of a single txn] / 2 -// This gives us: -// [wait time] <= [validation time of a single txn] / 2 -const waitForNextTxnDuration = 5 * time.Millisecond +// waitForNextTxnDuration is the time to wait before sending the batch for evaluation. +// this does not mean that every batch will wait this long. when the load level is high, +// the batch will fill up quickly and will be sent for evaluation before this timeout +const waitForNextTxnDuration = 1 * time.Millisecond + +// waitForFirstTxnDuration is the time to wait for the first transaction in the batch const waitForFirstTxnDuration = 2000 * time.Millisecond // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups @@ -615,6 +614,7 @@ func (sv *StreamVerifier) batchingLoop() { isFirstInBatch := numberOfSigsInCurrent == 0 numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) if err != nil { + // wrong number of signatures sv.sendResult(stx.TxnGroup, stx.Context, err) continue } @@ -624,9 +624,9 @@ func (sv *StreamVerifier) batchingLoop() { // no point in blocking and waiting for a seat here, let it wait in the exec pool err := sv.addVerificationTaskToThePool(makeSingleUnverifiedElement(sv.nbw, sv.ledger, stx), false) if err != nil { - sv.sendResult(stx.TxnGroup, stx.Context, err) + continue // pool is terminated. nothing to report to the txn sender } - continue + continue // stx is handled, continue } // add this txngrp to the list of batchable txn groups @@ -640,9 +640,8 @@ func (sv *StreamVerifier) batchingLoop() { // bypass the seat count and block if the exec pool is busy // this is to prevent creation of very large batches err := sv.addVerificationTaskToThePool(uel, false) - // if the pool is already terminated if err != nil { - return + continue // pool is terminated. } added = true } else { @@ -684,9 +683,8 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfSigsInCurrent > 0 { // send the current accumulated transactions to the pool err := sv.addVerificationTaskToThePool(uel, false) - // if the pool is already terminated if err != nil { - return + continue // pool is terminated. } } // wait for the pending tasks, then close the result chan @@ -703,19 +701,9 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veCont Err: err, } // send the txn result out the pipe + // this should never block. the receiver end of this channel will drop transactions if the + // postVerificationQueue is blocked sv.resultChan <- vr - /* - select { - case sv.resultChan <- vr: - - // if the channel is not accepting, should not block here - // report dropped txn. caching is fine, if it comes back in the block - default: - fmt.Println("skipped!!") - //TODO: report this - - } - */ } func (sv *StreamVerifier) processBatch(uel unverifiedElementList) (added bool) { @@ -743,18 +731,18 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList defer sv.pendingTasksWg.Done() if sv.ctx.Err() != nil { - return sv.ctx.Err() + return struct{}{} } uel := arg.(*unverifiedElementList) batchVerifier := crypto.MakeBatchVerifier() bl := makeBatchLoad() - // TODO: separate operations here, and get the sig verification inside LogicSig outside + // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here for _, ue := range uel.elementList { groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) if err != nil { - // verification failed, cannot go to the batch. return here. + // verification failed, no need to add the sig to the batch, report the error sv.sendResult(ue.TxnGroup, ue.Context, err) continue } @@ -763,11 +751,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList } failed, err := batchVerifier.VerifyWithFeedback() - if err != nil && err != crypto.ErrBatchHasFailedSigs { - fmt.Println(err) - // something bad happened - // TODO: report error and discard the batch - } + // this error can only be crypto.ErrBatchHasFailedSigs verifiedTxnGroups := make([][]transactions.SignedTxn, len(bl.txnGroups)) verifiedGroupCtxs := make([]*GroupContext, len(bl.groupCtxs)) @@ -796,13 +780,11 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList sv.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) } // loading them all at once by locking the cache once - err = sv.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) - if err != nil { - // TODO: handle the error - fmt.Println(err) - } + sv.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) return struct{}{} } + + // if the task has an allocated seat, should release it var retChan chan interface{} if returnSeat { retChan = sv.seatReturnChan diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index c26318e395..8afd729b7f 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -337,6 +337,11 @@ func TestDecodeNil(t *testing.T) { func TestPaysetGroups(t *testing.T) { partitiontest.PartitionTest(t) + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + _, signedTxn, secrets, addrs := generateTestObjects(10000, 20, 50) blkHdr := createDummyBlockHeader() diff --git a/data/transactions/verify/verifiedTxnCache.go b/data/transactions/verify/verifiedTxnCache.go index 82f0e9772e..e29b92bd7b 100644 --- a/data/transactions/verify/verifiedTxnCache.go +++ b/data/transactions/verify/verifiedTxnCache.go @@ -59,7 +59,7 @@ type VerifiedTransactionCache interface { // in the cache, the new entry overrides the old one. Add(txgroup []transactions.SignedTxn, groupCtx *GroupContext) // AddPayset works in a similar way to Add, but is intended for adding an array of transaction groups, along with their corresponding contexts. - AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) error + AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) // GetUnverifiedTransactionGroups compares the provided payset against the currently cached transactions and figure which transaction groups aren't fully cached. GetUnverifiedTransactionGroups(payset [][]transactions.SignedTxn, CurrSpecAddrs transactions.SpecialAddresses, CurrProto protocol.ConsensusVersion) [][]transactions.SignedTxn // UpdatePinned replaces the pinned entries with the one provided in the pinnedTxns map. This is typically expected to be a subset of the @@ -106,13 +106,13 @@ func (v *verifiedTransactionCache) Add(txgroup []transactions.SignedTxn, groupCt } // AddPayset works in a similar way to Add, but is intended for adding an array of transaction groups, along with their corresponding contexts. -func (v *verifiedTransactionCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) error { +func (v *verifiedTransactionCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) { v.bucketsLock.Lock() defer v.bucketsLock.Unlock() for i := range txgroup { v.add(txgroup[i], groupCtxs[i]) } - return nil + return } // GetUnverifiedTransactionGroups compares the provided payset against the currently cached transactions and figure which transaction groups aren't fully cached. @@ -268,8 +268,8 @@ func (v *mockedCache) Add(txgroup []transactions.SignedTxn, groupCtx *GroupConte return } -func (v *mockedCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) error { - return nil +func (v *mockedCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) { + return } func (v *mockedCache) GetUnverifiedTransactionGroups(txnGroups [][]transactions.SignedTxn, currSpecAddrs transactions.SpecialAddresses, currProto protocol.ConsensusVersion) (unverifiedGroups [][]transactions.SignedTxn) { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 424c93c0cb..a91dc738f0 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -136,8 +136,8 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { type vtCache struct{} func (vtCache) Add(txgroup []transactions.SignedTxn, groupCtx *verify.GroupContext) {} -func (vtCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*verify.GroupContext) error { - return nil +func (vtCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*verify.GroupContext) { + return } func (vtCache) GetUnverifiedTransactionGroups(payset [][]transactions.SignedTxn, CurrSpecAddrs transactions.SpecialAddresses, CurrProto protocol.ConsensusVersion) [][]transactions.SignedTxn { return nil @@ -252,11 +252,19 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { } } +// TestIncomingTxHandle checks the correctness with single txns func TestIncomingTxHandle(t *testing.T) { - numberOfTransactionGroups := 100 + numberOfTransactionGroups := 1000 incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) } +// TestIncomingTxHandle checks the correctness with txn groups +func TestIncomingTxGroupHandle(t *testing.T) { + numberOfTransactionGroups := 1000 / proto.MaxTxGroupSize + incomingTxHandlerProcessing(proto.MaxTxGroupSize, numberOfTransactionGroups, t) +} + +// TestIncomingTxHandleDrops accounts for the dropped txns when the verifier/exec pool is saturated func TestIncomingTxHandleDrops(t *testing.T) { // use smaller backlog size to test the message drops origValue := txBacklogSize @@ -269,11 +277,6 @@ func TestIncomingTxHandleDrops(t *testing.T) { incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) } -func TestIncomingTxGroupHandle(t *testing.T) { - numberOfTransactionGroups := 1000 / proto.MaxTxGroupSize - incomingTxHandlerProcessing(proto.MaxTxGroupSize, numberOfTransactionGroups, t) -} - // incomingTxHandlerProcessing is a comprehensive transaction handling test // It handles the singed transactions by passing them to the backlog for verification func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t *testing.T) { @@ -394,7 +397,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t t.Logf("processed %d txn groups (%d txns)\n", groupCounter, txnCounter) handler.Stop() // cancel the handler ctx }() - timer := time.NewTicker(25 * time.Millisecond) + timer := time.NewTicker(250 * time.Millisecond) for { select { case wi := <-outChan: @@ -415,7 +418,8 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t // all the benchmark txns processed return } - time.Sleep(25 * time.Millisecond) + time.Sleep(250 * time.Millisecond) + timer.Reset(250 * time.Millisecond) } } }() From 06ed13ffa7a011c747e78ff028dde3aae8e9d628 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 27 Oct 2022 17:07:04 -0400 Subject: [PATCH 072/156] lint fix --- data/transactions/verify/verifiedTxnCache.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/transactions/verify/verifiedTxnCache.go b/data/transactions/verify/verifiedTxnCache.go index e29b92bd7b..58ce23ae1f 100644 --- a/data/transactions/verify/verifiedTxnCache.go +++ b/data/transactions/verify/verifiedTxnCache.go @@ -112,7 +112,6 @@ func (v *verifiedTransactionCache) AddPayset(txgroup [][]transactions.SignedTxn, for i := range txgroup { v.add(txgroup[i], groupCtxs[i]) } - return } // GetUnverifiedTransactionGroups compares the provided payset against the currently cached transactions and figure which transaction groups aren't fully cached. @@ -268,8 +267,7 @@ func (v *mockedCache) Add(txgroup []transactions.SignedTxn, groupCtx *GroupConte return } -func (v *mockedCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) { - return +func (v *mockedCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*GroupContext) { } func (v *mockedCache) GetUnverifiedTransactionGroups(txnGroups [][]transactions.SignedTxn, currSpecAddrs transactions.SpecialAddresses, currProto protocol.ConsensusVersion) (unverifiedGroups [][]transactions.SignedTxn) { From c2108e9a0ccfa67ef737545e639b4956a306d59b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 27 Oct 2022 18:48:09 -0400 Subject: [PATCH 073/156] gofmt --- data/txHandler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index a91dc738f0..f6616394ab 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -137,7 +137,7 @@ type vtCache struct{} func (vtCache) Add(txgroup []transactions.SignedTxn, groupCtx *verify.GroupContext) {} func (vtCache) AddPayset(txgroup [][]transactions.SignedTxn, groupCtxs []*verify.GroupContext) { - return + return } func (vtCache) GetUnverifiedTransactionGroups(payset [][]transactions.SignedTxn, CurrSpecAddrs transactions.SpecialAddresses, CurrProto protocol.ConsensusVersion) [][]transactions.SignedTxn { return nil From d0a6b3dc9cd336fa5963be014afc82eebff953a3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 31 Oct 2022 19:38:30 -0400 Subject: [PATCH 074/156] fix caching bug, add caching to the test --- data/transactions/verify/txn.go | 4 ++-- data/transactions/verify/txn_test.go | 32 +++++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 48b22eafd9..3ce0aa49d6 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -753,8 +753,8 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList failed, err := batchVerifier.VerifyWithFeedback() // this error can only be crypto.ErrBatchHasFailedSigs - verifiedTxnGroups := make([][]transactions.SignedTxn, len(bl.txnGroups)) - verifiedGroupCtxs := make([]*GroupContext, len(bl.groupCtxs)) + verifiedTxnGroups := make([][]transactions.SignedTxn, 0, len(bl.txnGroups)) + verifiedGroupCtxs := make([]*GroupContext, 0, len(bl.groupCtxs)) failedSigIdx := 0 for txgIdx := range bl.txnGroups { txGroupSigFailed := false diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 8afd729b7f..02f0b8deb5 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "math/rand" + "sync" "testing" "time" @@ -850,7 +851,7 @@ func BenchmarkTxn(b *testing.B) { func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxns := 10000 + numOfTxns := 4000 _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) blkHdr := createDummyBlockHeader() @@ -882,8 +883,14 @@ func TestStreamVerifier(t *testing.T) { } } + wg := sync.WaitGroup{} + + var badSigResultCounter int + var goodSigResultCounter int errChan := make(chan error) + wg.Add(1) go func() { + defer wg.Done() defer close(errChan) // process the results for x := 0; x < numOfTxnGroups; x++ { @@ -891,12 +898,13 @@ func TestStreamVerifier(t *testing.T) { case result := <-resultChan: if _, has := badTxnGroups[result.TxnGroup[0].Sig]; has { - delete(badTxnGroups, result.TxnGroup[0].Sig) + badSigResultCounter++ if result.Err == nil { err := fmt.Errorf("%dth transaction varified with a bad sig", x) errChan <- err } } else { + goodSigResultCounter++ if result.Err != nil { err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) errChan <- err @@ -908,8 +916,10 @@ func TestStreamVerifier(t *testing.T) { } }() + wg.Add(1) // send txn groups to be verified go func() { + defer wg.Done() for _, tg := range txnGroups { select { case <-ctx.Done(): @@ -924,5 +934,21 @@ func TestStreamVerifier(t *testing.T) { require.NoError(t, err) } - require.Equal(t, 0, len(badTxnGroups)) + wg.Wait() + + // check if all txns have been checked. + require.Equal(t, len(txnGroups), badSigResultCounter+goodSigResultCounter) + require.Equal(t, len(badTxnGroups), badSigResultCounter) + + // check the cached transactions + // note that the result of each verified txn group is send before the batch is added to the cache + // the test does not know if the batch is not added to the cache yet, so some elts might be missing from the cache + unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) + require.GreaterOrEqual(t, len(unverifiedGroups), badSigResultCounter) + for _, txn := range unverifiedGroups { + if _, has := badTxnGroups[txn[0].Sig]; has { + delete(badTxnGroups, txn[0].Sig) + } + } + require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } From 0587ec59f104c860d194e3d1f6949aa39a9d825c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 1 Nov 2022 01:23:20 -0400 Subject: [PATCH 075/156] add test for different txn cases --- data/transactions/verify/txn_test.go | 188 ++++++++++++++++-- .../verify/verifiedTxnCache_test.go | 10 +- data/txHandler_test.go | 8 +- 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 02f0b8deb5..fd2045b805 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -18,6 +18,7 @@ package verify import ( "context" + "encoding/binary" "fmt" "math/rand" "sync" @@ -48,6 +49,7 @@ var blockHeader = bookkeeping.BlockHeader{ CurrentProtocol: protocol.ConsensusCurrentVersion, }, } +var protoMaxGroupSize = config.Consensus[protocol.ConsensusCurrentVersion].MaxTxGroupSize var spec = transactions.SpecialAddresses{ FeeSink: feeSink, @@ -350,7 +352,7 @@ func TestPaysetGroups(t *testing.T) { verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) defer verificationPool.Shutdown() - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) startPaysetGroupsTime := time.Now() err := PaysetGroups(context.Background(), txnGroups, blkHdr, verificationPool, MakeVerifiedTransactionCache(50000), nil) @@ -372,7 +374,7 @@ func TestPaysetGroups(t *testing.T) { _, signedTxn, secrets, addrs = generateTestObjects(txnCount, 20, 50) - txnGroups = generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups = generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) ctx, ctxCancelFunc := context.WithTimeout(context.Background(), 1500*time.Millisecond) defer ctxCancelFunc() @@ -420,7 +422,7 @@ func BenchmarkPaysetGroups(b *testing.B) { verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, b) defer verificationPool.Shutdown() - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) cache := MakeVerifiedTransactionCache(50000) b.ResetTimer() @@ -443,7 +445,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= ==`) require.NoError(t, err) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) dummyLedger := DummyLedgerForSignature{} _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) @@ -505,30 +507,39 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= } -func generateTransactionGroups(signedTxns []transactions.SignedTxn, secrets []*crypto.SignatureSecrets, addrs []basics.Address) [][]transactions.SignedTxn { +func generateTransactionGroups(maxGroupSize int, signedTxns []transactions.SignedTxn, + secrets []*crypto.SignatureSecrets, addrs []basics.Address) [][]transactions.SignedTxn { addrToSecret := make(map[basics.Address]*crypto.SignatureSecrets) for i, addr := range addrs { addrToSecret[addr] = secrets[i] } txnGroups := make([][]transactions.SignedTxn, 0, len(signedTxns)) - for i := 0; i < len(signedTxns); i++ { - txnPerGroup := 1 + rand.Intn(15) - if i+txnPerGroup >= len(signedTxns) { - txnPerGroup = len(signedTxns) - i - 1 + for i := 0; i < len(signedTxns); { + txnsInGroup := rand.Intn(protoMaxGroupSize-1) + 1 + if txnsInGroup > maxGroupSize { + txnsInGroup = maxGroupSize } - newGroup := signedTxns[i : i+txnPerGroup+1] + if i+txnsInGroup > len(signedTxns) { + txnsInGroup = len(signedTxns) - i + } + + newGroup := signedTxns[i : i+txnsInGroup] var txGroup transactions.TxGroup - for _, txn := range newGroup { - txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.HashObj(txn.Txn)) + if txnsInGroup > 1 { + for _, txn := range newGroup { + txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.HashObj(txn.Txn)) + } } groupHash := crypto.HashObj(txGroup) for j := range newGroup { - newGroup[j].Txn.Group = groupHash + if txnsInGroup > 1 { + newGroup[j].Txn.Group = groupHash + } newGroup[j].Sig = addrToSecret[newGroup[j].Txn.Sender].Sign(&newGroup[j].Txn) } txnGroups = append(txnGroups, newGroup) - i += txnPerGroup + i += txnsInGroup } return txnGroups @@ -540,7 +551,7 @@ func TestTxnGroupCacheUpdate(t *testing.T) { _, signedTxn, secrets, addrs := generateTestObjects(100, 20, 50) blkHdr := createDummyBlockHeader() - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) breakSignatureFunc := func(txn *transactions.SignedTxn) { txn.Sig[0]++ } @@ -834,7 +845,7 @@ func BenchmarkTxn(b *testing.B) { } _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 50) blk := bookkeeping.Block{BlockHeader: createDummyBlockHeader()} - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) b.ResetTimer() for _, txnGroup := range txnGroups { @@ -859,7 +870,7 @@ func TestStreamVerifier(t *testing.T) { verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) defer verificationPool.Shutdown() - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) numOfTxnGroups := len(txnGroups) ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) @@ -952,3 +963,146 @@ func TestStreamVerifier(t *testing.T) { } require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } + +func TestStreamVerifierCases(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfTxns := 10 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + blkHdr := createDummyBlockHeader() + + execPool := execpool.MakePool(t) + verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + defer verificationPool.Shutdown() + + txnGroups := generateTransactionGroups(1, signedTxn, secrets, addrs) + numOfTxnGroups := len(txnGroups) + ctx, cancel := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50) + + defer cancel() + + nbw := MakeNewBlockWatcher(blkHdr) + stxnChan := make(chan UnverifiedElement) + resultChan := make(chan VerificationResult) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, nil, nbw, verificationPool, cache) + sv.Start() + + badTxnGroups := make(map[uint64]struct{}) + + mod := 1 + + // txn with 0 sigs + txnGroups[mod][0].Sig = crypto.Signature{} + { + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(mod)) + txnGroups[mod][0].Txn.Note = noteField + badTxnGroups[uint64(mod)] = struct{}{} + } + mod++ + + // invalid stateproof txn + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Type = protocol.StateProofTx + txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender + { + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(mod)) + txnGroups[mod][0].Txn.Note = noteField + badTxnGroups[uint64(mod)] = struct{}{} + } + mod++ + + // acceptable stateproof txn + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Type = protocol.StateProofTx + txnGroups[mod][0].Txn.Header.Fee = basics.MicroAlgos{Raw: 0} + txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender + txnGroups[mod][0].Txn.PaymentTxnFields = transactions.PaymentTxnFields{} + mod++ + + // multisig + _, mSigTxn, _, _ := generateMultiSigTxn(1, 6, 50, t) + txnGroups[mod] = mSigTxn + mod++ + + // txn with sig and msig + txnGroups[mod][0].Msig = mSigTxn[0].Msig + { + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(mod)) + txnGroups[mod][0].Txn.Note = noteField + badTxnGroups[uint64(mod)] = struct{}{} + } + + wg := sync.WaitGroup{} + + var badSigResultCounter int + var goodSigResultCounter int + errChan := make(chan error) + wg.Add(1) + go func() { + defer wg.Done() + defer close(errChan) + // process the results + for x := 0; x < numOfTxnGroups; x++ { + select { + case result := <-resultChan: + u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) + if _, has := badTxnGroups[u]; has { + badSigResultCounter++ + if result.Err == nil { + err := fmt.Errorf("%dth transaction varified with a bad sig", x) + errChan <- err + } + } else { + goodSigResultCounter++ + if result.Err != nil { + err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) + errChan <- err + } + } + case <-ctx.Done(): + break + } + } + }() + + wg.Add(1) + // send txn groups to be verified + go func() { + defer wg.Done() + for _, tg := range txnGroups { + select { + case <-ctx.Done(): + break + default: + stxnChan <- UnverifiedElement{TxnGroup: tg, Context: nil} + } + } + }() + + for err := range errChan { + require.NoError(t, err) + } + + wg.Wait() + + // check if all txns have been checked. + require.Equal(t, len(txnGroups), badSigResultCounter+goodSigResultCounter) + require.Equal(t, len(badTxnGroups), badSigResultCounter) + + // check the cached transactions + // note that the result of each verified txn group is send before the batch is added to the cache + // the test does not know if the batch is not added to the cache yet, so some elts might be missing from the cache + unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) + require.GreaterOrEqual(t, len(unverifiedGroups), badSigResultCounter) + for _, txn := range unverifiedGroups { + u, _ := binary.Uvarint(txn[0].Txn.Note) + if _, has := badTxnGroups[u]; has { + delete(badTxnGroups, u) + } + } + require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") +} diff --git a/data/transactions/verify/verifiedTxnCache_test.go b/data/transactions/verify/verifiedTxnCache_test.go index e3001db674..2070f537aa 100644 --- a/data/transactions/verify/verifiedTxnCache_test.go +++ b/data/transactions/verify/verifiedTxnCache_test.go @@ -33,7 +33,7 @@ func TestAddingToCache(t *testing.T) { icache := MakeVerifiedTransactionCache(500) impl := icache.(*verifiedTransactionCache) _, signedTxn, secrets, addrs := generateTestObjects(10, 5, 50) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) groupCtx, err := PrepareGroupContext(txnGroups[0], blockHeader, nil) require.NoError(t, err) impl.Add(txnGroups[0], groupCtx) @@ -83,7 +83,7 @@ func TestGetUnverifiedTransactionGroups50(t *testing.T) { icache := MakeVerifiedTransactionCache(size * 2) impl := icache.(*verifiedTransactionCache) _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10+size/1000, 0) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) expectedUnverifiedGroups := make([][]transactions.SignedTxn, 0, len(txnGroups)/2) // add every even transaction to the cache. @@ -108,7 +108,7 @@ func BenchmarkGetUnverifiedTransactionGroups50(b *testing.B) { icache := MakeVerifiedTransactionCache(b.N * 2) impl := icache.(*verifiedTransactionCache) _, signedTxn, secrets, addrs := generateTestObjects(b.N*2, 10+b.N/1000, 0) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) queryTxnGroups := make([][]transactions.SignedTxn, 0, b.N) // add every even transaction to the cache. @@ -141,7 +141,7 @@ func TestUpdatePinned(t *testing.T) { icache := MakeVerifiedTransactionCache(size * 10) impl := icache.(*verifiedTransactionCache) _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) // insert some entries. for i := 0; i < len(txnGroups); i++ { @@ -170,7 +170,7 @@ func TestPinningTransactions(t *testing.T) { icache := MakeVerifiedTransactionCache(size) impl := icache.(*verifiedTransactionCache) _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0) - txnGroups := generateTransactionGroups(signedTxn, secrets, addrs) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) // insert half of the entries. for i := 0; i < len(txnGroups)/2; i++ { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index f6616394ab..8846a7ad8a 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -472,13 +472,17 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, add Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, }, } - txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) + if grpSize > 1 { + txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) + } txns = append(txns, tx) } groupHash := crypto.HashObj(txGroup) signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) for g, txn := range txns { - txn.Group = groupHash + if grpSize > 1 { + txn.Group = groupHash + } signedTx := txn.Sign(secrets[(u+g)%numUsers]) signedTx.Txn = txn signedTxGroup = append(signedTxGroup, signedTx) From ccf30dc940073b1e7fdbb53ee821fa3ac5fdc72b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 1 Nov 2022 08:33:40 -0400 Subject: [PATCH 076/156] revert whitespace change --- ledger/internal/eval.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ledger/internal/eval.go b/ledger/internal/eval.go index 3f02eb5e10..f2750d8a08 100644 --- a/ledger/internal/eval.go +++ b/ledger/internal/eval.go @@ -850,6 +850,7 @@ func (eval *BlockEvaluator) TestTransactionGroup(txgroup []transactions.SignedTx if !txn.Txn.Group.IsZero() { txWithoutGroup := txn.Txn txWithoutGroup.Group = crypto.Digest{} + group.TxGroupHashes = append(group.TxGroupHashes, crypto.Digest(txWithoutGroup.ID())) } else if len(txgroup) > 1 { return fmt.Errorf("transactionGroup: [%d] had zero Group but was submitted in a group of %d", gi, len(txgroup)) From 2182ae935131bc36c731ae1c28d4e1c4da82d209 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 1 Nov 2022 13:53:08 -0400 Subject: [PATCH 077/156] CR comments --- data/transactions/verify/txn.go | 2 +- data/txHandler.go | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 3ce0aa49d6..ae3f8082fc 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -457,7 +457,7 @@ func (w *worksetBuilder) completed() bool { return w.idx >= len(w.payset) } -// UnverifiedElement is the element passed the Stream verifier +// UnverifiedElement is the element passed to the Stream verifier // Context is a reference associated with the txn group which is passed // with the result type UnverifiedElement struct { diff --git a/data/txHandler.go b/data/txHandler.go index 4aa5df6872..1e5e53d8b8 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -47,9 +47,6 @@ var transactionMessagesHandled = metrics.MakeCounter(metrics.TransactionMessages var transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) var transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) -// verifierStreamBufferSize is the number of txn that coult be accumulated before the verifier stream consumes them -var verifierStreamBufferSize = 0 - // The txBacklogMsg structure used to track a single incoming transaction from the gossip network, type txBacklogMsg struct { rawmsg *network.IncomingMessage // the raw message from the network @@ -97,8 +94,8 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go backlogQueue: make(chan *txBacklogMsg, txBacklogSize), postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, - streamVerifierChan: make(chan verify.UnverifiedElement, verifierStreamBufferSize), - streamReturnChan: make(chan verify.VerificationResult, verifierStreamBufferSize), + streamVerifierChan: make(chan verify.UnverifiedElement, 0), + streamReturnChan: make(chan verify.VerificationResult, 0), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) From 6ac8f50553f0e382863b5caf78f99bc82560d5b3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 1 Nov 2022 23:31:02 -0400 Subject: [PATCH 078/156] add logicsig test --- data/transactions/verify/txn_test.go | 39 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index fd2045b805..c1859d3b02 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -985,7 +985,7 @@ func TestStreamVerifierCases(t *testing.T) { nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, nil, nbw, verificationPool, cache) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) sv.Start() badTxnGroups := make(map[uint64]struct{}) @@ -1027,6 +1027,39 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod] = mSigTxn mod++ + // logicsig + // add a simple logic that verifies this condition: + // sha256(arg0) == base64decode(5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E=) + op, err := logic.AssembleString(`arg 0 +sha256 +byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= +==`) + require.NoError(t, err) + s := rand.Intn(len(secrets)) + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Sender = addrs[s] + txnGroups[mod][0].Lsig.Args = [][]byte{[]byte("=0\x97S\x85H\xe9\x91B\xfd\xdb;1\xf5Z\xaec?\xae\xf2I\x93\x08\x12\x94\xaa~\x06\x08\x849b")} + txnGroups[mod][0].Lsig.Logic = op.Program + program := logic.Program(op.Program) + txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) + mod++ + + // bad lgicsig + s = rand.Intn(len(secrets)) + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Sender = addrs[s] + txnGroups[mod][0].Lsig.Args = [][]byte{[]byte("=0\x97S\x85H\xe9\x91B\xfd\xdb;1\xf5Z\xaec?\xae\xf2I\x93\x08\x12\x94\xaa~\x06\x08\x849b")} + txnGroups[mod][0].Lsig.Args[0][0]++ + txnGroups[mod][0].Lsig.Logic = op.Program + txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) + { + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(mod)) + txnGroups[mod][0].Txn.Note = noteField + badTxnGroups[uint64(mod)] = struct{}{} + } + mod++ + // txn with sig and msig txnGroups[mod][0].Msig = mSigTxn[0].Msig { @@ -1053,13 +1086,13 @@ func TestStreamVerifierCases(t *testing.T) { if _, has := badTxnGroups[u]; has { badSigResultCounter++ if result.Err == nil { - err := fmt.Errorf("%dth transaction varified with a bad sig", x) + err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) errChan <- err } } else { goodSigResultCounter++ if result.Err != nil { - err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) + err := fmt.Errorf("%dth (%d) transaction failed to varify with good sigs", x, u) errChan <- err } } From fbd693a142fba27c92a75e0178adca3c7217f693 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 2 Nov 2022 00:59:42 -0400 Subject: [PATCH 079/156] refactor the tests --- data/transactions/verify/txn.go | 2 +- data/transactions/verify/txn_test.go | 198 ++++++++++----------------- 2 files changed, 70 insertions(+), 130 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index ae3f8082fc..5a0f1d021b 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -568,7 +568,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext const waitForNextTxnDuration = 1 * time.Millisecond // waitForFirstTxnDuration is the time to wait for the first transaction in the batch -const waitForFirstTxnDuration = 2000 * time.Millisecond +var waitForFirstTxnDuration = 2000 * time.Millisecond // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index c1859d3b02..42760dc874 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -174,6 +174,10 @@ func generateTestObjects(numTxs, numAccs int, blockRound basics.Round) ([]transa } txs[i] = createPayTransaction(f, iss, exp, a, addresses[s], addresses[r]) + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(i)) + txs[i].Note = noteField + signed[i] = txs[i].Sign(secrets[s]) u += 100 } @@ -859,41 +863,25 @@ func BenchmarkTxn(b *testing.B) { b.StopTimer() } -func TestStreamVerifier(t *testing.T) { - partitiontest.PartitionTest(t) - - numOfTxns := 4000 - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - blkHdr := createDummyBlockHeader() +func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, t *testing.T) { + numOfTxnGroups := len(txnGroups) execPool := execpool.MakePool(t) verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) defer verificationPool.Shutdown() - txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) - numOfTxnGroups := len(txnGroups) ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) defer cancel() + blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, nil, nbw, verificationPool, cache) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) sv.Start() - badTxnGroups := make(map[crypto.Signature]struct{}) - - for tgi := range txnGroups { - if rand.Float32() > 0.7 { - // make a bad sig - t := rand.Intn(len(txnGroups[tgi])) - txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 - badTxnGroups[txnGroups[tgi][0].Sig] = struct{}{} - } - } - wg := sync.WaitGroup{} var badSigResultCounter int @@ -907,17 +895,17 @@ func TestStreamVerifier(t *testing.T) { for x := 0; x < numOfTxnGroups; x++ { select { case result := <-resultChan: - - if _, has := badTxnGroups[result.TxnGroup[0].Sig]; has { + u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) + if _, has := badTxnGroups[u]; has { badSigResultCounter++ if result.Err == nil { - err := fmt.Errorf("%dth transaction varified with a bad sig", x) + err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) errChan <- err } } else { goodSigResultCounter++ if result.Err != nil { - err := fmt.Errorf("%dth transaction failed to varify with good sigs", x) + err := fmt.Errorf("%dth (%d) transaction failed to varify with good sigs", x, u) errChan <- err } } @@ -957,65 +945,65 @@ func TestStreamVerifier(t *testing.T) { unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) require.GreaterOrEqual(t, len(unverifiedGroups), badSigResultCounter) for _, txn := range unverifiedGroups { - if _, has := badTxnGroups[txn[0].Sig]; has { - delete(badTxnGroups, txn[0].Sig) + u, _ := binary.Uvarint(txn[0].Txn.Note) + if _, has := badTxnGroups[u]; has { + delete(badTxnGroups, u) } } require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } -func TestStreamVerifierCases(t *testing.T) { +// TestStreamVerifier tests the basic functionality +func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxns := 10 + numOfTxns := 4000 _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - blkHdr := createDummyBlockHeader() + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) - execPool := execpool.MakePool(t) - verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) - defer verificationPool.Shutdown() + badTxnGroups := make(map[uint64]struct{}) - txnGroups := generateTransactionGroups(1, signedTxn, secrets, addrs) - numOfTxnGroups := len(txnGroups) - ctx, cancel := context.WithCancel(context.Background()) - cache := MakeVerifiedTransactionCache(50) + for tgi := range txnGroups { + if rand.Float32() > 0.7 { + // make a bad sig + t := rand.Intn(len(txnGroups[tgi])) + txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 + u, _ := binary.Uvarint(txnGroups[tgi][0].Txn.Note) + badTxnGroups[u] = struct{}{} + } + } + streamVerifier(txnGroups, badTxnGroups, t) +} - defer cancel() +// TestStreamVerifierCases tests various valid and invalid transaction signature cases +func TestStreamVerifierCases(t *testing.T) { + partitiontest.PartitionTest(t) - nbw := MakeNewBlockWatcher(blkHdr) - stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) - sv.Start() + numOfTxns := 10 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups := generateTransactionGroups(1, signedTxn, secrets, addrs) badTxnGroups := make(map[uint64]struct{}) mod := 1 // txn with 0 sigs txnGroups[mod][0].Sig = crypto.Signature{} - { - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(mod)) - txnGroups[mod][0].Txn.Note = noteField - badTxnGroups[uint64(mod)] = struct{}{} - } + u, _ := binary.Uvarint(txnGroups[mod][0].Txn.Note) + badTxnGroups[u] = struct{}{} mod++ // invalid stateproof txn txnGroups[mod][0].Sig = crypto.Signature{} txnGroups[mod][0].Txn.Type = protocol.StateProofTx txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender - { - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(mod)) - txnGroups[mod][0].Txn.Note = noteField - badTxnGroups[uint64(mod)] = struct{}{} - } + u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) + badTxnGroups[u] = struct{}{} mod++ // acceptable stateproof txn txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Note = nil txnGroups[mod][0].Txn.Type = protocol.StateProofTx txnGroups[mod][0].Txn.Header.Fee = basics.MicroAlgos{Raw: 0} txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender @@ -1052,90 +1040,42 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Lsig.Args[0][0]++ txnGroups[mod][0].Lsig.Logic = op.Program txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) - { - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(mod)) - txnGroups[mod][0].Txn.Note = noteField - badTxnGroups[uint64(mod)] = struct{}{} - } + u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) + badTxnGroups[u] = struct{}{} mod++ // txn with sig and msig txnGroups[mod][0].Msig = mSigTxn[0].Msig - { - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(mod)) - txnGroups[mod][0].Txn.Note = noteField - badTxnGroups[uint64(mod)] = struct{}{} - } - - wg := sync.WaitGroup{} + u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) + badTxnGroups[u] = struct{}{} - var badSigResultCounter int - var goodSigResultCounter int - errChan := make(chan error) - wg.Add(1) - go func() { - defer wg.Done() - defer close(errChan) - // process the results - for x := 0; x < numOfTxnGroups; x++ { - select { - case result := <-resultChan: - u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) - if _, has := badTxnGroups[u]; has { - badSigResultCounter++ - if result.Err == nil { - err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) - errChan <- err - } - } else { - goodSigResultCounter++ - if result.Err != nil { - err := fmt.Errorf("%dth (%d) transaction failed to varify with good sigs", x, u) - errChan <- err - } - } - case <-ctx.Done(): - break - } - } - }() - - wg.Add(1) - // send txn groups to be verified - go func() { - defer wg.Done() - for _, tg := range txnGroups { - select { - case <-ctx.Done(): - break - default: - stxnChan <- UnverifiedElement{TxnGroup: tg, Context: nil} - } - } - }() + streamVerifier(txnGroups, badTxnGroups, t) +} - for err := range errChan { - require.NoError(t, err) - } +// TestStreamVerifierIdel starts the verifer and sends nothing, to trigger the timer, then sends a txn +func TestStreamVerifierIdel(t *testing.T) { + partitiontest.PartitionTest(t) - wg.Wait() + numOfTxns := 10 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) - // check if all txns have been checked. - require.Equal(t, len(txnGroups), badSigResultCounter+goodSigResultCounter) - require.Equal(t, len(badTxnGroups), badSigResultCounter) + badTxnGroups := make(map[uint64]struct{}) - // check the cached transactions - // note that the result of each verified txn group is send before the batch is added to the cache - // the test does not know if the batch is not added to the cache yet, so some elts might be missing from the cache - unverifiedGroups := cache.GetUnverifiedTransactionGroups(txnGroups, spec, protocol.ConsensusCurrentVersion) - require.GreaterOrEqual(t, len(unverifiedGroups), badSigResultCounter) - for _, txn := range unverifiedGroups { - u, _ := binary.Uvarint(txn[0].Txn.Note) - if _, has := badTxnGroups[u]; has { - delete(badTxnGroups, u) + for tgi := range txnGroups { + if rand.Float32() > 0.5 { + // make a bad sig + t := rand.Intn(len(txnGroups[tgi])) + txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 + u, _ := binary.Uvarint(txnGroups[tgi][0].Txn.Note) + badTxnGroups[u] = struct{}{} } } - require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") + origValue := waitForFirstTxnDuration + defer func() { + waitForFirstTxnDuration = origValue + }() + // set this value too small to hit the timeout first + waitForFirstTxnDuration = 1 * time.Microsecond + streamVerifier(txnGroups, badTxnGroups, t) } From b2e6c4416407cd55548023db803974b2f0102284 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 2 Nov 2022 01:09:26 -0400 Subject: [PATCH 080/156] golint fix --- data/txHandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/txHandler.go b/data/txHandler.go index 1e5e53d8b8..0b4cc1d997 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -94,8 +94,8 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go backlogQueue: make(chan *txBacklogMsg, txBacklogSize), postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), net: net, - streamVerifierChan: make(chan verify.UnverifiedElement, 0), - streamReturnChan: make(chan verify.VerificationResult, 0), + streamVerifierChan: make(chan verify.UnverifiedElement), + streamReturnChan: make(chan verify.VerificationResult), } handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) From fcc3661857f84372db45fd51faf86855bac7b03e Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:28:54 -0400 Subject: [PATCH 081/156] CR Co-authored-by: John Jannotti --- crypto/batchverifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index d3f3b9bc9b..51b5637a05 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -104,7 +104,7 @@ func (b *BatchVerifier) expand() { b.signatures = signatures } -// GetNumberOfEnqueuedSignatures returns the number of signatures current enqueue onto the bacth verifier object +// GetNumberOfEnqueuedSignatures returns the number of signatures currently enqueued into the BatchVerifier func (b *BatchVerifier) GetNumberOfEnqueuedSignatures() int { return len(b.messages) } From 53da26825f0b0d4861ecda879376e7c43fb2dc47 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 2 Nov 2022 11:54:12 -0400 Subject: [PATCH 082/156] fix test race condition appearance --- data/transactions/verify/txn.go | 3 +++ data/transactions/verify/txn_test.go | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 5a0f1d021b..e92c05e562 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -484,6 +484,7 @@ type StreamVerifier struct { ctx context.Context cache VerifiedTransactionCache pendingTasksWg sync.WaitGroup + activeLoopWg sync.WaitGroup nbw *NewBlockWatcher ledger logic.LedgerForSignature } @@ -600,10 +601,12 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, // Start is called when the verifier is created and whenever it needs to restart after // the ctx is canceled func (sv *StreamVerifier) Start() { + sv.activeLoopWg.Add(1) go sv.batchingLoop() } func (sv *StreamVerifier) batchingLoop() { + defer sv.activeLoopWg.Done() timer := time.NewTicker(waitForFirstTxnDuration) var added bool var numberOfSigsInCurrent uint64 diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 42760dc874..f6e55ef7a2 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -863,7 +863,7 @@ func BenchmarkTxn(b *testing.B) { b.StopTimer() } -func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, t *testing.T) { +func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, t *testing.T) (sv *StreamVerifier) { numOfTxnGroups := len(txnGroups) execPool := execpool.MakePool(t) @@ -879,7 +879,7 @@ func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint6 nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv = MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) sv.Start() wg := sync.WaitGroup{} @@ -951,6 +951,7 @@ func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint6 } } require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") + return sv } // TestStreamVerifier tests the basic functionality @@ -1077,5 +1078,6 @@ func TestStreamVerifierIdel(t *testing.T) { }() // set this value too small to hit the timeout first waitForFirstTxnDuration = 1 * time.Microsecond - streamVerifier(txnGroups, badTxnGroups, t) + sv := streamVerifier(txnGroups, badTxnGroups, t) + sv.activeLoopWg.Wait() } From 8e2b863ac2f1d45ea38a3fd483e1540fa84536ba Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 18:32:44 -0400 Subject: [PATCH 083/156] CR improvements: (1) increase wait time from 1 to 20ms (2) drop pendingTasksWg (3) drop unverifiedElementList (4) simplify cleanup/shutdown --- data/transactions/verify/txn.go | 94 ++++++++++++++------------------- 1 file changed, 39 insertions(+), 55 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index e92c05e562..cdfb75d1f7 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -483,7 +483,6 @@ type StreamVerifier struct { verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache - pendingTasksWg sync.WaitGroup activeLoopWg sync.WaitGroup nbw *NewBlockWatcher ledger logic.LedgerForSignature @@ -520,26 +519,6 @@ func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { return nbw.blkHeader } -type unverifiedElementList struct { - elementList []UnverifiedElement - nbw *NewBlockWatcher - ledger logic.LedgerForSignature -} - -func makeUnverifiedElementList(nbw *NewBlockWatcher, ledger logic.LedgerForSignature) (uel unverifiedElementList) { - uel.nbw = nbw - uel.ledger = ledger - uel.elementList = make([]UnverifiedElement, 0) - return -} - -func makeSingleUnverifiedElement(nbw *NewBlockWatcher, ledger logic.LedgerForSignature, ue UnverifiedElement) (uel unverifiedElementList) { - uel.nbw = nbw - uel.ledger = ledger - uel.elementList = []UnverifiedElement{ue} - return -} - type batchLoad struct { txnGroups [][]transactions.SignedTxn groupCtxs []*GroupContext @@ -566,7 +545,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext // waitForNextTxnDuration is the time to wait before sending the batch for evaluation. // this does not mean that every batch will wait this long. when the load level is high, // the batch will fill up quickly and will be sent for evaluation before this timeout -const waitForNextTxnDuration = 1 * time.Millisecond +const waitForNextTxnDuration = 20 * time.Millisecond // waitForFirstTxnDuration is the time to wait for the first transaction in the batch var waitForFirstTxnDuration = 2000 * time.Millisecond @@ -605,12 +584,19 @@ func (sv *StreamVerifier) Start() { go sv.batchingLoop() } +func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { + // report an error for the unchecked txns + for _, uel := range pending { + sv.sendResult(uel.TxnGroup, uel.Context, errors.New("not verified, verifier is shutting down")) + } +} + func (sv *StreamVerifier) batchingLoop() { defer sv.activeLoopWg.Done() timer := time.NewTicker(waitForFirstTxnDuration) var added bool var numberOfSigsInCurrent uint64 - uel := makeUnverifiedElementList(sv.nbw, sv.ledger) + uelts := make([]UnverifiedElement, 0) for { select { case stx := <-sv.stxnChan: @@ -625,16 +611,17 @@ func (sv *StreamVerifier) batchingLoop() { // if no batchable signatures here, send this as a task of its own if numberOfBatchableSigsInGroup == 0 { // no point in blocking and waiting for a seat here, let it wait in the exec pool - err := sv.addVerificationTaskToThePool(makeSingleUnverifiedElement(sv.nbw, sv.ledger, stx), false) + err := sv.addVerificationTaskToThePool([]UnverifiedElement{stx}, false) if err != nil { - continue // pool is terminated. nothing to report to the txn sender + sv.cleanup(uelts) + return } continue // stx is handled, continue } // add this txngrp to the list of batchable txn groups numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup - uel.elementList = append(uel.elementList, stx) + uelts = append(uelts, stx) if numberOfSigsInCurrent > txnPerWorksetThreshold { // enough transaction in the batch to efficiently verify @@ -642,17 +629,22 @@ func (sv *StreamVerifier) batchingLoop() { // do not consider adding more txns to this batch. // bypass the seat count and block if the exec pool is busy // this is to prevent creation of very large batches - err := sv.addVerificationTaskToThePool(uel, false) + err := sv.addVerificationTaskToThePool(uelts, false) if err != nil { - continue // pool is terminated. + sv.cleanup(uelts) + return } added = true } else { - added = sv.processBatch(uel) + added, err = sv.processBatch(uelts) + if err != nil { + sv.cleanup(uelts) + return + } } if added { numberOfSigsInCurrent = 0 - uel = makeUnverifiedElementList(sv.nbw, sv.ledger) + uelts = make([]UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { @@ -672,10 +664,14 @@ func (sv *StreamVerifier) batchingLoop() { timer.Reset(waitForFirstTxnDuration) continue } - added = sv.processBatch(uel) + added, err := sv.processBatch(uelts) + if err != nil { + sv.cleanup(uelts) + return + } if added { numberOfSigsInCurrent = 0 - uel = makeUnverifiedElementList(sv.nbw, sv.ledger) + uelts = make([]UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { @@ -683,15 +679,7 @@ func (sv *StreamVerifier) batchingLoop() { timer.Reset(waitForNextTxnDuration) } case <-sv.ctx.Done(): - if numberOfSigsInCurrent > 0 { - // send the current accumulated transactions to the pool - err := sv.addVerificationTaskToThePool(uel, false) - if err != nil { - continue // pool is terminated. - } - } - // wait for the pending tasks, then close the result chan - sv.pendingTasksWg.Wait() + sv.cleanup(uelts) return } } @@ -709,41 +697,37 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veCont sv.resultChan <- vr } -func (sv *StreamVerifier) processBatch(uel unverifiedElementList) (added bool) { +func (sv *StreamVerifier) processBatch(uelts []UnverifiedElement) (added bool, err error) { // if cannot find a seat, can go back and collect // more signatures instead of waiting here // more signatures to the batch do not harm performance (see crypto.BenchmarkBatchVerifierBig) select { case <-sv.seatReturnChan: - err := sv.addVerificationTaskToThePool(uel, true) + err := sv.addVerificationTaskToThePool(uelts, true) if err != nil { // An error is returned when the context of the pool expires - // No need to report this - return false + return false, err } - return true + return true, nil default: // if no free seats, wait some more for more txns - return false + return false, nil } } -func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList, returnSeat bool) error { - sv.pendingTasksWg.Add(1) +func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement, returnSeat bool) error { function := func(arg interface{}) interface{} { - defer sv.pendingTasksWg.Done() - if sv.ctx.Err() != nil { return struct{}{} } - uel := arg.(*unverifiedElementList) + uelts := arg.([]UnverifiedElement) batchVerifier := crypto.MakeBatchVerifier() bl := makeBatchLoad() // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here - for _, ue := range uel.elementList { - groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, uel.nbw.getBlockHeader(), uel.ledger, batchVerifier) + for _, ue := range uelts { + groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, sv.nbw.getBlockHeader(), sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error sv.sendResult(ue.TxnGroup, ue.Context, err) @@ -793,7 +777,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uel unverifiedElementList retChan = sv.seatReturnChan } // EnqueueBacklog returns an error when the context is canceled - err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, &uel, retChan) + err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, uelts, retChan) return err } From 6d868fb6095924c6576de29a25a8b2dbc698852a Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Thu, 3 Nov 2022 18:41:19 -0400 Subject: [PATCH 084/156] CR perf improvement in for loop Co-authored-by: John Jannotti --- crypto/batchverifier.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 51b5637a05..b8ff87b797 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -124,8 +124,8 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { return nil, nil } var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) - for i, m := range b.messages { - messages[i] = HashRep(m) + for i := range b.messages { + messages[i] = HashRep(b.messages[i]) } allValid, failed := batchVerificationImpl(messages, b.publicKeys, b.signatures) if allValid { From b30fb157cd629e17db6ad83319d87a49517f4ce8 Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Thu, 3 Nov 2022 18:46:08 -0400 Subject: [PATCH 085/156] CR: use pointer to avoid copy Co-authored-by: cce <51567+cce@users.noreply.github.com> --- data/transactions/verify/txn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index cdfb75d1f7..4b8de0e8ec 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -793,7 +793,7 @@ func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs u return } -func getNumberOfBatchableSigsInTxn(stx transactions.SignedTxn) (batchSigs uint64, err error) { +func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (batchSigs uint64, err error) { var hasSig, hasMsig bool numSigs := 0 if stx.Sig != (crypto.Signature{}) { From e085d9da39b92d5046288df4e44434666df806f9 Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Thu, 3 Nov 2022 18:46:58 -0400 Subject: [PATCH 086/156] CR: for loop avoid copy Co-authored-by: cce <51567+cce@users.noreply.github.com> --- data/transactions/verify/txn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4b8de0e8ec..3fc3a3e5b4 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -783,8 +783,8 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs uint64, err error) { batchSigs = 0 - for _, stx := range stxs { - count, err := getNumberOfBatchableSigsInTxn(stx) + for i := range stxs { + count, err := getNumberOfBatchableSigsInTxn(&stxs[i]) if err != nil { return 0, err } From 983f6e1bebbb76d266caf6565bf0b2f9b088fd54 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 18:56:26 -0400 Subject: [PATCH 087/156] CR: use pointer for passing BlockHeader --- data/transactions/verify/txn.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index cdfb75d1f7..bfaaa6bdd5 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -79,7 +79,7 @@ type GroupContext struct { // PrepareGroupContext prepares a verification group parameter object for a given transaction // group. -func PrepareGroupContext(group []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, ledger logic.LedgerForSignature) (*GroupContext, error) { +func PrepareGroupContext(group []transactions.SignedTxn, contextHdr *bookkeeping.BlockHeader, ledger logic.LedgerForSignature) (*GroupContext, error) { if len(group) == 0 { return nil, nil } @@ -123,7 +123,7 @@ func txnBatchPrep(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, } // TxnGroup verifies a []SignedTxn as being signed and having no obviously inconsistent data. -func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature) (groupCtx *GroupContext, err error) { +func TxnGroup(stxs []transactions.SignedTxn, contextHdr *bookkeeping.BlockHeader, cache VerifiedTransactionCache, ledger logic.LedgerForSignature) (groupCtx *GroupContext, err error) { batchVerifier := crypto.MakeBatchVerifier() if groupCtx, err = txnGroupBatchPrep(stxs, contextHdr, ledger, batchVerifier); err != nil { @@ -143,7 +143,7 @@ func TxnGroup(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, // txnGroupBatchPrep verifies a []SignedTxn having no obviously inconsistent data. // it is the caller responsibility to call batchVerifier.Verify() -func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr bookkeeping.BlockHeader, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { +func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.BlockHeader, ledger logic.LedgerForSignature, verifier *crypto.BatchVerifier) (groupCtx *GroupContext, err error) { groupCtx, err = PrepareGroupContext(stxs, contextHdr, ledger) if err != nil { return nil, err @@ -513,10 +513,10 @@ func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore nbw.blkHeader = block.BlockHeader } -func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { +func (nbw *NewBlockWatcher) getBlockHeader() (bh *bookkeeping.BlockHeader) { nbw.mu.RLock() defer nbw.mu.RUnlock() - return nbw.blkHeader + return &nbw.blkHeader } type batchLoad struct { @@ -726,8 +726,9 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement bl := makeBatchLoad() // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here + blockHeader := sv.nbw.getBlockHeader() for _, ue := range uelts { - groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, sv.nbw.getBlockHeader(), sv.ledger, batchVerifier) + groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, blockHeader, sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error sv.sendResult(ue.TxnGroup, ue.Context, err) From 598c4740557967d0d11cfae56733a98a5cdc3ce8 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 19:28:47 -0400 Subject: [PATCH 088/156] CR: pass around point to the block header --- cmd/goal/clerk.go | 4 ++-- data/transactions/verify/txn.go | 2 +- data/transactions/verify/txn_test.go | 32 ++++++++++++++-------------- data/txHandler_test.go | 2 +- node/node.go | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cmd/goal/clerk.go b/cmd/goal/clerk.go index 8ab3293267..5f3e677e4e 100644 --- a/cmd/goal/clerk.go +++ b/cmd/goal/clerk.go @@ -424,7 +424,7 @@ var sendCmd = &cobra.Command{ CurrentProtocol: proto, }, } - groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, blockHeader, nil) + groupCtx, err := verify.PrepareGroupContext([]transactions.SignedTxn{uncheckedTxn}, &blockHeader, nil) if err == nil { err = verify.LogicSigSanityCheck(&uncheckedTxn, 0, groupCtx) } @@ -825,7 +825,7 @@ var signCmd = &cobra.Command{ } var groupCtx *verify.GroupContext if lsig.Logic != nil { - groupCtx, err = verify.PrepareGroupContext(txnGroup, contextHdr, nil) + groupCtx, err = verify.PrepareGroupContext(txnGroup, &contextHdr, nil) if err != nil { // this error has to be unsupported protocol reportErrorf("%s: %v", txFilename, err) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 3c3976c16a..d6ceaad46f 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -383,7 +383,7 @@ func PaysetGroups(ctx context.Context, payset [][]transactions.SignedTxn, blkHea batchVerifier := crypto.MakeBatchVerifierWithHint(len(payset)) for i, signTxnsGrp := range txnGroups { - groupCtxs[i], grpErr = txnGroupBatchPrep(signTxnsGrp, blkHeader, ledger, batchVerifier) + groupCtxs[i], grpErr = txnGroupBatchPrep(signTxnsGrp, &blkHeader, ledger, batchVerifier) // abort only if it's a non-cache error. if grpErr != nil { return grpErr diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index f6e55ef7a2..1d4ee57ee5 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -40,7 +40,7 @@ import ( var feeSink = basics.Address{0x7, 0xda, 0xcb, 0x4b, 0x6d, 0x9e, 0xd1, 0x41, 0xb1, 0x75, 0x76, 0xbd, 0x45, 0x9a, 0xe6, 0x42, 0x1d, 0x48, 0x6d, 0xa3, 0xd4, 0xef, 0x22, 0x47, 0xc4, 0x9, 0xa3, 0x96, 0xb8, 0x2e, 0xa2, 0x21} var poolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} -var blockHeader = bookkeeping.BlockHeader{ +var blockHeader = &bookkeeping.BlockHeader{ RewardsState: bookkeeping.RewardsState{ FeeSink: feeSink, RewardsPool: poolAddr, @@ -271,7 +271,7 @@ func TestTxnValidationStateProof(t *testing.T) { }, } - var blockHeader = bookkeeping.BlockHeader{ + var blockHeader = &bookkeeping.BlockHeader{ RewardsState: bookkeeping.RewardsState{ FeeSink: feeSink, RewardsPool: poolAddr, @@ -452,13 +452,13 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) dummyLedger := DummyLedgerForSignature{} - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.NoError(t, err) ///// no sig tmpSig := txnGroups[0][0].Sig txnGroups[0][0].Sig = crypto.Signature{} - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.Error(t, err) require.Contains(t, err.Error(), "has no sig") txnGroups[0][0].Sig = tmpSig @@ -469,14 +469,14 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= Key: crypto.PublicKey{0x1}, Sig: crypto.Signature{0x2}, } - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.Error(t, err) require.Contains(t, err.Error(), "should only have one of Sig or Msig or LogicSig") txnGroups[0][0].Msig.Subsigs = nil ///// Sig + logic txnGroups[0][0].Lsig.Logic = op.Program - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.Error(t, err) require.Contains(t, err.Error(), "should only have one of Sig or Msig or LogicSig") txnGroups[0][0].Lsig.Logic = []byte{} @@ -489,7 +489,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= Key: crypto.PublicKey{0x1}, Sig: crypto.Signature{0x2}, } - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.Error(t, err) require.Contains(t, err.Error(), "should only have one of Sig or Msig or LogicSig") txnGroups[0][0].Lsig.Logic = []byte{} @@ -505,7 +505,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= Key: crypto.PublicKey{0x1}, Sig: crypto.Signature{0x2}, } - _, err = TxnGroup(txnGroups[0], blkHdr, nil, &dummyLedger) + _, err = TxnGroup(txnGroups[0], &blkHdr, nil, &dummyLedger) require.Error(t, err) require.Contains(t, err.Error(), "should only have one of Sig or Msig") @@ -562,7 +562,7 @@ func TestTxnGroupCacheUpdate(t *testing.T) { restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) } // TestTxnGroupCacheUpdateMultiSig makes sure that a payment transaction signed with multisig @@ -584,7 +584,7 @@ func TestTxnGroupCacheUpdateMultiSig(t *testing.T) { restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Msig.Subsigs[0].Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) } // TestTxnGroupCacheUpdateFailLogic test makes sure that a payment transaction contains a logic (and no signature) @@ -623,7 +623,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Lsig.Args[0][0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") } @@ -667,7 +667,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= restoreSignatureFunc := func(txn *transactions.SignedTxn) { txn.Lsig.Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) // signature is correct and logic fails breakSignatureFunc = func(txn *transactions.SignedTxn) { @@ -676,7 +676,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= restoreSignatureFunc = func(txn *transactions.SignedTxn) { txn.Lsig.Args[0][0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") } // TestTxnGroupCacheUpdateLogicWithMultiSig makes sure that a payment transaction contains logicsig signed with multisig is valid @@ -738,7 +738,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txn.Lsig.Msig.Subsigs[0].Sig[0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, crypto.ErrBatchHasFailedSigs.Error()) // signature is correct and logic fails breakSignatureFunc = func(txn *transactions.SignedTxn) { txn.Lsig.Args[0][0]++ @@ -746,7 +746,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= restoreSignatureFunc = func(txn *transactions.SignedTxn) { txn.Lsig.Args[0][0]-- } - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") } func createDummyBlockHeader() bookkeeping.BlockHeader { @@ -783,7 +783,7 @@ func createPayTransaction(fee uint64, fv, lv, amount int, sender, receiver basic // verifyGroup uses TxnGroup to verify txns and add them to the // cache. Then makes sure that only the valid txns are verified and added to // the cache. -func verifyGroup(t *testing.T, txnGroups [][]transactions.SignedTxn, blkHdr bookkeeping.BlockHeader, breakSig func(txn *transactions.SignedTxn), restoreSig func(txn *transactions.SignedTxn), errorString string) { +func verifyGroup(t *testing.T, txnGroups [][]transactions.SignedTxn, blkHdr *bookkeeping.BlockHeader, breakSig func(txn *transactions.SignedTxn), restoreSig func(txn *transactions.SignedTxn), errorString string) { cache := MakeVerifiedTransactionCache(1000) breakSig(&txnGroups[0][0]) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 8846a7ad8a..7fbbad77d5 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -127,7 +127,7 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { b.Logf("verifying %d signedTransactionGroups", len(signedTransactionGroups)) b.ResetTimer() for i := range signedTransactionGroups { - verify.TxnGroup(signedTransactionGroups[i], hdr, vtc, l) + verify.TxnGroup(signedTransactionGroups[i], &hdr, vtc, l) } }) } diff --git a/node/node.go b/node/node.go index a21aad1889..e983b9a128 100644 --- a/node/node.go +++ b/node/node.go @@ -500,7 +500,7 @@ func (node *AlgorandFullNode) broadcastSignedTxGroup(txgroup []transactions.Sign return err } - _, err = verify.TxnGroup(txgroup, b, node.ledger.VerifiedTransactionCache(), node.ledger) + _, err = verify.TxnGroup(txgroup, &b, node.ledger.VerifiedTransactionCache(), node.ledger) if err != nil { node.log.Warnf("malformed transaction: %v", err) return err From 4fcacb258a561369a2f83defd31549bd867d4192 Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Thu, 3 Nov 2022 19:39:29 -0400 Subject: [PATCH 089/156] CR: shortcut when all passed Co-authored-by: cce <51567+cce@users.noreply.github.com> --- data/transactions/verify/txn.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index d6ceaad46f..697469e07b 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -740,6 +740,13 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement failed, err := batchVerifier.VerifyWithFeedback() // this error can only be crypto.ErrBatchHasFailedSigs + if err == nil { // success, all signatures verified + for i := range bl.txnGroups { + sv.sendResult(bl.txnGroups[i], bl.elementContext[i], nil) + } + sv.cache.AddPayset(bl.txnGroups, bl.groupCtxs) + return nil + } verifiedTxnGroups := make([][]transactions.SignedTxn, 0, len(bl.txnGroups)) verifiedGroupCtxs := make([]*GroupContext, 0, len(bl.groupCtxs)) From a86659a9484892c6e95ae789164993db40afdd76 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 19:43:30 -0400 Subject: [PATCH 090/156] merge fix --- data/transactions/verify/txn.go | 3 +-- data/transactions/verify/txn_test.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 697469e07b..a63ec07b2f 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -753,8 +753,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement failedSigIdx := 0 for txgIdx := range bl.txnGroups { txGroupSigFailed := false - // if err == nil, then all sigs are verified, no need to check for the failed - for err != nil && failedSigIdx < bl.messagesForTxn[txgIdx] { + for failedSigIdx < bl.messagesForTxn[txgIdx] { if failed[failedSigIdx] { // if there is a failed sig check, then no need to check the rest of the // sigs for this txnGroup diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 1d4ee57ee5..c1b500e6ab 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -853,7 +853,7 @@ func BenchmarkTxn(b *testing.B) { b.ResetTimer() for _, txnGroup := range txnGroups { - groupCtx, err := PrepareGroupContext(txnGroup, blk.BlockHeader, nil) + groupCtx, err := PrepareGroupContext(txnGroup, &blk.BlockHeader, nil) require.NoError(b, err) for i, txn := range txnGroup { err := verifyTxn(&txn, i, groupCtx) From 9749709120c8817ff69979009ac5f36cddf6331f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 19:46:59 -0400 Subject: [PATCH 091/156] gofmt and return val --- data/transactions/verify/txn.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a63ec07b2f..0bc7558fad 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -743,9 +743,9 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement if err == nil { // success, all signatures verified for i := range bl.txnGroups { sv.sendResult(bl.txnGroups[i], bl.elementContext[i], nil) - } - sv.cache.AddPayset(bl.txnGroups, bl.groupCtxs) - return nil + } + sv.cache.AddPayset(bl.txnGroups, bl.groupCtxs) + return struct{}{} } verifiedTxnGroups := make([][]transactions.SignedTxn, 0, len(bl.txnGroups)) From a639935c3bb2f015fe8ebd862d108082f213cb72 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 3 Nov 2022 20:02:42 -0400 Subject: [PATCH 092/156] CR: rename Context to backlogMessage --- data/transactions/verify/txn.go | 48 ++++++++++++++-------------- data/transactions/verify/txn_test.go | 2 +- data/txHandler.go | 4 +-- data/txHandler_test.go | 4 +-- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 0bc7558fad..93f9fa5a06 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -458,20 +458,20 @@ func (w *worksetBuilder) completed() bool { } // UnverifiedElement is the element passed to the Stream verifier -// Context is a reference associated with the txn group which is passed +// BacklogMessage is a reference to the backlog message, which needs to be passed // with the result type UnverifiedElement struct { - TxnGroup []transactions.SignedTxn - Context interface{} + TxnGroup []transactions.SignedTxn + BacklogMessage interface{} } // VerificationResult is the result of the txn group verification -// Context is a reference associated with the txn group which was +// BacklogMessage is the reference associated with the txn group which was // initially passed to the stream verifier type VerificationResult struct { - TxnGroup []transactions.SignedTxn - Context interface{} - Err error + TxnGroup []transactions.SignedTxn + BacklogMessage interface{} + Err error } // StreamVerifier verifies txn groups received through the stxnChan channel, and returns the @@ -520,24 +520,24 @@ func (nbw *NewBlockWatcher) getBlockHeader() (bh *bookkeeping.BlockHeader) { } type batchLoad struct { - txnGroups [][]transactions.SignedTxn - groupCtxs []*GroupContext - elementContext []interface{} - messagesForTxn []int + txnGroups [][]transactions.SignedTxn + groupCtxs []*GroupContext + elementBacklogMessage []interface{} + messagesForTxn []int } func makeBatchLoad() (bl batchLoad) { bl.txnGroups = make([][]transactions.SignedTxn, 0) bl.groupCtxs = make([]*GroupContext, 0) - bl.elementContext = make([]interface{}, 0) + bl.elementBacklogMessage = make([]interface{}, 0) bl.messagesForTxn = make([]int, 0) return bl } -func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext, eltctx interface{}, numBatchableSigs int) { +func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext, backlogMsg interface{}, numBatchableSigs int) { bl.txnGroups = append(bl.txnGroups, txngrp) bl.groupCtxs = append(bl.groupCtxs, gctx) - bl.elementContext = append(bl.elementContext, eltctx) + bl.elementBacklogMessage = append(bl.elementBacklogMessage, backlogMsg) bl.messagesForTxn = append(bl.messagesForTxn, numBatchableSigs) } @@ -587,7 +587,7 @@ func (sv *StreamVerifier) Start() { func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { // report an error for the unchecked txns for _, uel := range pending { - sv.sendResult(uel.TxnGroup, uel.Context, errors.New("not verified, verifier is shutting down")) + sv.sendResult(uel.TxnGroup, uel.BacklogMessage, errors.New("not verified, verifier is shutting down")) } } @@ -604,7 +604,7 @@ func (sv *StreamVerifier) batchingLoop() { numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) if err != nil { // wrong number of signatures - sv.sendResult(stx.TxnGroup, stx.Context, err) + sv.sendResult(stx.TxnGroup, stx.BacklogMessage, err) continue } @@ -685,11 +685,11 @@ func (sv *StreamVerifier) batchingLoop() { } } -func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veContext interface{}, err error) { +func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBacklogMessage interface{}, err error) { vr := VerificationResult{ - TxnGroup: veTxnGroup, - Context: veContext, - Err: err, + TxnGroup: veTxnGroup, + BacklogMessage: veBacklogMessage, + Err: err, } // send the txn result out the pipe // this should never block. the receiver end of this channel will drop transactions if the @@ -731,18 +731,18 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, blockHeader, sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error - sv.sendResult(ue.TxnGroup, ue.Context, err) + sv.sendResult(ue.TxnGroup, ue.BacklogMessage, err) continue } totalBatchCount := batchVerifier.GetNumberOfEnqueuedSignatures() - bl.addLoad(ue.TxnGroup, groupCtx, ue.Context, totalBatchCount) + bl.addLoad(ue.TxnGroup, groupCtx, ue.BacklogMessage, totalBatchCount) } failed, err := batchVerifier.VerifyWithFeedback() // this error can only be crypto.ErrBatchHasFailedSigs if err == nil { // success, all signatures verified for i := range bl.txnGroups { - sv.sendResult(bl.txnGroups[i], bl.elementContext[i], nil) + sv.sendResult(bl.txnGroups[i], bl.elementBacklogMessage[i], nil) } sv.cache.AddPayset(bl.txnGroups, bl.groupCtxs) return struct{}{} @@ -771,7 +771,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement } else { result = ErrInvalidSignature } - sv.sendResult(bl.txnGroups[txgIdx], bl.elementContext[txgIdx], result) + sv.sendResult(bl.txnGroups[txgIdx], bl.elementBacklogMessage[txgIdx], result) } // loading them all at once by locking the cache once sv.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index c1b500e6ab..0d482a0ee4 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -924,7 +924,7 @@ func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint6 case <-ctx.Done(): break default: - stxnChan <- UnverifiedElement{TxnGroup: tg, Context: nil} + stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} } } }() diff --git a/data/txHandler.go b/data/txHandler.go index 0b4cc1d997..4183e0be28 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -121,7 +121,7 @@ func (handler *TxHandler) processTxnStreamVerifiedResults() { for { select { case result := <-handler.streamReturnChan: - txBLMsg := result.Context.(*txBacklogMsg) + txBLMsg := result.BacklogMessage.(*txBacklogMsg) txBLMsg.verificationErr = result.Err select { case handler.postVerificationQueue <- txBLMsg: @@ -191,7 +191,7 @@ func (handler *TxHandler) backlogWorker() { if handler.checkAlreadyCommitted(wi) { continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 7fbbad77d5..37b4b6ef1e 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -356,7 +356,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t // this is not expected during the test continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, Context: wi} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { return @@ -594,7 +594,7 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { tt = time.Now() for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, Context: &blm} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} } wg.Wait() handler.Stop() // cancel the handler ctx From 9d61283ae3469fd827b87fe4eca1c3673a28a1fe Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 4 Nov 2022 13:54:10 -0400 Subject: [PATCH 093/156] CR: cleanup error messages --- data/transactions/verify/txn.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 93f9fa5a06..682c775eba 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -45,6 +45,10 @@ var logicErrTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_l // ErrInvalidSignature is the error returned to report that at least one signature is invalid var ErrInvalidSignature = errors.New("At least one signature didn't pass verification") +var signedTxnHasNoSig = errors.New("signedtxn has no sig") +var signedTxnMaxOneSig = errors.New("signedtxn should only have one of Sig or Msig or LogicSig") +var shuttingDownError = errors.New("not verified, verifier is shutting down") + // The PaysetGroups is taking large set of transaction groups and attempt to verify their validity using multiple go-routines. // When doing so, it attempts to break these into smaller "worksets" where each workset takes about 2ms of execution time in order // to avoid context switching overhead while providing good validation cancelation responsiveness. Each one of these worksets is @@ -205,10 +209,10 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex return nil } - return errors.New("signedtxn has no sig") + return signedTxnHasNoSig } if numSigs > 1 { - return errors.New("signedtxn should only have one of Sig or Msig or LogicSig") + return signedTxnMaxOneSig } if hasSig { @@ -587,7 +591,7 @@ func (sv *StreamVerifier) Start() { func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { // report an error for the unchecked txns for _, uel := range pending { - sv.sendResult(uel.TxnGroup, uel.BacklogMessage, errors.New("not verified, verifier is shutting down")) + sv.sendResult(uel.TxnGroup, uel.BacklogMessage, shuttingDownError) } } @@ -769,7 +773,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement verifiedTxnGroups = append(verifiedTxnGroups, bl.txnGroups[txgIdx]) verifiedGroupCtxs = append(verifiedGroupCtxs, bl.groupCtxs[txgIdx]) } else { - result = ErrInvalidSignature + result = err } sv.sendResult(bl.txnGroups[txgIdx], bl.elementBacklogMessage[txgIdx], result) } @@ -823,10 +827,10 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (batchSigs uint6 if stx.Txn.Sender == transactions.StateProofSender && stx.Txn.Type == protocol.StateProofTx { return 0, nil } - return 0, errors.New("signedtxn has no sig") + return 0, signedTxnHasNoSig } if numSigs != 1 { - return 0, errors.New("signedtxn should only have one of Sig or Msig or LogicSig") + return 0, signedTxnMaxOneSig } if hasSig { return 1, nil From e30d9a4605efb930e957155016cd5f648b70fd93 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 4 Nov 2022 22:57:14 -0400 Subject: [PATCH 094/156] CR: (1) replace the seats count with the exec bool buffer size. (2) benchmark based on incoming tps rate --- data/transactions/verify/txn.go | 91 +++++++++++++++------------------ data/txHandler_test.go | 34 ++++++++---- util/execpool/backlog.go | 6 +++ 3 files changed, 71 insertions(+), 60 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 682c775eba..799061eb3b 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -56,6 +56,22 @@ var shuttingDownError = errors.New("not verified, verifier is shutting down") // show that these are realistic numbers ) const txnPerWorksetThreshold = 64 +// batchSizeBlockLimit is the limit when the batch exceeds, will be added to the exec pool, even if the pool is saturated +// and the batch verifier will block until the exec pool accepts the batch +const batchSizeBlockLimit = 512 + +// waitForNextTxnDuration is the time to wait before sending the batch to the exec pool +// If the incoming txn rate is low, a txn in the batch may wait no less than +// waitForNextTxnDuration before it is set for verification. +// This can introduce a latency to the propagation of a transaction in the network, +// since every relay will go through this wait time before broadcasting the txn. +// However, when the incoming txn rate is high, the batch will fill up quickly and will send +// for signature evaluation before waitForNextTxnDuration. +const waitForNextTxnDuration = 5 * time.Millisecond + +// waitForFirstTxnDuration is the time to wait for the first transaction in the batch +var waitForFirstTxnDuration = 2000 * time.Millisecond + // When the PaysetGroups is generating worksets, it enqueues up to concurrentWorksets entries to the execution pool. This serves several // purposes : // - if the verification task need to be aborted, there are only concurrentWorksets entries that are currently redundant on the execution pool queue. @@ -481,7 +497,6 @@ type VerificationResult struct { // StreamVerifier verifies txn groups received through the stxnChan channel, and returns the // results through the resultChan type StreamVerifier struct { - seatReturnChan chan interface{} resultChan chan<- VerificationResult stxnChan <-chan UnverifiedElement verificationPool execpool.BacklogPool @@ -546,27 +561,13 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext } -// waitForNextTxnDuration is the time to wait before sending the batch for evaluation. -// this does not mean that every batch will wait this long. when the load level is high, -// the batch will fill up quickly and will be sent for evaluation before this timeout -const waitForNextTxnDuration = 20 * time.Millisecond - -// waitForFirstTxnDuration is the time to wait for the first transaction in the batch -var waitForFirstTxnDuration = 2000 * time.Millisecond - // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) (sv *StreamVerifier) { - // limit the number of tasks queued to the execution pool - // the purpose of this parameter is to create bigger batches - // instead of having many smaller batching waiting in the execution pool - numberOfExecPoolSeats := verificationPool.GetParallelism() * 2 - sv = &StreamVerifier{ - seatReturnChan: make(chan interface{}, numberOfExecPoolSeats), resultChan: resultChan, stxnChan: stxnChan, verificationPool: verificationPool, @@ -575,9 +576,6 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, nbw: nbw, ledger: ledger, } - for x := 0; x < numberOfExecPoolSeats; x++ { - sv.seatReturnChan <- struct{}{} - } return sv } @@ -614,8 +612,7 @@ func (sv *StreamVerifier) batchingLoop() { // if no batchable signatures here, send this as a task of its own if numberOfBatchableSigsInGroup == 0 { - // no point in blocking and waiting for a seat here, let it wait in the exec pool - err := sv.addVerificationTaskToThePool([]UnverifiedElement{stx}, false) + err := sv.addVerificationTaskToThePoolNow([]UnverifiedElement{stx}) if err != nil { sv.cleanup(uelts) return @@ -629,18 +626,18 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfSigsInCurrent > txnPerWorksetThreshold { // enough transaction in the batch to efficiently verify - if numberOfSigsInCurrent > 4*txnPerWorksetThreshold { + if numberOfSigsInCurrent > batchSizeBlockLimit { // do not consider adding more txns to this batch. // bypass the seat count and block if the exec pool is busy // this is to prevent creation of very large batches - err := sv.addVerificationTaskToThePool(uelts, false) + err := sv.addVerificationTaskToThePoolNow(uelts) if err != nil { sv.cleanup(uelts) return } added = true } else { - added, err = sv.processBatch(uelts) + added, err = sv.canAddVerificationTaskToThePool(uelts) if err != nil { sv.cleanup(uelts) return @@ -652,7 +649,7 @@ func (sv *StreamVerifier) batchingLoop() { // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // was not added because no-available-seats. wait for some more txns + // was not added because of the exec pool buffer lenght timer.Reset(waitForNextTxnDuration) } } else { @@ -668,7 +665,7 @@ func (sv *StreamVerifier) batchingLoop() { timer.Reset(waitForFirstTxnDuration) continue } - added, err := sv.processBatch(uelts) + added, err := sv.canAddVerificationTaskToThePool(uelts) if err != nil { sv.cleanup(uelts) return @@ -679,7 +676,7 @@ func (sv *StreamVerifier) batchingLoop() { // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // was not added because no-available-seats. wait for some more txns + // was not added because of the exec pool buffer lenght. wait for some more txns timer.Reset(waitForNextTxnDuration) } case <-sv.ctx.Done(): @@ -701,28 +698,27 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack sv.resultChan <- vr } -func (sv *StreamVerifier) processBatch(uelts []UnverifiedElement) (added bool, err error) { - // if cannot find a seat, can go back and collect - // more signatures instead of waiting here - // more signatures to the batch do not harm performance (see crypto.BenchmarkBatchVerifierBig) - select { - case <-sv.seatReturnChan: - err := sv.addVerificationTaskToThePool(uelts, true) - if err != nil { - // An error is returned when the context of the pool expires - return false, err - } - return true, nil - default: - // if no free seats, wait some more for more txns +func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []UnverifiedElement) (added bool, err error) { + // if the exec pool buffer is (half) full, can go back and collect + // more signatures instead of waiting in the exec pool buffer + // more signatures to the batch do not harm performance but introduce latency when delayed (see crypto.BenchmarkBatchVerifierBig) + + // if buffer is half full + if l, c := sv.verificationPool.BufferLength(); l > c/2 { return false, nil } + err = sv.addVerificationTaskToThePoolNow(uelts) + if err != nil { + // An error is returned when the context of the pool expires + return false, err + } + return true, nil } -func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement, returnSeat bool) error { +func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []UnverifiedElement) error { function := func(arg interface{}) interface{} { if sv.ctx.Err() != nil { - return struct{}{} + return nil } uelts := arg.([]UnverifiedElement) @@ -749,7 +745,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement sv.sendResult(bl.txnGroups[i], bl.elementBacklogMessage[i], nil) } sv.cache.AddPayset(bl.txnGroups, bl.groupCtxs) - return struct{}{} + return nil } verifiedTxnGroups := make([][]transactions.SignedTxn, 0, len(bl.txnGroups)) @@ -779,16 +775,11 @@ func (sv *StreamVerifier) addVerificationTaskToThePool(uelts []UnverifiedElement } // loading them all at once by locking the cache once sv.cache.AddPayset(verifiedTxnGroups, verifiedGroupCtxs) - return struct{}{} + return nil } - // if the task has an allocated seat, should release it - var retChan chan interface{} - if returnSeat { - retChan = sv.seatReturnChan - } // EnqueueBacklog returns an error when the context is canceled - err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, uelts, retChan) + err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, uelts, nil) return err } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 37b4b6ef1e..5e75b6e2eb 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -498,20 +498,32 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, add return } -// BenchmarkHandler sends singed transactions the the verifier +// BenchmarkHandleTxns sends singed transactions the the verifier func BenchmarkHandleTxns(b *testing.B) { - b.N = b.N * proto.MaxTxGroupSize / 2 - runHandlerBenchmark(1, b) + maxGroupSize := 1 + tpss := []int{600000, 60000, 6000, 600} + for _, tps := range tpss { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + rateAdjuster := time.Second / time.Duration(tps) + runHandlerBenchmark(rateAdjuster, maxGroupSize, tps, b) + }) + } } - -// BenchmarkHandler sends singed transaction groups to the verifier +// BenchmarkHandleTransactionGroups sends singed transaction groups to the verifier func BenchmarkHandleTxnGroups(b *testing.B) { - runHandlerBenchmark(proto.MaxTxGroupSize, b) + maxGroupSize := proto.MaxTxGroupSize / 2 + tpss := []int{600000, 60000, 6000, 600} + for _, tps := range tpss { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + rateAdjuster := time.Second / time.Duration(tps) + runHandlerBenchmark(rateAdjuster, maxGroupSize, tps, b) + }) + } } // runHandlerBenchmark has a similar workflow to incomingTxHandlerProcessing, // but bypasses the backlog, and sends the transactions directly to the verifier -func runHandlerBenchmark(maxGroupSize int, b *testing.B) { +func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B) { const numUsers = 100 log := logging.TestingLog(b) log.SetLevel(logging.Warn) @@ -567,9 +579,10 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { invalidCounter := 0 defer func() { if txnCounter > 0 { - b.Logf("TPS: %d\n", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - b.Logf("Time/txn: %d(microsec)\n", uint64((time.Since(tt)/time.Microsecond))/txnCounter) - b.Logf("processed total: [%d groups (%d invalid)] [%d txns]\n", groupCounter, invalidCounter, txnCounter) + b.Logf("Input TPS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + b.Logf("Verified TPS: %d", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) } }() for wi := range outChan { @@ -595,6 +608,7 @@ func runHandlerBenchmark(maxGroupSize int, b *testing.B) { for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} + time.Sleep(rateAdjuster) } wg.Wait() handler.Stop() // cancel the handler ctx diff --git a/util/execpool/backlog.go b/util/execpool/backlog.go index 966aafe97f..a212f87909 100644 --- a/util/execpool/backlog.go +++ b/util/execpool/backlog.go @@ -43,6 +43,7 @@ type backlogItemTask struct { type BacklogPool interface { ExecutionPool EnqueueBacklog(enqueueCtx context.Context, t ExecFunc, arg interface{}, out chan interface{}) error + BufferLength() (length, capacity int) } // MakeBacklog creates a backlog @@ -94,6 +95,11 @@ func (b *backlog) Enqueue(enqueueCtx context.Context, t ExecFunc, arg interface{ } } +// BufferLength returns the length and the capacity of the buffer +func (b *backlog) BufferLength() (length, capacity int) { + return len(b.buffer), cap(b.buffer) +} + // Enqueue enqueues a single task into the backlog func (b *backlog) EnqueueBacklog(enqueueCtx context.Context, t ExecFunc, arg interface{}, out chan interface{}) error { select { From a756aaf359aacd4fbaffcf90c353ab00b9400889 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 5 Nov 2022 01:36:50 -0400 Subject: [PATCH 095/156] (1) do not block sending messages during shutdown, (2) test error messages --- data/transactions/verify/txn.go | 16 ++- data/transactions/verify/txn_test.go | 184 +++++++++++++++++++++------ 2 files changed, 155 insertions(+), 45 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 799061eb3b..d964d64ce0 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -588,8 +588,18 @@ func (sv *StreamVerifier) Start() { func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { // report an error for the unchecked txns + // drop the messages without reporting if the receiver does not consume for _, uel := range pending { - sv.sendResult(uel.TxnGroup, uel.BacklogMessage, shuttingDownError) + vr := VerificationResult{ + TxnGroup: uel.TxnGroup, + BacklogMessage: uel.BacklogMessage, + Err: shuttingDownError, + } + select { + case sv.resultChan <- vr: + default: + return + } } } @@ -693,8 +703,8 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack Err: err, } // send the txn result out the pipe - // this should never block. the receiver end of this channel will drop transactions if the - // postVerificationQueue is blocked + // this is expected not to block. the receiver end of this channel will drop transactions if the + // postVerificationQueue is blocked, and report it sv.resultChan <- vr } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 0d482a0ee4..38f5760397 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -19,6 +19,7 @@ package verify import ( "context" "encoding/binary" + "errors" "fmt" "math/rand" "sync" @@ -863,7 +864,8 @@ func BenchmarkTxn(b *testing.B) { b.StopTimer() } -func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, t *testing.T) (sv *StreamVerifier) { +func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, + expectedError error, t *testing.T) (sv *StreamVerifier) { numOfTxnGroups := len(txnGroups) execPool := execpool.MakePool(t) @@ -884,57 +886,67 @@ func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint6 wg := sync.WaitGroup{} + errChan := make(chan error) var badSigResultCounter int var goodSigResultCounter int - errChan := make(chan error) + wg.Add(1) - go func() { - defer wg.Done() - defer close(errChan) - // process the results - for x := 0; x < numOfTxnGroups; x++ { - select { - case result := <-resultChan: - u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) - if _, has := badTxnGroups[u]; has { - badSigResultCounter++ - if result.Err == nil { - err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) - errChan <- err - } - } else { - goodSigResultCounter++ - if result.Err != nil { - err := fmt.Errorf("%dth (%d) transaction failed to varify with good sigs", x, u) - errChan <- err - } - } - case <-ctx.Done(): - break - } - } - }() + go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx, &badSigResultCounter, &goodSigResultCounter, &wg) wg.Add(1) // send txn groups to be verified go func() { defer wg.Done() for _, tg := range txnGroups { - select { - case <-ctx.Done(): - break - default: - stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} - } + stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} } }() for err := range errChan { - require.NoError(t, err) + require.ErrorContains(t, err, expectedError.Error()) } wg.Wait() + verifyResults(txnGroups, badTxnGroups, cache, badSigResultCounter, goodSigResultCounter, t) + return sv +} + +func processResults(errChan chan<- error, resultChan <-chan VerificationResult, numOfTxnGroups int, + badTxnGroups map[uint64]struct{}, ctx context.Context, + badSigResultCounter, goodSigResultCounter *int, wg *sync.WaitGroup) { + defer wg.Done() + defer close(errChan) + // process the results + for x := 0; x < numOfTxnGroups; x++ { + select { + case <-ctx.Done(): + case result := <-resultChan: + u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) + if _, has := badTxnGroups[u]; has { + (*badSigResultCounter)++ + // we expected an error, but it is not the general crypto error + if result.Err != crypto.ErrBatchHasFailedSigs { + errChan <- result.Err + } + if result.Err == nil { + err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) + errChan <- err + return + } + } else { + (*goodSigResultCounter)++ + if result.Err != nil { + errChan <- result.Err + } + } + } + } +} + +func verifyResults(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, + cache VerifiedTransactionCache, + badSigResultCounter, goodSigResultCounter int, t *testing.T) { // check if all txns have been checked. require.Equal(t, len(txnGroups), badSigResultCounter+goodSigResultCounter) require.Equal(t, len(badTxnGroups), badSigResultCounter) @@ -951,7 +963,6 @@ func streamVerifier(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint6 } } require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") - return sv } // TestStreamVerifier tests the basic functionality @@ -973,7 +984,7 @@ func TestStreamVerifier(t *testing.T) { badTxnGroups[u] = struct{}{} } } - streamVerifier(txnGroups, badTxnGroups, t) + streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) } // TestStreamVerifierCases tests various valid and invalid transaction signature cases @@ -982,7 +993,6 @@ func TestStreamVerifierCases(t *testing.T) { numOfTxns := 10 _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - txnGroups := generateTransactionGroups(1, signedTxn, secrets, addrs) badTxnGroups := make(map[uint64]struct{}) @@ -992,16 +1002,27 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Sig = crypto.Signature{} u, _ := binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} + streamVerifierTestCore(txnGroups, badTxnGroups, signedTxnHasNoSig, t) mod++ + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + badTxnGroups = make(map[uint64]struct{}) + // invalid stateproof txn txnGroups[mod][0].Sig = crypto.Signature{} txnGroups[mod][0].Txn.Type = protocol.StateProofTx txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} + errFeeMustBeZeroInStateproofTxn := errors.New("fee must be zero in state-proof transaction") + streamVerifierTestCore(txnGroups, badTxnGroups, errFeeMustBeZeroInStateproofTxn, t) mod++ + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + badTxnGroups = make(map[uint64]struct{}) + // acceptable stateproof txn txnGroups[mod][0].Sig = crypto.Signature{} txnGroups[mod][0].Txn.Note = nil @@ -1009,13 +1030,19 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Txn.Header.Fee = basics.MicroAlgos{Raw: 0} txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender txnGroups[mod][0].Txn.PaymentTxnFields = transactions.PaymentTxnFields{} + streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) mod++ // multisig _, mSigTxn, _, _ := generateMultiSigTxn(1, 6, 50, t) txnGroups[mod] = mSigTxn + streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) mod++ + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + badTxnGroups = make(map[uint64]struct{}) + // logicsig // add a simple logic that verifies this condition: // sha256(arg0) == base64decode(5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E=) @@ -1031,6 +1058,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Lsig.Logic = op.Program program := logic.Program(op.Program) txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) + streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) mod++ // bad lgicsig @@ -1043,14 +1071,18 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} + streamVerifierTestCore(txnGroups, badTxnGroups, errors.New("rejected by logic"), t) mod++ + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + badTxnGroups = make(map[uint64]struct{}) + // txn with sig and msig txnGroups[mod][0].Msig = mSigTxn[0].Msig u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - - streamVerifier(txnGroups, badTxnGroups, t) + streamVerifierTestCore(txnGroups, badTxnGroups, signedTxnMaxOneSig, t) } // TestStreamVerifierIdel starts the verifer and sends nothing, to trigger the timer, then sends a txn @@ -1076,8 +1108,76 @@ func TestStreamVerifierIdel(t *testing.T) { defer func() { waitForFirstTxnDuration = origValue }() - // set this value too small to hit the timeout first + // set this value too small to hit the timeout first, then make sure can + // resume and process the incoming transactions waitForFirstTxnDuration = 1 * time.Microsecond - sv := streamVerifier(txnGroups, badTxnGroups, t) + sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv.activeLoopWg.Wait() +} + +// TestStreamVerifierCancel +func TestStreamVerifierCancel(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfTxns := 1000 + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) + + badTxnGroups := make(map[uint64]struct{}) + + for tgi := range txnGroups { + if rand.Float32() > 0.5 { + // make a bad sig + t := rand.Intn(len(txnGroups[tgi])) + txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 + u, _ := binary.Uvarint(txnGroups[tgi][0].Txn.Note) + badTxnGroups[u] = struct{}{} + } + } + + // prepare the stream verifier + numOfTxnGroups := len(txnGroups) + execPool := execpool.MakePool(t) + verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + verificationPool.Shutdown() + + ctx, cancel := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50000) + + blkHdr := createDummyBlockHeader() + nbw := MakeNewBlockWatcher(blkHdr) + stxnChan := make(chan UnverifiedElement) + resultChan := make(chan VerificationResult) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv.Start() + + errChan := make(chan error) + + var badSigResultCounter int + var goodSigResultCounter int + + wg := sync.WaitGroup{} + wg.Add(1) + go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx, &badSigResultCounter, &goodSigResultCounter, &wg) + + wg.Add(1) + // send txn groups to be verified + go func() { + defer wg.Done() + for _, tg := range txnGroups { + select { + case <-ctx.Done(): + break + case stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + } + } + }() + errored := false + for err := range errChan { + require.ErrorIs(t, err, shuttingDownError) + cancel() + errored = true + } + require.True(t, errored) sv.activeLoopWg.Wait() } From 4e6ef67af1ae50c940655a4fd1f53e4b4b0081be Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 11:31:44 -0500 Subject: [PATCH 096/156] test progress --- data/transactions/verify/txn_test.go | 114 +++++++++++++++++---------- 1 file changed, 74 insertions(+), 40 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 38f5760397..cfa3697df7 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -965,18 +965,15 @@ func verifyResults(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64 require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } -// TestStreamVerifier tests the basic functionality -func TestStreamVerifier(t *testing.T) { - partitiontest.PartitionTest(t) +func getSignedTransactions(numOfTxns, maxGrpSize int, badTxnProb Float64) (txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}) { - numOfTxns := 4000 _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) + txnGroups = generateTransactionGroups(maxGrpSize, signedTxn, secrets, addrs) - badTxnGroups := make(map[uint64]struct{}) + badTxnGroups = make(map[uint64]struct{}) for tgi := range txnGroups { - if rand.Float32() > 0.7 { + if rand.Float32() < badTxnProb { // make a bad sig t := rand.Intn(len(txnGroups[tgi])) txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 @@ -984,6 +981,17 @@ func TestStreamVerifier(t *testing.T) { badTxnGroups[u] = struct{}{} } } + return + +} + +// TestStreamVerifier tests the basic functionality +func TestStreamVerifier(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfTxns := 4000 + txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) } @@ -992,10 +1000,7 @@ func TestStreamVerifierCases(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - txnGroups := generateTransactionGroups(1, signedTxn, secrets, addrs) - badTxnGroups := make(map[uint64]struct{}) - + txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, 1, 0) mod := 1 // txn with 0 sigs @@ -1090,20 +1095,8 @@ func TestStreamVerifierIdel(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) - - badTxnGroups := make(map[uint64]struct{}) + txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) - for tgi := range txnGroups { - if rand.Float32() > 0.5 { - // make a bad sig - t := rand.Intn(len(txnGroups[tgi])) - txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 - u, _ := binary.Uvarint(txnGroups[tgi][0].Txn.Note) - badTxnGroups[u] = struct{}{} - } - } origValue := waitForFirstTxnDuration defer func() { waitForFirstTxnDuration = origValue @@ -1115,25 +1108,12 @@ func TestStreamVerifierIdel(t *testing.T) { sv.activeLoopWg.Wait() } -// TestStreamVerifierCancel -func TestStreamVerifierCancel(t *testing.T) { +// TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down +func TestStreamVerifierPoolShutdown(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1000 - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) - txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) - - badTxnGroups := make(map[uint64]struct{}) - - for tgi := range txnGroups { - if rand.Float32() > 0.5 { - // make a bad sig - t := rand.Intn(len(txnGroups[tgi])) - txnGroups[tgi][t].Sig[0] = txnGroups[tgi][t].Sig[0] + 1 - u, _ := binary.Uvarint(txnGroups[tgi][0].Txn.Note) - badTxnGroups[u] = struct{}{} - } - } + txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) @@ -1181,3 +1161,57 @@ func TestStreamVerifierCancel(t *testing.T) { require.True(t, errored) sv.activeLoopWg.Wait() } + +// TestStreamVerifierCtxCancel tests what happens when the context is canceled +func TestStreamVerifierCtxCancel(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfTxns := 1000 + txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + + // prepare the stream verifier + numOfTxnGroups := len(txnGroups) + execPool := execpool.MakePool(t) + verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cache := MakeVerifiedTransactionCache(50000) + + blkHdr := createDummyBlockHeader() + nbw := MakeNewBlockWatcher(blkHdr) + stxnChan := make(chan UnverifiedElement) + resultChan := make(chan VerificationResult) + sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv.Start() + + errChan := make(chan error) + + var badSigResultCounter int + var goodSigResultCounter int + + ctx2, cancel2 := context.WithCancel(context.Background()) + + wg := sync.WaitGroup{} + wg.Add(1) + go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx2, &badSigResultCounter, &goodSigResultCounter, &wg) + + wg.Add(1) + // send txn groups to be verified + go func() { + defer wg.Done() + for _, tg := range txnGroups { + select { + case <-ctx2.Done(): + break + case stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + } + } + }() + for err := range errChan { + require.ErrorIs(t, err, shuttingDownError) + cancel() + } + cancel2() + sv.activeLoopWg.Wait() +} From 7476588e35c2958bd6b39c56b708ec6156ef1114 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 18:20:52 -0500 Subject: [PATCH 097/156] fix tests --- data/transactions/verify/txn_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index cfa3697df7..6360c6090b 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -965,7 +965,7 @@ func verifyResults(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64 require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } -func getSignedTransactions(numOfTxns, maxGrpSize int, badTxnProb Float64) (txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}) { +func getSignedTransactions(numOfTxns, maxGrpSize int, badTxnProb float32) (txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}) { _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) txnGroups = generateTransactionGroups(maxGrpSize, signedTxn, secrets, addrs) @@ -990,7 +990,7 @@ func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 4000 - txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) } @@ -1000,7 +1000,7 @@ func TestStreamVerifierCases(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, 1, 0) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0) mod := 1 // txn with 0 sigs @@ -1010,8 +1010,8 @@ func TestStreamVerifierCases(t *testing.T) { streamVerifierTestCore(txnGroups, badTxnGroups, signedTxnHasNoSig, t) mod++ - _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) - txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) // invalid stateproof txn @@ -1024,8 +1024,8 @@ func TestStreamVerifierCases(t *testing.T) { streamVerifierTestCore(txnGroups, badTxnGroups, errFeeMustBeZeroInStateproofTxn, t) mod++ - _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) - txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + _, signedTxns, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) // acceptable stateproof txn @@ -1044,7 +1044,7 @@ func TestStreamVerifierCases(t *testing.T) { streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) mod++ - _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1095,7 +1095,7 @@ func TestStreamVerifierIdel(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) origValue := waitForFirstTxnDuration defer func() { @@ -1113,7 +1113,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1000 - txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) @@ -1167,7 +1167,7 @@ func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1000 - txnGroup, badTxnGroups := getSignedTransactions(numOfTxn, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) From db1c50253177d844d1fccb1426129deb23c121aa Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 18:24:15 -0500 Subject: [PATCH 098/156] draft test is not ready --- data/transactions/verify/txn_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 6360c6090b..2a836a9f73 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1161,7 +1161,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { require.True(t, errored) sv.activeLoopWg.Wait() } - +/* // TestStreamVerifierCtxCancel tests what happens when the context is canceled func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) @@ -1215,3 +1215,4 @@ func TestStreamVerifierCtxCancel(t *testing.T) { cancel2() sv.activeLoopWg.Wait() } +*/ From 6fad429a60d7bffcd30203d8181fc6254f0a2132 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 18:25:33 -0500 Subject: [PATCH 099/156] gofmt --- data/transactions/verify/txn_test.go | 3 ++- data/txHandler_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 2a836a9f73..3f54572360 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -982,7 +982,7 @@ func getSignedTransactions(numOfTxns, maxGrpSize int, badTxnProb float32) (txnGr } } return - + } // TestStreamVerifier tests the basic functionality @@ -1161,6 +1161,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { require.True(t, errored) sv.activeLoopWg.Wait() } + /* // TestStreamVerifierCtxCancel tests what happens when the context is canceled func TestStreamVerifierCtxCancel(t *testing.T) { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index e1bc3d060b..5c853ec387 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -510,6 +510,7 @@ func BenchmarkHandleTxns(b *testing.B) { }) } } + // BenchmarkHandleTransactionGroups sends singed transaction groups to the verifier func BenchmarkHandleTxnGroups(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 From 523a539959fc8e2beb983266b7580c63bac834bc Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 18:34:28 -0500 Subject: [PATCH 100/156] merge conflict fixes --- data/txHandler.go | 4 ++-- data/txHandler_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/txHandler.go b/data/txHandler.go index 8983ebd994..3549044f72 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -205,7 +205,7 @@ func (handler *TxHandler) backlogWorker() { if !ok { return } - handler.postprocessCheckedTxn(wi) + handler.postProcessCheckedTxn(wi) // restart the loop so that we could empty out the post verification queue. continue @@ -239,7 +239,7 @@ func (handler *TxHandler) backlogWorker() { } func (handler *TxHandler) postProcessReportErrors(err error) { - if errors.Is(err, crypto.ErrBatchVerificationFailed) { + if errors.Is(err, crypto.ErrBatchHasFailedSigs) { transactionMessagesTxnSigVerificationFailed.Inc(nil) return } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 5c853ec387..361d7f45d8 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -667,7 +667,7 @@ func TestTxHandlerPostProcessError(t *testing.T) { const expected = int(verify.TxGroupErrorReasonNumValues) - 3 require.Len(t, result, expected) - errVerify := crypto.ErrBatchVerificationFailed + errVerify := crypto.ErrBatchHasFailedSigs txh.postProcessReportErrors(errVerify) result = collect() require.Len(t, result, expected+1) @@ -685,7 +685,7 @@ func TestTxHandlerPostProcessErrorWithVerify(t *testing.T) { CurrentProtocol: protocol.ConsensusCurrentVersion, }, } - _, err := verify.TxnGroup([]transactions.SignedTxn{stxn}, hdr, nil, nil) + _, err := verify.TxnGroup([]transactions.SignedTxn{stxn}, &hdr, nil, nil) var txGroupErr *verify.ErrTxGroupError require.ErrorAs(t, err, &txGroupErr) From 4759d8e5293d47b9c8cbcbd7ef5a32f7d4fa71f1 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 18:50:31 -0500 Subject: [PATCH 101/156] golint fixes --- data/transactions/verify/txn.go | 16 ++++++++-------- data/transactions/verify/txn_test.go | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index d336f75346..61912c38cd 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -45,9 +45,9 @@ var logicErrTotal = metrics.MakeCounter(metrics.MetricName{Name: "algod_ledger_l // ErrInvalidSignature is the error returned to report that at least one signature is invalid var ErrInvalidSignature = errors.New("At least one signature didn't pass verification") -var signedTxnHasNoSig = errors.New("signedtxn has no sig") -var signedTxnMaxOneSig = errors.New("signedtxn should only have one of Sig or Msig or LogicSig") -var shuttingDownError = errors.New("not verified, verifier is shutting down") +var errSignedTxnHasNoSig = errors.New("signedtxn has no sig") +var errSignedTxnMaxOneSig = errors.New("signedtxn should only have one of Sig or Msig or LogicSig") +var errShuttingDownError = errors.New("not verified, verifier is shutting down") // The PaysetGroups is taking large set of transaction groups and attempt to verify their validity using multiple go-routines. // When doing so, it attempts to break these into smaller "worksets" where each workset takes about 2ms of execution time in order @@ -647,7 +647,7 @@ func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { vr := VerificationResult{ TxnGroup: uel.TxnGroup, BacklogMessage: uel.BacklogMessage, - Err: shuttingDownError, + Err: errShuttingDownError, } select { case sv.resultChan <- vr: @@ -713,7 +713,7 @@ func (sv *StreamVerifier) batchingLoop() { // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // was not added because of the exec pool buffer lenght + // was not added because of the exec pool buffer length timer.Reset(waitForNextTxnDuration) } } else { @@ -740,7 +740,7 @@ func (sv *StreamVerifier) batchingLoop() { // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) } else { - // was not added because of the exec pool buffer lenght. wait for some more txns + // was not added because of the exec pool buffer length. wait for some more txns timer.Reset(waitForNextTxnDuration) } case <-sv.ctx.Done(): @@ -882,10 +882,10 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (batchSigs uint6 if stx.Txn.Sender == transactions.StateProofSender && stx.Txn.Type == protocol.StateProofTx { return 0, nil } - return 0, signedTxnHasNoSig + return 0, errSignedTxnHasNoSig } if numSigs != 1 { - return 0, signedTxnMaxOneSig + return 0, errSignedTxnMaxOneSig } if hasSig { return 1, nil diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 3f54572360..7a76997923 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -891,7 +891,7 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m var goodSigResultCounter int wg.Add(1) - go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx, &badSigResultCounter, &goodSigResultCounter, &wg) + go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) wg.Add(1) // send txn groups to be verified @@ -912,8 +912,8 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m return sv } -func processResults(errChan chan<- error, resultChan <-chan VerificationResult, numOfTxnGroups int, - badTxnGroups map[uint64]struct{}, ctx context.Context, +func processResults(ctx context.Context, errChan chan<- error, resultChan <-chan VerificationResult, + numOfTxnGroups int, badTxnGroups map[uint64]struct{}, badSigResultCounter, goodSigResultCounter *int, wg *sync.WaitGroup) { defer wg.Done() defer close(errChan) @@ -1007,7 +1007,7 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Sig = crypto.Signature{} u, _ := binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - streamVerifierTestCore(txnGroups, badTxnGroups, signedTxnHasNoSig, t) + streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnHasNoSig, t) mod++ _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) @@ -1087,7 +1087,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Msig = mSigTxn[0].Msig u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - streamVerifierTestCore(txnGroups, badTxnGroups, signedTxnMaxOneSig, t) + streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnMaxOneSig, t) } // TestStreamVerifierIdel starts the verifer and sends nothing, to trigger the timer, then sends a txn @@ -1138,7 +1138,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx, &badSigResultCounter, &goodSigResultCounter, &wg) + go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) wg.Add(1) // send txn groups to be verified @@ -1154,7 +1154,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { }() errored := false for err := range errChan { - require.ErrorIs(t, err, shuttingDownError) + require.ErrorIs(t, err, errShuttingDownError) cancel() errored = true } @@ -1195,7 +1195,7 @@ func TestStreamVerifierCtxCancel(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - go processResults(errChan, resultChan, numOfTxnGroups, badTxnGroups, ctx2, &badSigResultCounter, &goodSigResultCounter, &wg) + go processResults(ctx2, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) wg.Add(1) // send txn groups to be verified @@ -1210,7 +1210,7 @@ func TestStreamVerifierCtxCancel(t *testing.T) { } }() for err := range errChan { - require.ErrorIs(t, err, shuttingDownError) + require.ErrorIs(t, err, errShuttingDownError) cancel() } cancel2() From 7a6edf7b8755847ac9f65d13cd034d938fe546df Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 17 Nov 2022 23:59:27 -0500 Subject: [PATCH 102/156] CR comments --- data/transactions/verify/txn.go | 8 ++------ data/transactions/verify/txn_test.go | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 61912c38cd..03d719a60e 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -54,7 +54,7 @@ var errShuttingDownError = errors.New("not verified, verifier is shutting down") // to avoid context switching overhead while providing good validation cancelation responsiveness. Each one of these worksets is // "populated" with roughly txnPerWorksetThreshold transactions. ( note that the real evaluation time is unknown, but benchmarks // show that these are realistic numbers ) -const txnPerWorksetThreshold = 64 +const txnPerWorksetThreshold = 32 // batchSizeBlockLimit is the limit when the batch exceeds, will be added to the exec pool, even if the pool is saturated // and the batch verifier will block until the exec pool accepts the batch @@ -663,6 +663,7 @@ func (sv *StreamVerifier) batchingLoop() { var added bool var numberOfSigsInCurrent uint64 uelts := make([]UnverifiedElement, 0) + defer sv.cleanup(uelts) for { select { case stx := <-sv.stxnChan: @@ -678,7 +679,6 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfBatchableSigsInGroup == 0 { err := sv.addVerificationTaskToThePoolNow([]UnverifiedElement{stx}) if err != nil { - sv.cleanup(uelts) return } continue // stx is handled, continue @@ -696,14 +696,12 @@ func (sv *StreamVerifier) batchingLoop() { // this is to prevent creation of very large batches err := sv.addVerificationTaskToThePoolNow(uelts) if err != nil { - sv.cleanup(uelts) return } added = true } else { added, err = sv.canAddVerificationTaskToThePool(uelts) if err != nil { - sv.cleanup(uelts) return } } @@ -731,7 +729,6 @@ func (sv *StreamVerifier) batchingLoop() { } added, err := sv.canAddVerificationTaskToThePool(uelts) if err != nil { - sv.cleanup(uelts) return } if added { @@ -744,7 +741,6 @@ func (sv *StreamVerifier) batchingLoop() { timer.Reset(waitForNextTxnDuration) } case <-sv.ctx.Done(): - sv.cleanup(uelts) return } } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 7a76997923..e188d6103c 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1107,7 +1107,7 @@ func TestStreamVerifierIdel(t *testing.T) { sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) sv.activeLoopWg.Wait() } - +/* // TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down func TestStreamVerifierPoolShutdown(t *testing.T) { partitiontest.PartitionTest(t) @@ -1162,7 +1162,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { sv.activeLoopWg.Wait() } -/* + // TestStreamVerifierCtxCancel tests what happens when the context is canceled func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) From 522562bc574c4bfd776c00656bc33710855f310b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 00:35:57 -0500 Subject: [PATCH 103/156] add test for sig counting --- data/transactions/verify/txn_test.go | 63 ++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index e188d6103c..4861786bb8 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1107,6 +1107,69 @@ func TestStreamVerifierIdel(t *testing.T) { sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) sv.activeLoopWg.Wait() } + +func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { + numOfTxns := 10 + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0) + mod := 1 + + // txn with 0 sigs + txnGroups[mod][0].Sig = crypto.Signature{} + batchSigs, err := getNumberOfBatchableSigsInGroup(txnGroups[mod]) + require.Error(t, err, errSignedTxnHasNoSig) + mod++ + + _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) + batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[0]) + require.NoError(t, err) + require.Equal(t, uint64(1), batchSigs) + + // stateproof txn + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Type = protocol.StateProofTx + txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender + batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[mod]) + require.NoError(t, err) + require.Equal(t, uint64(0), batchSigs) + mod++ + + // multisig + _, mSigTxn, _, _ := generateMultiSigTxn(1, 6, 50, t) + batchSigs, err = getNumberOfBatchableSigsInGroup(mSigTxn) + require.NoError(t, err) + require.Equal(t, uint64(2), batchSigs) + mod++ + + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + + // logicsig + op, err := logic.AssembleString(`arg 0 +sha256 +byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= +==`) + require.NoError(t, err) + s := rand.Intn(len(secrets)) + txnGroups[mod][0].Sig = crypto.Signature{} + txnGroups[mod][0].Txn.Sender = addrs[s] + txnGroups[mod][0].Lsig.Args = [][]byte{[]byte("=0\x97S\x85H\xe9\x91B\xfd\xdb;1\xf5Z\xaec?\xae\xf2I\x93\x08\x12\x94\xaa~\x06\x08\x849b")} + txnGroups[mod][0].Lsig.Logic = op.Program + program := logic.Program(op.Program) + txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) + batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[mod]) + require.NoError(t, err) + require.Equal(t, uint64(0), batchSigs) + mod++ + + // txn with sig and msig + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) + txnGroups[mod][0].Msig = mSigTxn[0].Msig + batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[mod]) + require.Error(t, err, errSignedTxnMaxOneSig) +} + /* // TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down func TestStreamVerifierPoolShutdown(t *testing.T) { From 350911459a7bda55c0cc496d0e5a79557354e55d Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 20:54:03 -0500 Subject: [PATCH 104/156] fix batch verifier restart context --- data/accountManager_test.go | 10 ++++++++++ data/ledger_test.go | 12 ++++++++++++ data/transactions/verify/txn.go | 6 +++--- data/txHandler.go | 4 ++-- data/txHandler_test.go | 6 ++++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/data/accountManager_test.go b/data/accountManager_test.go index 1fcfe56bf7..d62a0490db 100644 --- a/data/accountManager_test.go +++ b/data/accountManager_test.go @@ -84,6 +84,11 @@ func registryCloseTest(t testing.TB, registry account.ParticipationRegistry, dbf } func TestAccountManagerKeysRegistry(t *testing.T) { + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + partitiontest.PartitionTest(t) registry, dbName := getRegistryImpl(t, false, true) defer registryCloseTest(t, registry, dbName) @@ -91,6 +96,11 @@ func TestAccountManagerKeysRegistry(t *testing.T) { } func testAccountManagerKeys(t *testing.T, registry account.ParticipationRegistry, flushRegistry bool) { + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + log := logging.TestingLog(t) log.SetLevel(logging.Error) diff --git a/data/ledger_test.go b/data/ledger_test.go index 9b74ddc854..fef8f840c0 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -117,6 +117,10 @@ func testGenerateInitState(tb testing.TB, proto protocol.ConsensusVersion) (gene } func TestLedgerCirculation(t *testing.T) { + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } partitiontest.PartitionTest(t) genesisInitState, keys := testGenerateInitState(t, protocol.ConsensusCurrentVersion) @@ -325,6 +329,10 @@ func TestLedgerSeed(t *testing.T) { } func TestConsensusVersion(t *testing.T) { + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } partitiontest.PartitionTest(t) // find a consensus protocol that leads to ConsensusCurrentVersion @@ -471,6 +479,10 @@ func (lm loggedMessages) Errorf(s string, args ...interface{}) { // The purpose here is to simulate the scenario where the catchup and the agreement compete to add blocks to the ledger. // The error messages reported can be excessive or unnecessary. This test evaluates what messages are generate and at what frequency. func TestLedgerErrorValidate(t *testing.T) { + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } partitiontest.PartitionTest(t) var testPoolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 03d719a60e..8525493ee8 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -617,7 +617,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, +func MakeStreamVerifier(stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) (sv *StreamVerifier) { @@ -625,7 +625,6 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, resultChan: resultChan, stxnChan: stxnChan, verificationPool: verificationPool, - ctx: ctx, cache: cache, nbw: nbw, ledger: ledger, @@ -635,7 +634,8 @@ func MakeStreamVerifier(ctx context.Context, stxnChan <-chan UnverifiedElement, // Start is called when the verifier is created and whenever it needs to restart after // the ctx is canceled -func (sv *StreamVerifier) Start() { +func (sv *StreamVerifier) Start(ctx context.Context) { + sv.ctx = ctx sv.activeLoopWg.Add(1) go sv.batchingLoop() } diff --git a/data/txHandler.go b/data/txHandler.go index 1c8839f7c6..2da5391616 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -123,7 +123,7 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } nbw := verify.MakeNewBlockWatcher(latestHdr) handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - handler.streamVerifier = verify.MakeStreamVerifier(handler.ctx, handler.streamVerifierChan, handler.streamReturnChan, + handler.streamVerifier = verify.MakeStreamVerifier(handler.streamVerifierChan, handler.streamReturnChan, handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) return handler } @@ -161,7 +161,7 @@ func (handler *TxHandler) Start() { go handler.backlogWorker() go handler.backlogGaugeThread() go handler.processTxnStreamVerifiedResults() - handler.streamVerifier.Start() + handler.streamVerifier.Start(handler.ctx) } // Stop suspends the processing of incoming messages at the transaction handler diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 41751d6ea2..6ab0eeb448 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -327,8 +327,9 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t defer handler.ctxCancel() // emulate handler.Start() without the backlog + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) go handler.processTxnStreamVerifiedResults() - handler.streamVerifier.Start() + handler.streamVerifier.Start(handler.ctx) outChan := make(chan *txBacklogMsg, 10) wg := sync.WaitGroup{} @@ -570,8 +571,9 @@ func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *t defer handler.ctxCancel() // emulate handler.Start() without the backlog + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) go handler.processTxnStreamVerifiedResults() - handler.streamVerifier.Start() + handler.streamVerifier.Start(handler.ctx) // Prepare the transactions signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) From 75c568acf46c6f76ec6c052ac750e1deac2df7af Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 21:50:12 -0500 Subject: [PATCH 105/156] fix test with new interface --- agreement/asyncVoteVerifier_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/agreement/asyncVoteVerifier_test.go b/agreement/asyncVoteVerifier_test.go index 8a4e819595..bd7aa60139 100644 --- a/agreement/asyncVoteVerifier_test.go +++ b/agreement/asyncVoteVerifier_test.go @@ -34,6 +34,9 @@ func (fp *expiredExecPool) EnqueueBacklog(enqueueCtx context.Context, t execpool // generate an error, to see if we correctly report that on the verifyVote() call. return context.Canceled } +func (fp *expiredExecPool) BufferLength() (length, capacity int) { + return +} // Test async vote verifier against a full execution pool. func TestVerificationAgainstFullExecutionPool(t *testing.T) { From 7a246fcfb24330ac4c7c005992e5d060b5aae1e9 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 22:04:25 -0500 Subject: [PATCH 106/156] update test --- data/transactions/verify/txn_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 4861786bb8..5b2e69579d 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -881,8 +881,8 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv = MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) - sv.Start() + sv = MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv.Start(ctx) wg := sync.WaitGroup{} From 29cd209178d1f45fa9efd16c21e0c9dbddc02a6c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 22:52:23 -0500 Subject: [PATCH 107/156] fix wg done --- data/txHandler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/data/txHandler.go b/data/txHandler.go index 2da5391616..d3e7f8cdb4 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -132,6 +132,7 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go // this is needed so that the exec pool never gets blocked when it is pushing out the result // if the postVerificationQueue is full, it will be reported to the transactionMessagesDroppedFromPool metric func (handler *TxHandler) processTxnStreamVerifiedResults() { + defer handler.backlogWg.Done() for { select { case result := <-handler.streamReturnChan: From ddb923f54fe55221a4f94ef6b7ff2f5d74cb9d61 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 18 Nov 2022 23:18:32 -0500 Subject: [PATCH 108/156] add pool shutdown test --- data/transactions/verify/txn_test.go | 24 +++++++++++++++++------- data/txHandler_test.go | 1 + 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 5b2e69579d..11a0657810 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1170,12 +1170,14 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= require.Error(t, err, errSignedTxnMaxOneSig) } -/* + // TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down func TestStreamVerifierPoolShutdown(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxns := 1000 + // only one transaction should be sufficient for the batch verifier + // to realize the pool is terminated and to shut down + numOfTxns := 1 txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) // prepare the stream verifier @@ -1191,8 +1193,8 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) - sv.Start() + sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv.Start(ctx) errChan := make(chan error) @@ -1203,6 +1205,15 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { wg.Add(1) go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) + // When the exec pool shuts down, the batch verifier should gracefully stop + // cancel the context so that the test can terminate + wg.Add(1) + go func() { + defer wg.Done() + sv.activeLoopWg.Wait() + cancel() + }() + wg.Add(1) // send txn groups to be verified go func() { @@ -1221,11 +1232,10 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { cancel() errored = true } - require.True(t, errored) - sv.activeLoopWg.Wait() + require.False(t, errored) } - + /* // TestStreamVerifierCtxCancel tests what happens when the context is canceled func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 6ab0eeb448..7a030dc1e5 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -328,6 +328,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t // emulate handler.Start() without the backlog handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + handler.backlogWg.Add(1) go handler.processTxnStreamVerifiedResults() handler.streamVerifier.Start(handler.ctx) From 4076e42de1ee6318ecf0aefd0a77e63a53666f5b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 19 Nov 2022 07:45:22 -0500 Subject: [PATCH 109/156] gofmt --- data/transactions/verify/txn_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 11a0657810..19ca4ad21e 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1170,7 +1170,6 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= require.Error(t, err, errSignedTxnMaxOneSig) } - // TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down func TestStreamVerifierPoolShutdown(t *testing.T) { partitiontest.PartitionTest(t) @@ -1235,7 +1234,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { require.False(t, errored) } - /* +/* // TestStreamVerifierCtxCancel tests what happens when the context is canceled func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) From b2d216beafd7f704c2dfaa6b8ec91d36dd98b13a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 21 Nov 2022 10:23:12 -0500 Subject: [PATCH 110/156] fix bench --- data/txHandler_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 7a030dc1e5..97d43eddd2 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -573,6 +573,7 @@ func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *t // emulate handler.Start() without the backlog handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + handler.backlogWg.Add(1) go handler.processTxnStreamVerifiedResults() handler.streamVerifier.Start(handler.ctx) From c17a9cc5668f5355f530b6df14282934b3c279fd Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 21 Nov 2022 13:46:56 -0500 Subject: [PATCH 111/156] limit the number of times a batch is held from the exec pool when the pool buffer is full. fix the execpool shutdown test --- data/transactions/verify/txn.go | 20 ++++++++++++++-- data/transactions/verify/txn_test.go | 35 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 8525493ee8..10e933ae8a 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -662,6 +662,7 @@ func (sv *StreamVerifier) batchingLoop() { timer := time.NewTicker(waitForFirstTxnDuration) var added bool var numberOfSigsInCurrent uint64 + var numberOfTimerResets uint64 uelts := make([]UnverifiedElement, 0) defer sv.cleanup(uelts) for { @@ -692,7 +693,7 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfSigsInCurrent > batchSizeBlockLimit { // do not consider adding more txns to this batch. - // bypass the seat count and block if the exec pool is busy + // bypass the exec pool situation and queue anyway // this is to prevent creation of very large batches err := sv.addVerificationTaskToThePoolNow(uelts) if err != nil { @@ -710,14 +711,17 @@ func (sv *StreamVerifier) batchingLoop() { uelts = make([]UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) + numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length timer.Reset(waitForNextTxnDuration) + numberOfTimerResets++ } } else { if isFirstInBatch { // an element is added and is waiting. shorten the waiting time timer.Reset(waitForNextTxnDuration) + numberOfTimerResets = 0 } } case <-timer.C: @@ -725,9 +729,19 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfSigsInCurrent == 0 { // nothing batched yet... wait some more timer.Reset(waitForFirstTxnDuration) + numberOfTimerResets = 0 continue } - added, err := sv.canAddVerificationTaskToThePool(uelts) + var err error + if numberOfTimerResets > 1 { + // bypass the exec pool situation and queue anyway + // this is to prevent long delays in transaction propagation + // at least one transaction here has waited 3 x waitForFirstTxnDuration + err = sv.addVerificationTaskToThePoolNow(uelts) + added = true + } else { + added, err = sv.canAddVerificationTaskToThePool(uelts) + } if err != nil { return } @@ -736,9 +750,11 @@ func (sv *StreamVerifier) batchingLoop() { uelts = make([]UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) + numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length. wait for some more txns timer.Reset(waitForNextTxnDuration) + numberOfTimerResets++ } case <-sv.ctx.Done(): return diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 19ca4ad21e..6c50aa7d19 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/rand" + "runtime" "sync" "testing" "time" @@ -1182,8 +1183,37 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { // prepare the stream verifier numOfTxnGroups := len(txnGroups) execPool := execpool.MakePool(t) - verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) - verificationPool.Shutdown() + backlogQueueSize := 64 + verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) + _, buffLen := verificationPool.BufferLength() + + // make sure the pool is shut down and the buffer is full + holdTasks := make(chan interface{}) + for x := 0; x < buffLen+runtime.NumCPU(); x++ { + verificationPool.EnqueueBacklog(context.Background(), + func(arg interface{}) interface{} { <-holdTasks; return nil }, nil, nil) + } + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + // Shutdown will block until all tasks held by holdTasks is released + verificationPool.Shutdown() + }() + // Send more tasks to break the backlog worker after b.pool.Enqueue returns the error + for x := 0; x < 100; x++ { + verificationPool.EnqueueBacklog(context.Background(), + func(arg interface{}) interface{} { <-holdTasks; return nil }, nil, nil) + } + // release the tasks + close(holdTasks) + + // make sure the EnqueueBacklogis returning err + for x := 0; x < 10; x++ { + err := verificationPool.EnqueueBacklog(context.Background(), + func(arg interface{}) interface{} { return nil }, nil, nil) + require.Error(t, err, fmt.Sprintf("x = %d", x)) + } ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) @@ -1200,7 +1230,6 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { var badSigResultCounter int var goodSigResultCounter int - wg := sync.WaitGroup{} wg.Add(1) go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) From 777e0ac0d9dad05cf2360e9065a426624f0256fd Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 22 Nov 2022 00:29:04 -0500 Subject: [PATCH 112/156] Code: Fix cleanup param, block cleanup on return chan, hold task if buffer is full (instead of half full), fix comments. Test: add streamVerifier restart, test stability enhancements. --- data/transactions/verify/txn.go | 16 +++----- data/transactions/verify/txn_test.go | 59 +++++++++++++++++----------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 10e933ae8a..d28bd4ef01 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -640,20 +640,16 @@ func (sv *StreamVerifier) Start(ctx context.Context) { go sv.batchingLoop() } -func (sv *StreamVerifier) cleanup(pending []UnverifiedElement) { +func (sv *StreamVerifier) cleanup(pending *[]UnverifiedElement) { // report an error for the unchecked txns // drop the messages without reporting if the receiver does not consume - for _, uel := range pending { + for _, uel := range *pending { vr := VerificationResult{ TxnGroup: uel.TxnGroup, BacklogMessage: uel.BacklogMessage, Err: errShuttingDownError, } - select { - case sv.resultChan <- vr: - default: - return - } + sv.resultChan <- vr } } @@ -664,7 +660,7 @@ func (sv *StreamVerifier) batchingLoop() { var numberOfSigsInCurrent uint64 var numberOfTimerResets uint64 uelts := make([]UnverifiedElement, 0) - defer sv.cleanup(uelts) + defer sv.cleanup(&uelts) for { select { case stx := <-sv.stxnChan: @@ -736,7 +732,7 @@ func (sv *StreamVerifier) batchingLoop() { if numberOfTimerResets > 1 { // bypass the exec pool situation and queue anyway // this is to prevent long delays in transaction propagation - // at least one transaction here has waited 3 x waitForFirstTxnDuration + // at least one transaction here has waited 3 x waitForNextTxnDuration err = sv.addVerificationTaskToThePoolNow(uelts) added = true } else { @@ -780,7 +776,7 @@ func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []UnverifiedElem // more signatures to the batch do not harm performance but introduce latency when delayed (see crypto.BenchmarkBatchVerifierBig) // if buffer is half full - if l, c := sv.verificationPool.BufferLength(); l > c/2 { + if l, c := sv.verificationPool.BufferLength(); l == c { return false, nil } err = sv.addVerificationTaskToThePoolNow(uelts) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 6c50aa7d19..01ae265469 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -993,7 +993,8 @@ func TestStreamVerifier(t *testing.T) { numOfTxns := 4000 txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) - streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv.activeLoopWg.Wait() } // TestStreamVerifierCases tests various valid and invalid transaction signature cases @@ -1008,7 +1009,8 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Sig = crypto.Signature{} u, _ := binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnHasNoSig, t) + sv := streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnHasNoSig, t) + sv.activeLoopWg.Wait() mod++ _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) @@ -1022,7 +1024,8 @@ func TestStreamVerifierCases(t *testing.T) { u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} errFeeMustBeZeroInStateproofTxn := errors.New("fee must be zero in state-proof transaction") - streamVerifierTestCore(txnGroups, badTxnGroups, errFeeMustBeZeroInStateproofTxn, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, errFeeMustBeZeroInStateproofTxn, t) + sv.activeLoopWg.Wait() mod++ _, signedTxns, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) @@ -1036,13 +1039,15 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Txn.Header.Fee = basics.MicroAlgos{Raw: 0} txnGroups[mod][0].Txn.Header.Sender = transactions.StateProofSender txnGroups[mod][0].Txn.PaymentTxnFields = transactions.PaymentTxnFields{} - streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv.activeLoopWg.Wait() mod++ // multisig _, mSigTxn, _, _ := generateMultiSigTxn(1, 6, 50, t) txnGroups[mod] = mSigTxn - streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv.activeLoopWg.Wait() mod++ _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) @@ -1064,7 +1069,8 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Lsig.Logic = op.Program program := logic.Program(op.Program) txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) - streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) + sv.activeLoopWg.Wait() mod++ // bad lgicsig @@ -1077,7 +1083,8 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Lsig.Sig = secrets[s].Sign(program) u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - streamVerifierTestCore(txnGroups, badTxnGroups, errors.New("rejected by logic"), t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, errors.New("rejected by logic"), t) + sv.activeLoopWg.Wait() mod++ _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) @@ -1088,7 +1095,8 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Msig = mSigTxn[0].Msig u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnMaxOneSig, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnMaxOneSig, t) + sv.activeLoopWg.Wait() } // TestStreamVerifierIdel starts the verifer and sends nothing, to trigger the timer, then sends a txn @@ -1254,38 +1262,33 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { } } }() - errored := false for err := range errChan { require.ErrorIs(t, err, errShuttingDownError) - cancel() - errored = true } - require.False(t, errored) } -/* -// TestStreamVerifierCtxCancel tests what happens when the context is canceled -func TestStreamVerifierCtxCancel(t *testing.T) { +// TestStreamVerifierRestart tests what happens when the context is canceled +func TestStreamVerifierRestart(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxns := 1000 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) + numOfTxns := 100 + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) execPool := execpool.MakePool(t) verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) - ctx, cancel := context.WithCancel(context.Background()) - cancel() cache := MakeVerifiedTransactionCache(50000) blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(ctx, stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) - sv.Start() + + ctx, cancel := context.WithCancel(context.Background()) + sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv.Start(ctx) errChan := make(chan error) @@ -1302,19 +1305,27 @@ func TestStreamVerifierCtxCancel(t *testing.T) { // send txn groups to be verified go func() { defer wg.Done() - for _, tg := range txnGroups { + for i, tg := range txnGroups { + if (i+1)%10 == 0 { + cancel() + sv.activeLoopWg.Wait() + ctx, cancel = context.WithCancel(context.Background()) + sv.Start(ctx) + } select { case <-ctx2.Done(): break case stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: } } + cancel() }() for err := range errChan { require.ErrorIs(t, err, errShuttingDownError) - cancel() + if err != errShuttingDownError { + cancel2() + } } cancel2() sv.activeLoopWg.Wait() } -*/ From f72489d65a9fe879f1300d1f3a0c6a4a32521441 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 23 Nov 2022 01:04:36 -0500 Subject: [PATCH 113/156] Use postVerificationQueue directly for returning results from StreamVerifier --- data/transactions/verify/txn.go | 21 +++++++------ data/transactions/verify/txn_test.go | 16 ++++++---- data/txHandler.go | 46 ++++++++-------------------- data/txHandler_test.go | 32 +++++++++---------- 4 files changed, 49 insertions(+), 66 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index d28bd4ef01..5bf498eda0 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -559,6 +559,7 @@ type StreamVerifier struct { activeLoopWg sync.WaitGroup nbw *NewBlockWatcher ledger logic.LedgerForSignature + droppedFromPool *metrics.Counter } // NewBlockWatcher is a struct used to provide a new block header to the @@ -619,7 +620,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext // to it and obtain the txn signature verification result from func MakeStreamVerifier(stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, - cache VerifiedTransactionCache) (sv *StreamVerifier) { + cache VerifiedTransactionCache, droppedFromPool *metrics.Counter) (sv *StreamVerifier) { sv = &StreamVerifier{ resultChan: resultChan, @@ -628,6 +629,7 @@ func MakeStreamVerifier(stxnChan <-chan UnverifiedElement, resultChan chan<- Ver cache: cache, nbw: nbw, ledger: ledger, + droppedFromPool: droppedFromPool, } return sv } @@ -644,12 +646,7 @@ func (sv *StreamVerifier) cleanup(pending *[]UnverifiedElement) { // report an error for the unchecked txns // drop the messages without reporting if the receiver does not consume for _, uel := range *pending { - vr := VerificationResult{ - TxnGroup: uel.TxnGroup, - BacklogMessage: uel.BacklogMessage, - Err: errShuttingDownError, - } - sv.resultChan <- vr + sv.sendResult(uel.TxnGroup, uel.BacklogMessage, errShuttingDownError) } } @@ -765,9 +762,13 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack Err: err, } // send the txn result out the pipe - // this is expected not to block. the receiver end of this channel will drop transactions if the - // postVerificationQueue is blocked, and report it - sv.resultChan <- vr + select { + case sv.resultChan <- vr: + default: + // we failed to write to the output queue, since the queue was full. + // adding the metric here allows us to monitor how frequently it happens. + sv.droppedFromPool.Inc(nil) + } } func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []UnverifiedElement) (added bool, err error) { diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 01ae265469..b9e0180e7a 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -38,6 +38,7 @@ import ( "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/execpool" + "github.com/algorand/go-algorand/util/metrics" ) var feeSink = basics.Address{0x7, 0xda, 0xcb, 0x4b, 0x6d, 0x9e, 0xd1, 0x41, 0xb1, 0x75, 0x76, 0xbd, 0x45, 0x9a, 0xe6, 0x42, 0x1d, 0x48, 0x6d, 0xa3, 0xd4, 0xef, 0x22, 0x47, 0xc4, 0x9, 0xa3, 0x96, 0xb8, 0x2e, 0xa2, 0x21} @@ -52,6 +53,7 @@ var blockHeader = &bookkeeping.BlockHeader{ }, } var protoMaxGroupSize = config.Consensus[protocol.ConsensusCurrentVersion].MaxTxGroupSize +var txBacklogSize = config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnBytesPerBlock / 200 var spec = transactions.SpecialAddresses{ FeeSink: feeSink, @@ -865,6 +867,8 @@ func BenchmarkTxn(b *testing.B) { b.StopTimer() } +var droppedFromPool = metrics.MakeCounter(metrics.MetricName{Name: "test_streamVerifierTestCore_messages_dropped_pool", Description: "Test streamVerifierTestCore messages dropped from pool"}) + func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}, expectedError error, t *testing.T) (sv *StreamVerifier) { @@ -881,8 +885,8 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult) - sv = MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + resultChan := make(chan VerificationResult, txBacklogSize) + sv = MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) sv.Start(ctx) wg := sync.WaitGroup{} @@ -1229,8 +1233,8 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult) - sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + resultChan := make(chan VerificationResult, txBacklogSize) + sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) sv.Start(ctx) errChan := make(chan error) @@ -1284,10 +1288,10 @@ func TestStreamVerifierRestart(t *testing.T) { blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult) + resultChan := make(chan VerificationResult, txBacklogSize) ctx, cancel := context.WithCancel(context.Background()) - sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache) + sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) sv.Start(ctx) errChan := make(chan error) diff --git a/data/txHandler.go b/data/txHandler.go index d3e7f8cdb4..e823f16183 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -78,14 +78,13 @@ type TxHandler struct { genesisHash crypto.Digest txVerificationPool execpool.BacklogPool backlogQueue chan *txBacklogMsg - postVerificationQueue chan *txBacklogMsg + postVerificationQueue chan verify.VerificationResult backlogWg sync.WaitGroup net network.GossipNode ctx context.Context ctxCancel context.CancelFunc streamVerifier *verify.StreamVerifier streamVerifierChan chan verify.UnverifiedElement - streamReturnChan chan verify.VerificationResult } // MakeTxHandler makes a new handler for transaction messages @@ -108,10 +107,9 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go ledger: ledger, txVerificationPool: executionPool, backlogQueue: make(chan *txBacklogMsg, txBacklogSize), - postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), + postVerificationQueue: make(chan verify.VerificationResult, txBacklogSize), net: net, streamVerifierChan: make(chan verify.UnverifiedElement), - streamReturnChan: make(chan verify.VerificationResult), } // prepare the transaction stream verifer @@ -123,45 +121,21 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } nbw := verify.MakeNewBlockWatcher(latestHdr) handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - handler.streamVerifier = verify.MakeStreamVerifier(handler.streamVerifierChan, handler.streamReturnChan, - handler.ledger, nbw, handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) + handler.streamVerifier = verify.MakeStreamVerifier(handler.streamVerifierChan, + handler.postVerificationQueue, handler.ledger, nbw, handler.txVerificationPool, + handler.ledger.VerifiedTransactionCache(), transactionMessagesDroppedFromPool) return handler } -// processTxnStreamVerifiedResults relays the results from the stream verifier to the postVerificationQueue -// this is needed so that the exec pool never gets blocked when it is pushing out the result -// if the postVerificationQueue is full, it will be reported to the transactionMessagesDroppedFromPool metric -func (handler *TxHandler) processTxnStreamVerifiedResults() { - defer handler.backlogWg.Done() - for { - select { - case result := <-handler.streamReturnChan: - txBLMsg := result.BacklogMessage.(*txBacklogMsg) - txBLMsg.verificationErr = result.Err - select { - case handler.postVerificationQueue <- txBLMsg: - default: - // we failed to write to the output queue, since the queue was full. - // adding the metric here allows us to monitor how frequently it happens. - transactionMessagesDroppedFromPool.Inc(nil) - } - case <-handler.ctx.Done(): - return - - } - } -} - // Start enables the processing of incoming messages at the transaction handler func (handler *TxHandler) Start() { handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) handler.net.RegisterHandlers([]network.TaggedMessageHandler{ {Tag: protocol.TxnTag, MessageHandler: network.HandlerFunc(handler.processIncomingTxn)}, }) - handler.backlogWg.Add(3) + handler.backlogWg.Add(2) go handler.backlogWorker() go handler.backlogGaugeThread() - go handler.processTxnStreamVerifiedResults() handler.streamVerifier.Start(handler.ctx) } @@ -206,7 +180,9 @@ func (handler *TxHandler) backlogWorker() { if !ok { return } - handler.postProcessCheckedTxn(wi) + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + handler.postProcessCheckedTxn(txBLMsg) // restart the loop so that we could empty out the post verification queue. continue @@ -231,7 +207,9 @@ func (handler *TxHandler) backlogWorker() { // this is never happening since handler.postVerificationQueue is never closed return } - handler.postProcessCheckedTxn(wi) + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + handler.postProcessCheckedTxn(txBLMsg) case <-handler.ctx.Done(): return diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 97d43eddd2..39a793617e 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -260,7 +260,7 @@ func TestIncomingTxHandle(t *testing.T) { incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) } -// TestIncomingTxHandle checks the correctness with txn groups +// TestIncomingTxGroupHandle checks the correctness with txn groups func TestIncomingTxGroupHandle(t *testing.T) { numberOfTransactionGroups := 1000 / proto.MaxTxGroupSize incomingTxHandlerProcessing(proto.MaxTxGroupSize, numberOfTransactionGroups, t) @@ -288,7 +288,6 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t const numUsers = 100 log := logging.TestingLog(t) - log.SetLevel(logging.Warn) addresses := make([]basics.Address, numUsers) secrets := make([]*crypto.SignatureSecrets, numUsers) @@ -328,15 +327,13 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t // emulate handler.Start() without the backlog handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - handler.backlogWg.Add(1) - go handler.processTxnStreamVerifiedResults() handler.streamVerifier.Start(handler.ctx) - outChan := make(chan *txBacklogMsg, 10) + testResultChan := make(chan *txBacklogMsg, 10) wg := sync.WaitGroup{} wg.Add(1) // Make a test backlog worker, which is simiar to backlogWorker, but sends the results - // through the outChan instead of passing it to postprocessCheckedTxn + // through the testResultChan instead of passing it to postprocessCheckedTxn go func() { defer wg.Done() for { @@ -346,7 +343,10 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t if !ok { return } - outChan <- wi + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg + // restart the loop so that we could empty out the post verification queue. continue default: @@ -367,7 +367,9 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t if !ok { return } - outChan <- wi + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg case <-handler.ctx.Done(): return @@ -406,7 +408,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t timer := time.NewTicker(250 * time.Millisecond) for { select { - case wi := <-outChan: + case wi := <-testResultChan: txnCounter = txnCounter + len(wi.unverifiedTxGroup) groupCounter++ u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) @@ -573,13 +575,11 @@ func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *t // emulate handler.Start() without the backlog handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - handler.backlogWg.Add(1) - go handler.processTxnStreamVerifiedResults() handler.streamVerifier.Start(handler.ctx) // Prepare the transactions signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) - outChan := handler.postVerificationQueue + testResultChan := handler.postVerificationQueue wg := sync.WaitGroup{} var tt time.Time @@ -598,12 +598,12 @@ func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *t b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) } }() - for wi := range outChan { - txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) + for wi := range testResultChan { + txnCounter = txnCounter + uint64(len(wi.TxnGroup)) groupCounter++ - u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + u, _ := binary.Uvarint(wi.TxnGroup[0].Txn.Note) _, inBad := badTxnGroups[u] - if wi.verificationErr == nil { + if wi.Err == nil { require.False(b, inBad, "No error for invalid signature") } else { invalidCounter++ From f83101b839c9cb0c1d9439bdb90dd6ecce032c6f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 23 Nov 2022 02:37:48 -0500 Subject: [PATCH 114/156] add benchmark with backlog worker --- data/txHandler_test.go | 220 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 213 insertions(+), 7 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 39a793617e..fe0d29b96b 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -282,9 +282,11 @@ func TestIncomingTxHandleDrops(t *testing.T) { // incomingTxHandlerProcessing is a comprehensive transaction handling test // It handles the singed transactions by passing them to the backlog for verification func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t *testing.T) { - // reset the counters - transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) - transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + defer func() { + // reset the counters + transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) + transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + }() const numUsers = 100 log := logging.TestingLog(t) @@ -346,7 +348,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t txBLMsg := wi.BacklogMessage.(*txBacklogMsg) txBLMsg.verificationErr = wi.Err testResultChan <- txBLMsg - + // restart the loop so that we could empty out the post verification queue. continue default: @@ -420,8 +422,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t require.True(t, inBad, "Error for good signature") } case <-timer.C: - droppedBacklog, droppedPool, err = getDropped() - require.NoError(t, err) + droppedBacklog, droppedPool = getDropped() if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { // all the benchmark txns processed return @@ -439,7 +440,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t wg.Wait() } -func getDropped() (droppedBacklog, droppedPool uint64, err error) { +func getDropped() (droppedBacklog, droppedPool uint64) { droppedBacklog = transactionMessagesDroppedFromBacklog.GetValue() droppedPool = transactionMessagesDroppedFromPool.GetValue() return @@ -530,6 +531,30 @@ func BenchmarkHandleTxnGroups(b *testing.B) { } } +// BenchmarkBacklogWorkerHandleTxns sends singed transactions the the verifier +func BenchmarkBacklogWorkerHandleTxns(b *testing.B) { + maxGroupSize := 1 + tpss := []int{6000000, 600000, 60000, 6000} + for _, tps := range tpss { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + rateAdjuster := time.Second / time.Duration(tps) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b) + }) + } +} + +// BenchmarkBacklogWorkerHandleTxnGroups sends singed transaction groups to the verifier +func BenchmarkBacklogWorkerHandleTxnGroups(b *testing.B) { + maxGroupSize := proto.MaxTxGroupSize / 2 + tpss := []int{6000000, 600000, 60000, 6000} + for _, tps := range tpss { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + rateAdjuster := time.Second / time.Duration(tps) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b) + }) + } +} + // runHandlerBenchmark has a similar workflow to incomingTxHandlerProcessing, // but bypasses the backlog, and sends the transactions directly to the verifier func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B) { @@ -709,3 +734,184 @@ func TestTxHandlerPostProcessErrorWithVerify(t *testing.T) { transactionMessagesTxnNotWellFormed.AddMetric(result) require.Len(t, result, 1) } + +// runHandlerBenchmarkWithBacklog is the same as runHandlerBenchmarkWithBacklog +// but uses the backlog worker +func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B) { + defer func() { + // reset the counters + transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) + transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + }() + + const numUsers = 100 + log := logging.TestingLog(b) + log.SetLevel(logging.Warn) + addresses := make([]basics.Address, numUsers) + secrets := make([]*crypto.SignatureSecrets, numUsers) + + // prepare the accounts + genesis := make(map[basics.Address]basics.AccountData) + for i := 0; i < numUsers; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + genesis[addr] = basics.AccountData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, + } + } + genesis[poolAddr] = basics.AccountData{ + Status: basics.NotParticipating, + MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, + } + + require.Equal(b, len(genesis), numUsers+1) + genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) + require.NoError(b, err) + + l := ledger + tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) + handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + // since Start is not called, set the context here + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + defer handler.ctxCancel() + + // emulate handler.Start() without the backlog + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + handler.streamVerifier.Start(handler.ctx) + + testResultChan := make(chan *txBacklogMsg, 10) + wg := sync.WaitGroup{} + wg.Add(1) + // Make a test backlog worker, which is simiar to backlogWorker, but sends the results + // through the testResultChan instead of passing it to postprocessCheckedTxn + go func() { + defer wg.Done() + for { + // prioritize the postVerificationQueue + select { + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg + + // restart the loop so that we could empty out the post verification queue. + continue + default: + } + + // we have no more post verification items. wait for either backlog queue item or post verification item. + select { + case wi, ok := <-handler.backlogQueue: + if !ok { + return + } + if handler.checkAlreadyCommitted(wi) { + // this is not expected during the test + continue + } + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg + + case <-handler.ctx.Done(): + return + } + } + }() + + // Prepare the transactions + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) + encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) + for _, stxngrp := range signedTransactionGroups { + data := make([]byte, 0) + for _, stxn := range stxngrp { + data = append(data, protocol.Encode(&stxn)...) + } + encodedSignedTransactionGroups = + append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) + } + + var tt time.Time + // Process the results and make sure they are correct + wg.Add(1) + go func() { + defer wg.Done() + groupCounter := uint64(0) + var txnCounter uint64 + invalidCounter := 0 + defer func() { + if txnCounter > 0 { + droppedBacklog, droppedPool := getDropped() + b.Logf("Input TPS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + b.Logf("Verified TPS: %d", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) + b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) + handler.Stop() // cancel the handler ctx + } + }() + stopChan := make(chan interface{}) + go func() { + for { + time.Sleep(200 * time.Millisecond) + droppedBacklog, droppedPool := getDropped() + if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { + // all the benchmark txns processed + close(stopChan) + return + } + } + }() + + for { + select { + case wi := <-testResultChan: + txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) + groupCounter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.verificationErr == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + if groupCounter == uint64(len(signedTransactionGroups)) { + // all the benchmark txns processed + return + } + case <-stopChan: + return + } + } + }() + + b.ResetTimer() + tt = time.Now() + for i, tg := range encodedSignedTransactionGroups { + handler.processIncomingTxn(tg) + if i == 0 { + fmt.Println(i) + } + time.Sleep(rateAdjuster) + } + wg.Wait() + handler.Stop() // cancel the handler ctx +} From 8245b96df0f7d4f93ff88a9a5a6ba8cb9be01a4b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 23 Nov 2022 16:13:56 -0500 Subject: [PATCH 115/156] renaming/refactoring --- data/transactions/verify/txn_test.go | 2 + data/txHandler_test.go | 368 ++++++++++++--------------- util/metrics/counter.go | 2 +- util/metrics/counter_test.go | 11 + 4 files changed, 170 insertions(+), 213 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index b9e0180e7a..07004eb571 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1122,6 +1122,8 @@ func TestStreamVerifierIdel(t *testing.T) { } func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { + partitiontest.PartitionTest(t) + numOfTxns := 10 txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0) mod := 1 diff --git a/data/txHandler_test.go b/data/txHandler_test.go index fe0d29b96b..29d969f5ee 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -256,18 +256,24 @@ func BenchmarkTxHandlerDecoderMsgp(b *testing.B) { // TestIncomingTxHandle checks the correctness with single txns func TestIncomingTxHandle(t *testing.T) { + partitiontest.PartitionTest(t) + numberOfTransactionGroups := 1000 incomingTxHandlerProcessing(1, numberOfTransactionGroups, t) } // TestIncomingTxGroupHandle checks the correctness with txn groups func TestIncomingTxGroupHandle(t *testing.T) { + partitiontest.PartitionTest(t) + numberOfTransactionGroups := 1000 / proto.MaxTxGroupSize incomingTxHandlerProcessing(proto.MaxTxGroupSize, numberOfTransactionGroups, t) } // TestIncomingTxHandleDrops accounts for the dropped txns when the verifier/exec pool is saturated func TestIncomingTxHandleDrops(t *testing.T) { + partitiontest.PartitionTest(t) + // use smaller backlog size to test the message drops origValue := txBacklogSize defer func() { @@ -441,8 +447,8 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t } func getDropped() (droppedBacklog, droppedPool uint64) { - droppedBacklog = transactionMessagesDroppedFromBacklog.GetValue() - droppedPool = transactionMessagesDroppedFromPool.GetValue() + droppedBacklog = transactionMessagesDroppedFromBacklog.GetUint64Value() + droppedPool = transactionMessagesDroppedFromPool.GetUint64Value() return } @@ -514,7 +520,7 @@ func BenchmarkHandleTxns(b *testing.B) { for _, tps := range tpss { b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmark(rateAdjuster, maxGroupSize, tps, b) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, false) }) } } @@ -526,38 +532,43 @@ func BenchmarkHandleTxnGroups(b *testing.B) { for _, tps := range tpss { b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmark(rateAdjuster, maxGroupSize, tps, b) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, false) }) } } // BenchmarkBacklogWorkerHandleTxns sends singed transactions the the verifier -func BenchmarkBacklogWorkerHandleTxns(b *testing.B) { +func BenchmarkHandleTxnsBacklogWorker(b *testing.B) { maxGroupSize := 1 tpss := []int{6000000, 600000, 60000, 6000} for _, tps := range tpss { b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, true) }) } } // BenchmarkBacklogWorkerHandleTxnGroups sends singed transaction groups to the verifier -func BenchmarkBacklogWorkerHandleTxnGroups(b *testing.B) { +func BenchmarkHandleTxnGroupsBacklogWorker(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 tpss := []int{6000000, 600000, 60000, 6000} for _, tps := range tpss { b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b) + runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, true) }) } } -// runHandlerBenchmark has a similar workflow to incomingTxHandlerProcessing, -// but bypasses the backlog, and sends the transactions directly to the verifier -func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B) { +// runHandlerBenchmarkWithBacklog benchmarks the number of transactions verfied or dropped +func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B, useBacklogWorker bool) { + defer func() { + // reset the counters + transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) + transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + }() + const numUsers = 100 log := logging.TestingLog(b) log.SetLevel(logging.Warn) @@ -602,51 +613,165 @@ func runHandlerBenchmark(rateAdjuster time.Duration, maxGroupSize, tps int, b *t handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) handler.streamVerifier.Start(handler.ctx) + testResultChan := make(chan *txBacklogMsg, 10) + wg := sync.WaitGroup{} + + if useBacklogWorker { + wg.Add(1) + // Make a test backlog worker, which is simiar to backlogWorker, but sends the results + // through the testResultChan instead of passing it to postprocessCheckedTxn + go func() { + defer wg.Done() + for { + // prioritize the postVerificationQueue + select { + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg + + // restart the loop so that we could empty out the post verification queue. + continue + default: + } + + // we have no more post verification items. wait for either backlog queue item or post verification item. + select { + case wi, ok := <-handler.backlogQueue: + if !ok { + return + } + if handler.checkAlreadyCommitted(wi) { + // this is not expected during the test + continue + } + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} + case wi, ok := <-handler.postVerificationQueue: + if !ok { + return + } + txBLMsg := wi.BacklogMessage.(*txBacklogMsg) + txBLMsg.verificationErr = wi.Err + testResultChan <- txBLMsg + + case <-handler.ctx.Done(): + return + } + } + }() + } + // Prepare the transactions signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) - testResultChan := handler.postVerificationQueue - wg := sync.WaitGroup{} + var encodedSignedTransactionGroups []network.IncomingMessage + if useBacklogWorker { + encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, b.N) + for _, stxngrp := range signedTransactionGroups { + data := make([]byte, 0) + for _, stxn := range stxngrp { + data = append(data, protocol.Encode(&stxn)...) + } + encodedSignedTransactionGroups = + append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) + } + } var tt time.Time // Process the results and make sure they are correct wg.Add(1) go func() { defer wg.Done() - groupCounter := 0 + groupCounter := uint64(0) var txnCounter uint64 invalidCounter := 0 defer func() { if txnCounter > 0 { + droppedBacklog, droppedPool := getDropped() b.Logf("Input TPS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) b.Logf("Verified TPS: %d", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) + b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) + handler.Stop() // cancel the handler ctx + } + }() + stopChan := make(chan interface{}) + go func() { + for { + time.Sleep(200 * time.Millisecond) + droppedBacklog, droppedPool := getDropped() + if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { + // all the benchmark txns processed + close(stopChan) + return + } } }() - for wi := range testResultChan { - txnCounter = txnCounter + uint64(len(wi.TxnGroup)) - groupCounter++ - u, _ := binary.Uvarint(wi.TxnGroup[0].Txn.Note) - _, inBad := badTxnGroups[u] - if wi.Err == nil { - require.False(b, inBad, "No error for invalid signature") - } else { - invalidCounter++ - require.True(b, inBad, "Error for good signature") + + if useBacklogWorker { + for { + select { + case wi := <-testResultChan: + txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) + groupCounter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.verificationErr == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + if groupCounter == uint64(len(signedTransactionGroups)) { + // all the benchmark txns processed + return + } + case <-stopChan: + return + } } - if groupCounter == len(signedTransactionGroups) { - // all the benchmark txns processed - break + } else { + for { + select { + case wi := <-handler.postVerificationQueue: + txnCounter = txnCounter + uint64(len(wi.TxnGroup)) + groupCounter++ + u, _ := binary.Uvarint(wi.TxnGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.Err == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + if groupCounter == uint64(len(signedTransactionGroups)) { + // all the benchmark txns processed + return + } + case <-stopChan: + return + } } + } }() b.ResetTimer() tt = time.Now() - for _, stxngrp := range signedTransactionGroups { - blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} - time.Sleep(rateAdjuster) + if useBacklogWorker { + for _, tg := range encodedSignedTransactionGroups { + handler.processIncomingTxn(tg) + time.Sleep(rateAdjuster) + } + } else { + for _, stxngrp := range signedTransactionGroups { + blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} + handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} + time.Sleep(rateAdjuster) + } } wg.Wait() handler.Stop() // cancel the handler ctx @@ -734,184 +859,3 @@ func TestTxHandlerPostProcessErrorWithVerify(t *testing.T) { transactionMessagesTxnNotWellFormed.AddMetric(result) require.Len(t, result, 1) } - -// runHandlerBenchmarkWithBacklog is the same as runHandlerBenchmarkWithBacklog -// but uses the backlog worker -func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B) { - defer func() { - // reset the counters - transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) - transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) - }() - - const numUsers = 100 - log := logging.TestingLog(b) - log.SetLevel(logging.Warn) - addresses := make([]basics.Address, numUsers) - secrets := make([]*crypto.SignatureSecrets, numUsers) - - // prepare the accounts - genesis := make(map[basics.Address]basics.AccountData) - for i := 0; i < numUsers; i++ { - secret := keypair() - addr := basics.Address(secret.SignatureVerifier) - secrets[i] = secret - addresses[i] = addr - genesis[addr] = basics.AccountData{ - Status: basics.Online, - MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, - } - } - genesis[poolAddr] = basics.AccountData{ - Status: basics.NotParticipating, - MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, - } - - require.Equal(b, len(genesis), numUsers+1) - genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) - ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) - const inMem = true - cfg := config.GetDefaultLocal() - cfg.Archival = true - ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) - require.NoError(b, err) - - l := ledger - tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) - // since Start is not called, set the context here - handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - defer handler.ctxCancel() - - // emulate handler.Start() without the backlog - handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - handler.streamVerifier.Start(handler.ctx) - - testResultChan := make(chan *txBacklogMsg, 10) - wg := sync.WaitGroup{} - wg.Add(1) - // Make a test backlog worker, which is simiar to backlogWorker, but sends the results - // through the testResultChan instead of passing it to postprocessCheckedTxn - go func() { - defer wg.Done() - for { - // prioritize the postVerificationQueue - select { - case wi, ok := <-handler.postVerificationQueue: - if !ok { - return - } - txBLMsg := wi.BacklogMessage.(*txBacklogMsg) - txBLMsg.verificationErr = wi.Err - testResultChan <- txBLMsg - - // restart the loop so that we could empty out the post verification queue. - continue - default: - } - - // we have no more post verification items. wait for either backlog queue item or post verification item. - select { - case wi, ok := <-handler.backlogQueue: - if !ok { - return - } - if handler.checkAlreadyCommitted(wi) { - // this is not expected during the test - continue - } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} - case wi, ok := <-handler.postVerificationQueue: - if !ok { - return - } - txBLMsg := wi.BacklogMessage.(*txBacklogMsg) - txBLMsg.verificationErr = wi.Err - testResultChan <- txBLMsg - - case <-handler.ctx.Done(): - return - } - } - }() - - // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) - encodedSignedTransactionGroups := make([]network.IncomingMessage, 0, b.N) - for _, stxngrp := range signedTransactionGroups { - data := make([]byte, 0) - for _, stxn := range stxngrp { - data = append(data, protocol.Encode(&stxn)...) - } - encodedSignedTransactionGroups = - append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) - } - - var tt time.Time - // Process the results and make sure they are correct - wg.Add(1) - go func() { - defer wg.Done() - groupCounter := uint64(0) - var txnCounter uint64 - invalidCounter := 0 - defer func() { - if txnCounter > 0 { - droppedBacklog, droppedPool := getDropped() - b.Logf("Input TPS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) - b.Logf("Verified TPS: %d", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) - b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) - b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) - b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) - handler.Stop() // cancel the handler ctx - } - }() - stopChan := make(chan interface{}) - go func() { - for { - time.Sleep(200 * time.Millisecond) - droppedBacklog, droppedPool := getDropped() - if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { - // all the benchmark txns processed - close(stopChan) - return - } - } - }() - - for { - select { - case wi := <-testResultChan: - txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) - groupCounter++ - u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) - _, inBad := badTxnGroups[u] - if wi.verificationErr == nil { - require.False(b, inBad, "No error for invalid signature") - } else { - invalidCounter++ - require.True(b, inBad, "Error for good signature") - } - if groupCounter == uint64(len(signedTransactionGroups)) { - // all the benchmark txns processed - return - } - case <-stopChan: - return - } - } - }() - - b.ResetTimer() - tt = time.Now() - for i, tg := range encodedSignedTransactionGroups { - handler.processIncomingTxn(tg) - if i == 0 { - fmt.Println(i) - } - time.Sleep(rateAdjuster) - } - wg.Wait() - handler.Stop() // cancel the handler ctx -} diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 86df08af0c..1dc43b8284 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -112,7 +112,7 @@ func (counter *Counter) AddMicrosecondsSince(t time.Time, labels map[string]stri } // GetValue returns the value of the counter. -func (counter *Counter) GetValue() (x uint64) { +func (counter *Counter) GetUint64Value() (x uint64) { return atomic.LoadUint64(&counter.intValue) } diff --git a/util/metrics/counter_test.go b/util/metrics/counter_test.go index ec253f150a..1db1b46511 100644 --- a/util/metrics/counter_test.go +++ b/util/metrics/counter_test.go @@ -198,3 +198,14 @@ testname{host="myhost"} 2.3 ` require.Equal(t, expected, sbOut.String()) } + +func TestGetValue(t *testing.T) { + partitiontest.PartitionTest(t) + + c := MakeCounter(MetricName{Name: "testname", Description: "testhelp"}) + require.Equal(t, uint64(0), c.GetUint64Value()) + c.Inc(nil) + require.Equal(t, uint64(1), c.GetUint64Value()) + c.Inc(nil) + require.Equal(t, uint64(2), c.GetUint64Value()) +} From 81d744a570c9430440632e3ed95a12e1ba314b3b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 23 Nov 2022 16:28:50 -0500 Subject: [PATCH 116/156] fix comment --- util/metrics/counter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 1dc43b8284..5c195b5f8a 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -111,7 +111,7 @@ func (counter *Counter) AddMicrosecondsSince(t time.Time, labels map[string]stri counter.AddUint64(uint64(time.Since(t).Microseconds()), labels) } -// GetValue returns the value of the counter. +// GetUint64Value returns the value of the counter. func (counter *Counter) GetUint64Value() (x uint64) { return atomic.LoadUint64(&counter.intValue) } From e6515a019d323d6a2c38101a7b643e68363ac963 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 24 Nov 2022 00:45:52 -0500 Subject: [PATCH 117/156] Fix race condition and add a test for the block watcher --- data/transactions/verify/txn.go | 6 ++--- data/transactions/verify/txn_test.go | 34 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 5bf498eda0..100e32a6b0 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -587,10 +587,10 @@ func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore nbw.blkHeader = block.BlockHeader } -func (nbw *NewBlockWatcher) getBlockHeader() (bh *bookkeeping.BlockHeader) { +func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { nbw.mu.RLock() defer nbw.mu.RUnlock() - return &nbw.blkHeader + return nbw.blkHeader } type batchLoad struct { @@ -801,7 +801,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []UnverifiedElem // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here blockHeader := sv.nbw.getBlockHeader() for _, ue := range uelts { - groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, blockHeader, sv.ledger, batchVerifier) + groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, &blockHeader, sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error sv.sendResult(ue.TxnGroup, ue.BacklogMessage, err) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 07004eb571..83aa5c9d98 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -35,6 +35,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" + "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/execpool" @@ -1335,3 +1336,36 @@ func TestStreamVerifierRestart(t *testing.T) { cancel2() sv.activeLoopWg.Wait() } + +// TestBlockWatcher runs multiple goroutines to check the concurency and correctness of the block watcher +func TestBlockWatcher(t *testing.T) { + blkHdr := createDummyBlockHeader() + nbw := MakeNewBlockWatcher(blkHdr) + startingRound := blkHdr.Round + + wg := sync.WaitGroup{} + + wg.Add(1) + go func() { + defer wg.Done() + for x := 1; x < 10; x++ { + blkHdr.Round++ + nbw.OnNewBlock(bookkeeping.Block{BlockHeader: blkHdr}, ledgercore.StateDelta{}) + time.Sleep(10 * time.Millisecond) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for { + bh := nbw.getBlockHeader() + if bh.Round == startingRound+10 { + break + } + } + }() + wg.Wait() + bh := nbw.getBlockHeader() + require.Equal(t, startingRound+10, bh.Round) +} From 1c663a299b55fca0c2ad82492eb892b7ef2ccb18 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 28 Nov 2022 12:37:52 -0500 Subject: [PATCH 118/156] fix test --- data/transactions/verify/txn_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 83aa5c9d98..6f4730cf10 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1348,7 +1348,7 @@ func TestBlockWatcher(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - for x := 1; x < 10; x++ { + for x := 0; x < 10; x++ { blkHdr.Round++ nbw.OnNewBlock(bookkeeping.Block{BlockHeader: blkHdr}, ledgercore.StateDelta{}) time.Sleep(10 * time.Millisecond) From 6c0b0670b7ee06bd60fd818530a07f40f1b347e1 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 28 Nov 2022 13:22:25 -0500 Subject: [PATCH 119/156] CR: addressing multiple review comments. --- data/transactions/verify/txn.go | 43 ++++++++++++++-------------- data/transactions/verify/txn_test.go | 20 ++++++------- data/txHandler.go | 10 +++---- data/txHandler_test.go | 6 ++-- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 100e32a6b0..9a89b1181e 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -551,8 +551,8 @@ type VerificationResult struct { // StreamVerifier verifies txn groups received through the stxnChan channel, and returns the // results through the resultChan type StreamVerifier struct { - resultChan chan<- VerificationResult - stxnChan <-chan UnverifiedElement + resultChan chan<- *VerificationResult + stxnChan <-chan *UnverifiedElement verificationPool execpool.BacklogPool ctx context.Context cache VerifiedTransactionCache @@ -618,7 +618,7 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from -func MakeStreamVerifier(stxnChan <-chan UnverifiedElement, resultChan chan<- VerificationResult, +func MakeStreamVerifier(stxnChan <-chan *UnverifiedElement, resultChan chan<- *VerificationResult, ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, cache VerifiedTransactionCache, droppedFromPool *metrics.Counter) (sv *StreamVerifier) { @@ -642,10 +642,10 @@ func (sv *StreamVerifier) Start(ctx context.Context) { go sv.batchingLoop() } -func (sv *StreamVerifier) cleanup(pending *[]UnverifiedElement) { +func (sv *StreamVerifier) cleanup(pending []*UnverifiedElement) { // report an error for the unchecked txns // drop the messages without reporting if the receiver does not consume - for _, uel := range *pending { + for _, uel := range pending { sv.sendResult(uel.TxnGroup, uel.BacklogMessage, errShuttingDownError) } } @@ -656,8 +656,8 @@ func (sv *StreamVerifier) batchingLoop() { var added bool var numberOfSigsInCurrent uint64 var numberOfTimerResets uint64 - uelts := make([]UnverifiedElement, 0) - defer sv.cleanup(&uelts) + uelts := make([]*UnverifiedElement, 0) + defer func() { sv.cleanup(uelts) }() for { select { case stx := <-sv.stxnChan: @@ -671,7 +671,7 @@ func (sv *StreamVerifier) batchingLoop() { // if no batchable signatures here, send this as a task of its own if numberOfBatchableSigsInGroup == 0 { - err := sv.addVerificationTaskToThePoolNow([]UnverifiedElement{stx}) + err := sv.addVerificationTaskToThePoolNow([]*UnverifiedElement{stx}) if err != nil { return } @@ -701,7 +701,7 @@ func (sv *StreamVerifier) batchingLoop() { } if added { numberOfSigsInCurrent = 0 - uelts = make([]UnverifiedElement, 0) + uelts = make([]*UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 @@ -740,7 +740,7 @@ func (sv *StreamVerifier) batchingLoop() { } if added { numberOfSigsInCurrent = 0 - uelts = make([]UnverifiedElement, 0) + uelts = make([]*UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 @@ -756,14 +756,13 @@ func (sv *StreamVerifier) batchingLoop() { } func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBacklogMessage interface{}, err error) { - vr := VerificationResult{ + // send the txn result out the pipe + select { + case sv.resultChan <- &VerificationResult{ TxnGroup: veTxnGroup, BacklogMessage: veBacklogMessage, Err: err, - } - // send the txn result out the pipe - select { - case sv.resultChan <- vr: + }: default: // we failed to write to the output queue, since the queue was full. // adding the metric here allows us to monitor how frequently it happens. @@ -771,12 +770,12 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack } } -func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []UnverifiedElement) (added bool, err error) { +func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []*UnverifiedElement) (added bool, err error) { // if the exec pool buffer is (half) full, can go back and collect // more signatures instead of waiting in the exec pool buffer // more signatures to the batch do not harm performance but introduce latency when delayed (see crypto.BenchmarkBatchVerifierBig) - // if buffer is half full + // if the buffer is full if l, c := sv.verificationPool.BufferLength(); l == c { return false, nil } @@ -788,13 +787,13 @@ func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []UnverifiedElem return true, nil } -func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []UnverifiedElement) error { +func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []*UnverifiedElement) error { function := func(arg interface{}) interface{} { if sv.ctx.Err() != nil { return nil } - uelts := arg.([]UnverifiedElement) + uelts := arg.([]*UnverifiedElement) batchVerifier := crypto.MakeBatchVerifier() bl := makeBatchLoad() @@ -868,7 +867,7 @@ func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs u return } -func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (batchSigs uint64, err error) { +func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) { var hasSig, hasMsig bool numSigs := 0 if stx.Sig != (crypto.Signature{}) { @@ -901,11 +900,13 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (batchSigs uint6 } if hasMsig { sig := stx.Msig + batchSigs := uint64(0) for _, subsigi := range sig.Subsigs { if (subsigi.Sig != crypto.Signature{}) { batchSigs++ } } + return batchSigs, nil } - return + return 0, nil } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 6f4730cf10..cc15e21c87 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -885,8 +885,8 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) - stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult, txBacklogSize) + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) sv = MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) sv.Start(ctx) @@ -904,7 +904,7 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m go func() { defer wg.Done() for _, tg := range txnGroups { - stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} + stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} } }() @@ -918,7 +918,7 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m return sv } -func processResults(ctx context.Context, errChan chan<- error, resultChan <-chan VerificationResult, +func processResults(ctx context.Context, errChan chan<- error, resultChan <-chan *VerificationResult, numOfTxnGroups int, badTxnGroups map[uint64]struct{}, badSigResultCounter, goodSigResultCounter *int, wg *sync.WaitGroup) { defer wg.Done() @@ -1235,8 +1235,8 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) - stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult, txBacklogSize) + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) sv.Start(ctx) @@ -1265,7 +1265,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { select { case <-ctx.Done(): break - case stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: } } }() @@ -1290,8 +1290,8 @@ func TestStreamVerifierRestart(t *testing.T) { blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) - stxnChan := make(chan UnverifiedElement) - resultChan := make(chan VerificationResult, txBacklogSize) + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) ctx, cancel := context.WithCancel(context.Background()) sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) @@ -1322,7 +1322,7 @@ func TestStreamVerifierRestart(t *testing.T) { select { case <-ctx2.Done(): break - case stxnChan <- UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: } } cancel() diff --git a/data/txHandler.go b/data/txHandler.go index e823f16183..3e8566031a 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -78,13 +78,13 @@ type TxHandler struct { genesisHash crypto.Digest txVerificationPool execpool.BacklogPool backlogQueue chan *txBacklogMsg - postVerificationQueue chan verify.VerificationResult + postVerificationQueue chan *verify.VerificationResult backlogWg sync.WaitGroup net network.GossipNode ctx context.Context ctxCancel context.CancelFunc streamVerifier *verify.StreamVerifier - streamVerifierChan chan verify.UnverifiedElement + streamVerifierChan chan *verify.UnverifiedElement } // MakeTxHandler makes a new handler for transaction messages @@ -107,9 +107,9 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go ledger: ledger, txVerificationPool: executionPool, backlogQueue: make(chan *txBacklogMsg, txBacklogSize), - postVerificationQueue: make(chan verify.VerificationResult, txBacklogSize), + postVerificationQueue: make(chan *verify.VerificationResult, txBacklogSize), net: net, - streamVerifierChan: make(chan verify.UnverifiedElement), + streamVerifierChan: make(chan *verify.UnverifiedElement), } // prepare the transaction stream verifer @@ -200,7 +200,7 @@ func (handler *TxHandler) backlogWorker() { transactionMessagesAlreadyCommitted.Inc(nil) continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} + handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 29d969f5ee..eac8989fc5 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -370,7 +370,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t // this is not expected during the test continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} + handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { return @@ -648,7 +648,7 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp // this is not expected during the test continue } - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} + handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} case wi, ok := <-handler.postVerificationQueue: if !ok { return @@ -769,7 +769,7 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp } else { for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} - handler.streamVerifierChan <- verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} + handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: stxngrp, BacklogMessage: &blm} time.Sleep(rateAdjuster) } } From 1b638c7fe20e9c59b522d09ca47c1ebf0b1b0f72 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 29 Nov 2022 15:03:17 -0500 Subject: [PATCH 120/156] CR: comments/renaming --- data/transactions/verify/txn.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 9a89b1181e..8b51680472 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -869,20 +869,20 @@ func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs u func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) { var hasSig, hasMsig bool - numSigs := 0 + numSigCategories := 0 if stx.Sig != (crypto.Signature{}) { - numSigs++ + numSigCategories++ hasSig = true } if !stx.Msig.Blank() { - numSigs++ + numSigCategories++ hasMsig = true } if !stx.Lsig.Blank() { - numSigs++ + numSigCategories++ } - if numSigs == 0 { + if numSigCategories == 0 { // Special case: special sender address can issue special transaction // types (state proof txn) without any signature. The well-formed // check ensures that this transaction cannot pay any fee, and @@ -892,7 +892,7 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) } return 0, errSignedTxnHasNoSig } - if numSigs != 1 { + if numSigCategories != 1 { return 0, errSignedTxnMaxOneSig } if hasSig { @@ -908,5 +908,6 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) } return batchSigs, nil } + // This is the Lsig case. Currently the sigs in here are not batched. Something to consider later. return 0, nil } From b6aa2ae130a58092234746d66a7c264ccb787f21 Mon Sep 17 00:00:00 2001 From: chris erway Date: Wed, 30 Nov 2022 08:04:56 +0400 Subject: [PATCH 121/156] introduce LedgerForStreamVerifier and remove Fatal --- data/ledger.go | 2 +- data/transactions/verify/txn.go | 26 +++++++++++++++++++++----- data/txHandler.go | 25 +++++++++---------------- ledger/ledger.go | 2 +- ledger/ledgercore/statedelta.go | 5 +++++ ledger/notifier.go | 9 ++------- node/node.go | 10 +++++++--- 7 files changed, 46 insertions(+), 33 deletions(-) diff --git a/data/ledger.go b/data/ledger.go index 101da721af..75150f2229 100644 --- a/data/ledger.go +++ b/data/ledger.go @@ -80,7 +80,7 @@ type roundSeed struct { func LoadLedger( log logging.Logger, dbFilenamePrefix string, memory bool, genesisProto protocol.ConsensusVersion, genesisBal bookkeeping.GenesisBalances, genesisID string, genesisHash crypto.Digest, - blockListeners []ledger.BlockListener, cfg config.Local, + blockListeners []ledgercore.BlockListener, cfg config.Local, ) (*Ledger, error) { if genesisBal.Balances == nil { genesisBal.Balances = make(map[basics.Address]basics.AccountData) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a8b2d7275f..63d822f2ff 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -648,13 +648,30 @@ func (bl *batchLoad) addLoad(txngrp []transactions.SignedTxn, gctx *GroupContext } +// LedgerForStreamVerifier defines the ledger methods used by the StreamVerifier. +type LedgerForStreamVerifier interface { + logic.LedgerForSignature + RegisterBlockListeners([]ledgercore.BlockListener) + Latest() basics.Round + BlockHdr(rnd basics.Round) (blk bookkeeping.BlockHeader, err error) +} + // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from func MakeStreamVerifier(stxnChan <-chan *UnverifiedElement, resultChan chan<- *VerificationResult, - ledger logic.LedgerForSignature, nbw *NewBlockWatcher, verificationPool execpool.BacklogPool, - cache VerifiedTransactionCache, droppedFromPool *metrics.Counter) (sv *StreamVerifier) { + ledger LedgerForStreamVerifier, verificationPool execpool.BacklogPool, + cache VerifiedTransactionCache, droppedFromPool *metrics.Counter) (*StreamVerifier, error) { + + latest := ledger.Latest() + latestHdr, err := ledger.BlockHdr(latest) + if err != nil { + return nil, errors.New("MakeStreamVerifier: Could not get header for previous block") + } + + nbw := MakeNewBlockWatcher(latestHdr) + ledger.RegisterBlockListeners([]ledgercore.BlockListener{nbw}) - sv = &StreamVerifier{ + return &StreamVerifier{ resultChan: resultChan, stxnChan: stxnChan, verificationPool: verificationPool, @@ -662,8 +679,7 @@ func MakeStreamVerifier(stxnChan <-chan *UnverifiedElement, resultChan chan<- *V nbw: nbw, ledger: ledger, droppedFromPool: droppedFromPool, - } - return sv + }, nil } // Start is called when the verifier is created and whenever it needs to restart after diff --git a/data/txHandler.go b/data/txHandler.go index 8e2410a116..8851251d97 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -31,7 +31,6 @@ import ( "github.com/algorand/go-algorand/data/pools" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/verify" - ledgerpkg "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" @@ -91,16 +90,14 @@ type TxHandler struct { } // MakeTxHandler makes a new handler for transaction messages -func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.GossipNode, genesisID string, genesisHash crypto.Digest, executionPool execpool.BacklogPool) *TxHandler { +func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.GossipNode, genesisID string, genesisHash crypto.Digest, executionPool execpool.BacklogPool) (*TxHandler, error) { if txPool == nil { - logging.Base().Fatal("MakeTxHandler: txPool is nil on initialization") - return nil + return nil, errors.New("MakeTxHandler: txPool is nil on initialization") } if ledger == nil { - logging.Base().Fatal("MakeTxHandler: ledger is nil on initialization") - return nil + return nil, errors.New("MakeTxHandler: ledger is nil on initialization") } handler := &TxHandler{ @@ -116,18 +113,14 @@ func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.Go } // prepare the transaction stream verifer - latest := handler.ledger.Latest() - latestHdr, err := handler.ledger.BlockHdr(latest) + var err error + handler.streamVerifier, err = verify.MakeStreamVerifier(handler.streamVerifierChan, + handler.postVerificationQueue, handler.ledger, handler.txVerificationPool, + handler.ledger.VerifiedTransactionCache(), transactionMessagesDroppedFromPool) if err != nil { - logging.Base().Fatal("MakeTxHandler: Could not get header for previous block") - return nil + return nil, err } - nbw := verify.MakeNewBlockWatcher(latestHdr) - handler.ledger.RegisterBlockListeners([]ledgerpkg.BlockListener{nbw}) - handler.streamVerifier = verify.MakeStreamVerifier(handler.streamVerifierChan, - handler.postVerificationQueue, handler.ledger, nbw, handler.txVerificationPool, - handler.ledger.VerifiedTransactionCache(), transactionMessagesDroppedFromPool) - return handler + return handler, nil } // Start enables the processing of incoming messages at the transaction handler diff --git a/ledger/ledger.go b/ledger/ledger.go index 09ec1b3cd3..8ccf970ec9 100644 --- a/ledger/ledger.go +++ b/ledger/ledger.go @@ -398,7 +398,7 @@ func (l *Ledger) Close() { // RegisterBlockListeners registers listeners that will be called when a // new block is added to the ledger. -func (l *Ledger) RegisterBlockListeners(listeners []BlockListener) { +func (l *Ledger) RegisterBlockListeners(listeners []ledgercore.BlockListener) { l.notifier.register(listeners) } diff --git a/ledger/ledgercore/statedelta.go b/ledger/ledgercore/statedelta.go index 688c8d093b..887ab81c98 100644 --- a/ledger/ledgercore/statedelta.go +++ b/ledger/ledgercore/statedelta.go @@ -32,6 +32,11 @@ const ( stateDeltaTargetOptimizationThreshold = uint64(50000000) ) +// BlockListener represents an object that needs to get notified on new blocks. +type BlockListener interface { + OnNewBlock(block bookkeeping.Block, delta StateDelta) +} + // ModifiedCreatable defines the changes to a single single creatable state type ModifiedCreatable struct { // Type of the creatable: app or asset diff --git a/ledger/notifier.go b/ledger/notifier.go index b7d220ac32..b34aeb4728 100644 --- a/ledger/notifier.go +++ b/ledger/notifier.go @@ -28,11 +28,6 @@ import ( "github.com/algorand/go-algorand/ledger/ledgercore" ) -// BlockListener represents an object that needs to get notified on new blocks. -type BlockListener interface { - OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) -} - type blockDeltaPair struct { block bookkeeping.Block delta ledgercore.StateDelta @@ -41,7 +36,7 @@ type blockDeltaPair struct { type blockNotifier struct { mu deadlock.Mutex cond *sync.Cond - listeners []BlockListener + listeners []ledgercore.BlockListener pendingBlocks []blockDeltaPair running bool // closing is the waitgroup used to synchronize closing the worker goroutine. It's being increased during loadFromDisk, and the worker is responsible to call Done on it once it's aborting it's goroutine. The close function waits on this to complete. @@ -96,7 +91,7 @@ func (bn *blockNotifier) loadFromDisk(l ledgerForTracker, _ basics.Round) error return nil } -func (bn *blockNotifier) register(listeners []BlockListener) { +func (bn *blockNotifier) register(listeners []ledgercore.BlockListener) { bn.mu.Lock() defer bn.mu.Unlock() diff --git a/node/node.go b/node/node.go index dc1fffee98..cb6f8ec17d 100644 --- a/node/node.go +++ b/node/node.go @@ -213,7 +213,7 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd node.cryptoPool = execpool.MakePool(node) node.lowPriorityCryptoVerificationPool = execpool.MakeBacklog(node.cryptoPool, 2*node.cryptoPool.GetParallelism(), execpool.LowPriority, node) node.highPriorityCryptoVerificationPool = execpool.MakeBacklog(node.cryptoPool, 2*node.cryptoPool.GetParallelism(), execpool.HighPriority, node) - node.ledger, err = data.LoadLedger(node.log, ledgerPathnamePrefix, false, genesis.Proto, genalloc, node.genesisID, node.genesisHash, []ledger.BlockListener{}, cfg) + node.ledger, err = data.LoadLedger(node.log, ledgerPathnamePrefix, false, genesis.Proto, genalloc, node.genesisID, node.genesisHash, []ledgercore.BlockListener{}, cfg) if err != nil { log.Errorf("Cannot initialize ledger (%s): %v", ledgerPathnamePrefix, err) return nil, err @@ -221,7 +221,7 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd node.transactionPool = pools.MakeTransactionPool(node.ledger.Ledger, cfg, node.log) - blockListeners := []ledger.BlockListener{ + blockListeners := []ledgercore.BlockListener{ node.transactionPool, node, } @@ -230,7 +230,11 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd blockListeners = append(blockListeners, &accountListener) } node.ledger.RegisterBlockListeners(blockListeners) - node.txHandler = data.MakeTxHandler(node.transactionPool, node.ledger, node.net, node.genesisID, node.genesisHash, node.lowPriorityCryptoVerificationPool) + node.txHandler, err = data.MakeTxHandler(node.transactionPool, node.ledger, node.net, node.genesisID, node.genesisHash, node.lowPriorityCryptoVerificationPool) + if err != nil { + log.Errorf("Cannot initialize TxHandler: %v", err) + return nil, err + } // Indexer setup if cfg.IsIndexerActive && cfg.Archival { From 59eb8a359a2abea31b0240a87e7d814b841b6534 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 30 Nov 2022 17:05:33 -0500 Subject: [PATCH 122/156] fix tests after the changes merged from Chris --- data/transactions/verify/txn.go | 2 +- data/transactions/verify/txn_test.go | 159 +++++++++++++++++++++++++-- data/txHandler_test.go | 9 +- 3 files changed, 157 insertions(+), 13 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 63d822f2ff..2040968016 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -64,7 +64,7 @@ const txnPerWorksetThreshold = 32 // batchSizeBlockLimit is the limit when the batch exceeds, will be added to the exec pool, even if the pool is saturated // and the batch verifier will block until the exec pool accepts the batch -const batchSizeBlockLimit = 512 +const batchSizeBlockLimit = 1024 // waitForNextTxnDuration is the time to wait before sending the batch to the exec pool // If the incoming txn rate is low, a txn in the batch may wait no less than diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index cc15e21c87..378a2117b3 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -76,6 +76,14 @@ type DummyLedgerForSignature struct { func (d *DummyLedgerForSignature) BlockHdrCached(basics.Round) (bookkeeping.BlockHeader, error) { return createDummyBlockHeader(), nil } +func (d *DummyLedgerForSignature) BlockHdr(rnd basics.Round) (blk bookkeeping.BlockHeader, err error) { + return createDummyBlockHeader(), nil +} +func (d *DummyLedgerForSignature) Latest() basics.Round { + return 0 +} +func (d *DummyLedgerForSignature) RegisterBlockListeners([]ledgercore.BlockListener) { +} func keypair() *crypto.SignatureSecrets { var seed crypto.Seed @@ -883,11 +891,10 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m defer cancel() - blkHdr := createDummyBlockHeader() - nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv = MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) sv.Start(ctx) wg := sync.WaitGroup{} @@ -1233,11 +1240,10 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50000) - blkHdr := createDummyBlockHeader() - nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) sv.Start(ctx) errChan := make(chan error) @@ -1288,13 +1294,12 @@ func TestStreamVerifierRestart(t *testing.T) { cache := MakeVerifiedTransactionCache(50000) - blkHdr := createDummyBlockHeader() - nbw := MakeNewBlockWatcher(blkHdr) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) ctx, cancel := context.WithCancel(context.Background()) - sv := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, nbw, verificationPool, cache, droppedFromPool) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) sv.Start(ctx) errChan := make(chan error) @@ -1352,6 +1357,7 @@ func TestBlockWatcher(t *testing.T) { blkHdr.Round++ nbw.OnNewBlock(bookkeeping.Block{BlockHeader: blkHdr}, ledgercore.StateDelta{}) time.Sleep(10 * time.Millisecond) + nbw.OnNewBlock(bookkeeping.Block{BlockHeader: blkHdr}, ledgercore.StateDelta{}) } }() @@ -1369,3 +1375,138 @@ func TestBlockWatcher(t *testing.T) { bh := nbw.getBlockHeader() require.Equal(t, startingRound+10, bh.Round) } + +/* +// TestStreamVerifierCtxCancel tests the termination when the ctx is canceled +func TestStreamVerifierCtxCancel(t *testing.T) { + partitiontest.PartitionTest(t) + + numOfTxns := batchSizeBlockLimit * 2 + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) + + // prepare the stream verifier + execPool := execpool.MakePool(t) + backlogQueueSize := 4 + verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) + _, buffLen := verificationPool.BufferLength() + + // make the buffer full to control when the tasks get executed (after canceling the ctx) + holdTasks := make(chan interface{}) + for x := 0; x < buffLen+runtime.NumCPU() + 0; x++ { + verificationPool.EnqueueBacklog(context.Background(), + func(arg interface{}) interface{} { <-holdTasks; return nil }, nil, nil) + } + + ctx, cancel := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50000) + + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) + sv.Start(ctx) + + // As the tasks are added, some will be queued in the exec pool, others in the batch preperation loop + // verifierBlocked := false + // timer := time.NewTicker(time.Millisecond * 5) + for i, tg := range txnGroups { + fmt.Println(i) + stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} + // select { + // case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + // case <-timer.C: + // verifierBlocked = true + // } + // timer.Reset(time.Millisecond * 5) + // if verifierBlocked { + // the batch verifier is no longer able to accept new sigs + // the exec pool is full, and the current batch has batchSizeBlockLimit txns + // this is the assumption. 5ms might not be enough and the loop might terminate + // sooner, but the test should still behave + // break + // } + } + + resultReceived := false + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + // no verification tasks should be executed + // at least one result (the batch held before the cancel, should be returned + for res := range resultChan { + require.ErrorIs(t, res.Err, errShuttingDownError) + resultReceived = true + return + } + }() + + // cancel the ctx as the tasks are waiting in the exc pool + cancel() + + // the main loop should stop after cancel() + sv.activeLoopWg.Wait() + + // release the tasks + close(holdTasks) + + wg.Wait() + require.True(t, resultReceived) +} + +// TestStreamVerifierPostVBlocked tests what happens when the ctx canceled when the task is in exec pool +func TestStreamVerifierPostVBlocked(t *testing.T) { + + // prepare the stream verifier + execPool := execpool.MakePool(t) + backlogQueueSize := 64 + verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) + errChan := make(chan error) + var badSigResultCounter int + var goodSigResultCounter int + + ctx, _ := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50000) + + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) + + // start the verifier + sv.Start(ctx) + //xxx + // send transactions to overflow the result chan + numOfTxns := txBacklogSize + 1 + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) + numOfTxnGroups := len(txnGroups) + for _, tg := range txnGroups { + select { + case <-ctx.Done(): + break + case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: + } + } + + // wait until the one extra transaction is dropped + for w := 0; w < 100; w++ { + droppedPool := sv.droppedFromPool.GetUint64Value() + if droppedPool > 0 { // there could be more from the previous iteration + break + } + time.Sleep(time.Millisecond * 5) + } + + wg := sync.WaitGroup{} + wg.Add(1) + // make sure the other results are fine + go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) + + for err := range errChan { + require.ErrorIs(t, err, errShuttingDownError) + fmt.Println(badTxnGroups) + } + + wg.Wait() +} +*/ diff --git a/data/txHandler_test.go b/data/txHandler_test.go index eac8989fc5..0d98daac2a 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -84,7 +84,8 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { cfg.EnableProcessBlockStats = false tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - txHandler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + txHandler, err := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + require.NoError(b, err) makeTxns := func(N int) [][]transactions.SignedTxn { ret := make([][]transactions.SignedTxn, 0, N) @@ -328,7 +329,8 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t l := ledger tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + handler, err := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + require.NoError(t, err) // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) defer handler.ctxCancel() @@ -604,7 +606,8 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp l := ledger tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - handler := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + handler, err := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + require.NoError(b, err) // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) defer handler.ctxCancel() From 9895d6e767981703d672ae9b41e4afec9024a6d2 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 1 Dec 2022 00:12:35 -0500 Subject: [PATCH 123/156] cr: store block hdr as pointer for new block watcher. add more tests for edge cases --- data/transactions/verify/txn.go | 10 +- data/transactions/verify/txn_test.go | 193 ++++++++++++++++++--------- 2 files changed, 136 insertions(+), 67 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 2040968016..09c46ef666 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -597,14 +597,14 @@ type StreamVerifier struct { // NewBlockWatcher is a struct used to provide a new block header to the // stream verifier type NewBlockWatcher struct { - blkHeader bookkeeping.BlockHeader + blkHeader *bookkeeping.BlockHeader mu deadlock.RWMutex } // MakeNewBlockWatcher construct a new block watcher with the initial blkHdr func MakeNewBlockWatcher(blkHdr bookkeeping.BlockHeader) (nbw *NewBlockWatcher) { nbw = &NewBlockWatcher{ - blkHeader: blkHdr, + blkHeader: &blkHdr, } return nbw } @@ -616,10 +616,10 @@ func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore } nbw.mu.Lock() defer nbw.mu.Unlock() - nbw.blkHeader = block.BlockHeader + nbw.blkHeader = &block.BlockHeader } -func (nbw *NewBlockWatcher) getBlockHeader() (bh bookkeeping.BlockHeader) { +func (nbw *NewBlockWatcher) getBlockHeader() (bh *bookkeeping.BlockHeader) { nbw.mu.RLock() defer nbw.mu.RUnlock() return nbw.blkHeader @@ -848,7 +848,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []*UnverifiedEle // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here blockHeader := sv.nbw.getBlockHeader() for _, ue := range uelts { - groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, &blockHeader, sv.ledger, batchVerifier) + groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, blockHeader, sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error sv.sendResult(ue.TxnGroup, ue.BacklogMessage, err) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 378a2117b3..4ab742e144 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -71,12 +71,16 @@ func verifyTxn(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext) er } type DummyLedgerForSignature struct { + badHdr bool } func (d *DummyLedgerForSignature) BlockHdrCached(basics.Round) (bookkeeping.BlockHeader, error) { return createDummyBlockHeader(), nil } func (d *DummyLedgerForSignature) BlockHdr(rnd basics.Round) (blk bookkeeping.BlockHeader, err error) { + if d.badHdr { + return bookkeeping.BlockHeader{}, fmt.Errorf("test error block hdr") + } return createDummyBlockHeader(), nil } func (d *DummyLedgerForSignature) Latest() basics.Round { @@ -1343,17 +1347,18 @@ func TestStreamVerifierRestart(t *testing.T) { } // TestBlockWatcher runs multiple goroutines to check the concurency and correctness of the block watcher -func TestBlockWatcher(t *testing.T) { +func TestStreamVerifierBlockWatcher(t *testing.T) { blkHdr := createDummyBlockHeader() nbw := MakeNewBlockWatcher(blkHdr) startingRound := blkHdr.Round wg := sync.WaitGroup{} + count := 100 wg.Add(1) go func() { defer wg.Done() - for x := 0; x < 10; x++ { + for x := 0; x < 100; x++ { blkHdr.Round++ nbw.OnNewBlock(bookkeeping.Block{BlockHeader: blkHdr}, ledgercore.StateDelta{}) time.Sleep(10 * time.Millisecond) @@ -1361,11 +1366,13 @@ func TestBlockWatcher(t *testing.T) { } }() + bhStore := make(map[basics.Round]*bookkeeping.BlockHeader) wg.Add(1) go func() { defer wg.Done() for { bh := nbw.getBlockHeader() + bhStore[bh.Round] = bh if bh.Round == startingRound+10 { break } @@ -1373,75 +1380,114 @@ func TestBlockWatcher(t *testing.T) { }() wg.Wait() bh := nbw.getBlockHeader() - require.Equal(t, startingRound+10, bh.Round) + require.Equal(t, uint64(startingRound)+uint64(count), uint64(bh.Round)) + // There should be no inconsistency after new blocks are added + for r, bh := range bhStore { + require.Equal(t, r, bh.Round) + } } -/* -// TestStreamVerifierCtxCancel tests the termination when the ctx is canceled -func TestStreamVerifierCtxCancel(t *testing.T) { - partitiontest.PartitionTest(t) - - numOfTxns := batchSizeBlockLimit * 2 - txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) - - // prepare the stream verifier +func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}) { execPool := execpool.MakePool(t) backlogQueueSize := 4 verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) _, buffLen := verificationPool.BufferLength() - // make the buffer full to control when the tasks get executed (after canceling the ctx) + // make the buffer full to control when the tasks get executed holdTasks := make(chan interface{}) - for x := 0; x < buffLen+runtime.NumCPU() + 0; x++ { + for x := 0; x < buffLen+runtime.NumCPU()+1; x++ { verificationPool.EnqueueBacklog(context.Background(), - func(arg interface{}) interface{} { <-holdTasks; return nil }, nil, nil) + func(arg interface{}) interface{} { + <-holdTasks + return nil + }, nil, nil) } + return verificationPool, holdTasks +} - ctx, cancel := context.WithCancel(context.Background()) - cache := MakeVerifiedTransactionCache(50000) +// TestStreamVerifierCtxCancel tests the termination when the ctx is canceled +// To make sure that the batchingLoop is still working on a batch when the +// ctx is cancled, this test first saturates the exec pool buffer, then +// sends a txn and immediately cancels the ctx so that the batch is not +// passed to the exec pool yet, but is in batchingLoop +func TestStreamVerifierCtxCancel(t *testing.T) { + partitiontest.PartitionTest(t) + + verificationPool, holdTasks := getSaturatedExecPool(t) + ctx, cancel := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) require.NoError(t, err) sv.Start(ctx) - // As the tasks are added, some will be queued in the exec pool, others in the batch preperation loop - // verifierBlocked := false - // timer := time.NewTicker(time.Millisecond * 5) - for i, tg := range txnGroups { - fmt.Println(i) - stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} - // select { - // case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: - // case <-timer.C: - // verifierBlocked = true - // } - // timer.Reset(time.Millisecond * 5) - // if verifierBlocked { - // the batch verifier is no longer able to accept new sigs - // the exec pool is full, and the current batch has batchSizeBlockLimit txns - // this is the assumption. 5ms might not be enough and the loop might terminate - // sooner, but the test should still behave - // break - // } - } + var result *VerificationResult + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + // no verification tasks should be executed + // one result should be returned + result = <-resultChan + }() + + // send batchSizeBlockLimit after the exec pool buffer is full + numOfTxns := 1 + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) + stxnChan <- &UnverifiedElement{TxnGroup: txnGroups[0], BacklogMessage: nil} + // cancel the ctx before the sig is sent to the exec pool + cancel() + + // the main loop should stop after cancel() + sv.activeLoopWg.Wait() + + // release the tasks + close(holdTasks) + + wg.Wait() + require.ErrorIs(t, result.Err, errShuttingDownError) +} - resultReceived := false +// TestStreamVerifierCtxCancelPoolQueue tests the termination when the ctx is canceled +// To make sure that the batchingLoop is still working on a batch when the +// ctx is cancled, this test first saturates the exec pool buffer, then +// sends a txn and cancels the ctx after multiple waitForNextTxnDuration +// so that the batch is sent to the pool. Since the pool is saturated, +// the task will be stuck waiting to be queued when the context is canceled +// everything should be gracefully terminated +func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { + partitiontest.PartitionTest(t) + + verificationPool, holdTasks := getSaturatedExecPool(t) + + ctx, cancel := context.WithCancel(context.Background()) + cache := MakeVerifiedTransactionCache(50) + stxnChan := make(chan *UnverifiedElement) + resultChan := make(chan *VerificationResult, txBacklogSize) + sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + require.NoError(t, err) + sv.Start(ctx) + + var result *VerificationResult wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() // no verification tasks should be executed - // at least one result (the batch held before the cancel, should be returned - for res := range resultChan { - require.ErrorIs(t, res.Err, errShuttingDownError) - resultReceived = true - return - } + // one result should be returned + result = <-resultChan }() - // cancel the ctx as the tasks are waiting in the exc pool + // send batchSizeBlockLimit after the exec pool buffer is full + numOfTxns := 1 + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) + stxnChan <- &UnverifiedElement{TxnGroup: txnGroups[0], BacklogMessage: nil} + // cancel the ctx as the sig is not yet sent to the exec pool + // the test might sporadically fail if between sending the txn above + // and the cancelation, 2 x waitForNextTxnDuration elapses (10ms) + time.Sleep(6 * waitForNextTxnDuration) cancel() // the main loop should stop after cancel() @@ -1451,10 +1497,11 @@ func TestStreamVerifierCtxCancel(t *testing.T) { close(holdTasks) wg.Wait() - require.True(t, resultReceived) + require.ErrorIs(t, result.Err, errShuttingDownError) } -// TestStreamVerifierPostVBlocked tests what happens when the ctx canceled when the task is in exec pool +// TestStreamVerifierPostVBlocked tests the behavior when the return channel (result chan) of verified +// transactions is blocked, and checks droppedFromPool counter to confirm the drops func TestStreamVerifierPostVBlocked(t *testing.T) { // prepare the stream verifier @@ -1466,42 +1513,60 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { var goodSigResultCounter int ctx, _ := context.WithCancel(context.Background()) - cache := MakeVerifiedTransactionCache(50000) + cache := MakeVerifiedTransactionCache(50) + + txBacklogSizeMod := txBacklogSize / 20 stxnChan := make(chan *UnverifiedElement) - resultChan := make(chan *VerificationResult, txBacklogSize) + resultChan := make(chan *VerificationResult, txBacklogSizeMod) sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) require.NoError(t, err) // start the verifier sv.Start(ctx) - //xxx - // send transactions to overflow the result chan - numOfTxns := txBacklogSize + 1 + overflow := 3 + // send txBacklogSizeMod + 3 transactions to overflow the result buffer + numOfTxns := txBacklogSizeMod + overflow txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) numOfTxnGroups := len(txnGroups) for _, tg := range txnGroups { - select { - case <-ctx.Done(): - break - case stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil}: - } + stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} } - // wait until the one extra transaction is dropped + var droppedPool uint64 + // wait until overflow transactions are dropped for w := 0; w < 100; w++ { - droppedPool := sv.droppedFromPool.GetUint64Value() - if droppedPool > 0 { // there could be more from the previous iteration + droppedPool = sv.droppedFromPool.GetUint64Value() + if droppedPool >= uint64(overflow) { break } - time.Sleep(time.Millisecond * 5) + time.Sleep(time.Millisecond * 20) } + require.Equal(t, uint64(overflow), droppedPool) + wg := sync.WaitGroup{} wg.Add(1) // make sure the other results are fine + go processResults(ctx, errChan, resultChan, numOfTxnGroups-overflow, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) + + for err := range errChan { + require.ErrorIs(t, err, errShuttingDownError) + fmt.Println(badTxnGroups) + } + + // check if more transactions can be accepted + errChan = make(chan error) + + wg.Add(1) + // make sure the other results are fine + txnGroups, badTxnGroups = getSignedTransactions(numOfTxns, 1, 0.5) go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) + for _, tg := range txnGroups { + stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} + } + for err := range errChan { require.ErrorIs(t, err, errShuttingDownError) fmt.Println(badTxnGroups) @@ -1509,4 +1574,8 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { wg.Wait() } -*/ + +func TestStreamVerifierMakeStreamVerifierErr(t *testing.T) { + _, err := MakeStreamVerifier(nil, nil, &DummyLedgerForSignature{badHdr: true}, nil, nil, nil) + require.Error(t, err) +} From ffefbe33636062ee0d3be57e34fb965063070530 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 1 Dec 2022 00:21:30 -0500 Subject: [PATCH 124/156] fix golint --- data/transactions/verify/txn_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 4ab742e144..28a37c7edf 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1512,7 +1512,8 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { var badSigResultCounter int var goodSigResultCounter int - ctx, _ := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() cache := MakeVerifiedTransactionCache(50) txBacklogSizeMod := txBacklogSize / 20 From 9c2b0efc08b36bd9b3d36b852328af2614de5e3e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 2 Dec 2022 01:14:31 -0500 Subject: [PATCH 125/156] fix: when the context is cancled, the streamVerifier input chan will not accept any txngrps. The backlog worker will get stuck trying to pass the txn pulled from the backlog. The chan should be selected with ctx.Done Other fixes/enhancements partial txhHanler test --- data/txHandler.go | 9 +- data/txHandler_test.go | 267 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 245 insertions(+), 31 deletions(-) diff --git a/data/txHandler.go b/data/txHandler.go index 8851251d97..a562cd39d5 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -196,8 +196,13 @@ func (handler *TxHandler) backlogWorker() { transactionMessagesAlreadyCommitted.Inc(nil) continue } - handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi} - + // handler.streamVerifierChan does not receive if ctx is cancled + select { + case handler.streamVerifierChan <- &verify.UnverifiedElement{TxnGroup: wi.unverifiedTxGroup, BacklogMessage: wi}: + case <-handler.ctx.Done(): + transactionMessagesDroppedFromBacklog.Inc(nil) + return + } case wi, ok := <-handler.postVerificationQueue: if !ok { // this is never happening since handler.postVerificationQueue is never closed diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 0d98daac2a..a053e4ed38 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "math/rand" + "strings" "sync" "testing" "time" @@ -518,53 +519,63 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, add // BenchmarkHandleTxns sends singed transactions the the verifier func BenchmarkHandleTxns(b *testing.B) { maxGroupSize := 1 - tpss := []int{600000, 60000, 6000, 600} + tpss := []int{6000000, 600000, 60000, 6000} + invalidRates := []float32{0.5, 0.001} for _, tps := range tpss { - b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { - rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, false) - }) + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, false) + }) + } } } -// BenchmarkHandleTransactionGroups sends singed transaction groups to the verifier +// BenchmarkHandleTxnGroups sends singed transaction groups to the verifier func BenchmarkHandleTxnGroups(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 - tpss := []int{600000, 60000, 6000, 600} + tpss := []int{6000000, 600000, 60000, 6000} + invalidRates := []float32{0.5, 0.001} for _, tps := range tpss { - b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { - rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, false) - }) + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, false) + }) + } } } // BenchmarkBacklogWorkerHandleTxns sends singed transactions the the verifier -func BenchmarkHandleTxnsBacklogWorker(b *testing.B) { +// using a backlog worker replica +func BenchmarkHandleBLWTxns(b *testing.B) { maxGroupSize := 1 tpss := []int{6000000, 600000, 60000, 6000} + invalidRates := []float32{0.5, 0.001} for _, tps := range tpss { - b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { - rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, true) - }) + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, true) + }) + } } } // BenchmarkBacklogWorkerHandleTxnGroups sends singed transaction groups to the verifier -func BenchmarkHandleTxnGroupsBacklogWorker(b *testing.B) { +// using a backlog worker replica +func BenchmarkHandleBLWTxnGroups(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 tpss := []int{6000000, 600000, 60000, 6000} + invalidRates := []float32{0.5, 0.001} for _, tps := range tpss { - b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { - rateAdjuster := time.Second / time.Duration(tps) - runHandlerBenchmarkWithBacklog(rateAdjuster, maxGroupSize, tps, b, true) - }) + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("tps: %d", tps), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, true) + }) + } } } // runHandlerBenchmarkWithBacklog benchmarks the number of transactions verfied or dropped -func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tps int, b *testing.B, useBacklogWorker bool) { +func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, b *testing.B, useBacklogWorker bool) { defer func() { // reset the counters transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) @@ -596,7 +607,9 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp require.Equal(b, len(genesis), numUsers+1) genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) - ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N) + ivrString := strings.IndexAny(fmt.Sprintf("%f", invalidRate), "1") + ledgerName := fmt.Sprintf("%s-mem-%d-%d", b.Name(), b.N, ivrString) + ledgerName = strings.Replace(ledgerName, "#", "-", 1) const inMem = true cfg := config.GetDefaultLocal() cfg.Archival = true @@ -668,7 +681,7 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp } // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, 0.001, addresses, secrets) + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, invalidRate, addresses, secrets) var encodedSignedTransactionGroups []network.IncomingMessage if useBacklogWorker { encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, b.N) @@ -684,6 +697,7 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp var tt time.Time // Process the results and make sure they are correct + rateAdjuster := time.Second / time.Duration(tps) wg.Add(1) go func() { defer wg.Done() @@ -691,15 +705,15 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp var txnCounter uint64 invalidCounter := 0 defer func() { - if txnCounter > 0 { + if groupCounter > 1 { droppedBacklog, droppedPool := getDropped() - b.Logf("Input TPS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) - b.Logf("Verified TPS: %d", uint64(txnCounter)*1000000000/uint64(time.Since(tt))) + b.Logf("Input T(grp)PS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + b.Logf("Verified TPS: %d", uint64(txnCounter)*uint64(time.Second)/uint64(time.Since(tt))) b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) - handler.Stop() // cancel the handler ctx } + handler.Stop() // cancel the handler ctx }() stopChan := make(chan interface{}) go func() { @@ -758,7 +772,6 @@ func runHandlerBenchmarkWithBacklog(rateAdjuster time.Duration, maxGroupSize, tp return } } - } }() @@ -862,3 +875,199 @@ func TestTxHandlerPostProcessErrorWithVerify(t *testing.T) { transactionMessagesTxnNotWellFormed.AddMetric(result) require.Len(t, result, 1) } + +func TestMakeTxHandlerErrors(t *testing.T) { + _, err := MakeTxHandler(nil, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) + require.Error(t, err, errors.New("MakeTxHandler: txPool is nil on initialization")) + + _, err = MakeTxHandler(&pools.TransactionPool{}, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) + require.Error(t, err, errors.New("MakeTxHandler: ledger is nil on initialization")) + + // it is not possible to test MakeStreamVerifier returning an error, because it is not possible to + // get the leger return an error for returining the header of its latest round +} + +func TestTxHandlerRealBLWRestar(t *testing.T) { + const numUsers = 100 + log := logging.TestingLog(t) + log.SetLevel(logging.Warn) + addresses := make([]basics.Address, numUsers) + secrets := make([]*crypto.SignatureSecrets, numUsers) + + // prepare the accounts + genesis := make(map[basics.Address]basics.AccountData) + for i := 0; i < numUsers; i++ { + secret := keypair() + addr := basics.Address(secret.SignatureVerifier) + secrets[i] = secret + addresses[i] = addr + genesis[addr] = basics.AccountData{ + Status: basics.Online, + MicroAlgos: basics.MicroAlgos{Raw: 10000000000000}, + } + } + genesis[poolAddr] = basics.AccountData{ + Status: basics.NotParticipating, + MicroAlgos: basics.MicroAlgos{Raw: config.Consensus[protocol.ConsensusCurrentVersion].MinBalance}, + } + + // setup the ledger + require.Equal(t, len(genesis), numUsers+1) + genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + ledgerName := fmt.Sprintf("%s-mem", t.Name()) + const inMem = true + cfg := config.GetDefaultLocal() + cfg.Archival = true + ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) + require.NoError(t, err) + + l := ledger + tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) + handler, err := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + require.NoError(t, err) + // since Start is not called, set the context here + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + defer handler.ctxCancel() + + // prepare the transactions + numTxns := 100 + maxGroupSize := 1 + tps := 6000 + invalidRate := float32(0.2) + rateAdjuster := time.Second / time.Duration(tps) + signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(numTxns, numUsers, maxGroupSize, invalidRate, addresses, secrets) + var encodedSignedTransactionGroups []network.IncomingMessage + + encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, numTxns) + for _, stxngrp := range signedTransactionGroups { + data := make([]byte, 0) + for _, stxn := range stxngrp { + data = append(data, protocol.Encode(&stxn)...) + } + encodedSignedTransactionGroups = + append(encodedSignedTransactionGroups, network.IncomingMessage{Data: data}) + } + + // start the handler + handler.Start() + + // send the transactions to the backlog worker + for _, tg := range encodedSignedTransactionGroups[0:numTxns/2] { + handler.processIncomingTxn(tg) + time.Sleep(rateAdjuster) + } + // handler.Stop() + for _, tg := range encodedSignedTransactionGroups[numTxns/2:] { + handler.processIncomingTxn(tg) + time.Sleep(rateAdjuster) + } + time.Sleep(4*time.Second) + dropped := transactionMessagesDroppedFromBacklog.GetUint64Value() + resultBadTxnCount := transactionMessagesTxnSigVerificationFailed.GetUint64Value() + resultGoodTxnCount := uint64(len(tp.PendingTxIDs())) + + require.Equal(t, numTxns, int(dropped+resultGoodTxnCount+resultBadTxnCount)) + + // restart the handler + handler.Start() + for _, tg := range encodedSignedTransactionGroups { + handler.processIncomingTxn(tg) + time.Sleep(rateAdjuster) + } + + defer handler.Stop() + + resultBadTxnCount = transactionMessagesTxnSigVerificationFailed.GetUint64Value() + inputGoodTxnCount := len(signedTransactionGroups) - len(badTxnGroups) + // require.Equal(t, len(badTxnGroups), int(resultBadTxnCount)) + require.Equal(t, inputGoodTxnCount, len(tp.PendingTxIDs())) + + for _, txg := range tp.PendingTxGroups() { + u, _ := binary.Uvarint(txg[0].Txn.Note) + _, inBad := badTxnGroups[u] + require.False(t, inBad, "invalid transaction accepted") + } +} + +/* + var tt time.Time + // Process the results and make sure they are correct + rateAdjuster := time.Second / time.Duration(tps) + wg.Add(1) + go func() { + defer wg.Done() + groupCounter := uint64(0) + var txnCounter uint64 + invalidCounter := 0 + defer func() { + if groupCounter > 1 { + droppedBacklog, droppedPool := getDropped() + b.Logf("Input T(grp)PS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + b.Logf("Verified TPS: %d", uint64(txnCounter)*uint64(time.Second)/uint64(time.Since(tt))) + b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) + b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) + } + handler.Stop() // cancel the handler ctx + }() + stopChan := make(chan interface{}) + go func() { + for { + time.Sleep(200 * time.Millisecond) + droppedBacklog, droppedPool := getDropped() + if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { + // all the benchmark txns processed + close(stopChan) + return + } + } + }() + + if useBacklogWorker { + for { + select { + case wi := <-testResultChan: + txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) + groupCounter++ + u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.verificationErr == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + if groupCounter == uint64(len(signedTransactionGroups)) { + // all the benchmark txns processed + return + } + case <-stopChan: + return + } + } + } else { + for { + select { + case wi := <-handler.postVerificationQueue: + txnCounter = txnCounter + uint64(len(wi.TxnGroup)) + groupCounter++ + u, _ := binary.Uvarint(wi.TxnGroup[0].Txn.Note) + _, inBad := badTxnGroups[u] + if wi.Err == nil { + require.False(b, inBad, "No error for invalid signature") + } else { + invalidCounter++ + require.True(b, inBad, "Error for good signature") + } + if groupCounter == uint64(len(signedTransactionGroups)) { + // all the benchmark txns processed + return + } + case <-stopChan: + return + } + } + } + }() +*/ From fbedc88f0d1f2c3ceabf812bbce2e6ba58c66f51 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Sat, 3 Dec 2022 00:42:02 -0500 Subject: [PATCH 126/156] shutdown verification pool in tests --- data/transactions/verify/txn_test.go | 31 +++++++++++----------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 4ded824a9e..d943cbb22b 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -886,8 +886,7 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m expectedError error, t *testing.T) (sv *StreamVerifier) { numOfTxnGroups := len(txnGroups) - execPool := execpool.MakePool(t) - verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) defer verificationPool.Shutdown() ctx, cancel := context.WithCancel(context.Background()) @@ -1208,9 +1207,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { // prepare the stream verifier numOfTxnGroups := len(txnGroups) - execPool := execpool.MakePool(t) - backlogQueueSize := 64 - verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) + verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) _, buffLen := verificationPool.BufferLength() // make sure the pool is shut down and the buffer is full @@ -1293,9 +1290,7 @@ func TestStreamVerifierRestart(t *testing.T) { // prepare the stream verifier numOfTxnGroups := len(txnGroups) - execPool := execpool.MakePool(t) - defer execPool.Shutdown() - verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t) + verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) defer verificationPool.Shutdown() cache := MakeVerifiedTransactionCache(50) @@ -1387,10 +1382,8 @@ func TestStreamVerifierBlockWatcher(t *testing.T) { } } -func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}) { - execPool := execpool.MakePool(t) - backlogQueueSize := 4 - verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) +func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}, execpool.BacklogPool) { + verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) _, buffLen := verificationPool.BufferLength() // make the buffer full to control when the tasks get executed @@ -1402,7 +1395,7 @@ func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}) return nil }, nil, nil) } - return verificationPool, holdTasks + return verificationPool, holdTasks, verificationPool } // TestStreamVerifierCtxCancel tests the termination when the ctx is canceled @@ -1413,8 +1406,8 @@ func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}) func TestStreamVerifierCtxCancel(t *testing.T) { partitiontest.PartitionTest(t) - verificationPool, holdTasks := getSaturatedExecPool(t) - + verificationPool, holdTasks, vp := getSaturatedExecPool(t) + defer vp.Shutdown() ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50) stxnChan := make(chan *UnverifiedElement) @@ -1460,7 +1453,8 @@ func TestStreamVerifierCtxCancel(t *testing.T) { func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { partitiontest.PartitionTest(t) - verificationPool, holdTasks := getSaturatedExecPool(t) + verificationPool, holdTasks, vp := getSaturatedExecPool(t) + defer vp.Shutdown() ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50) @@ -1505,9 +1499,8 @@ func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { func TestStreamVerifierPostVBlocked(t *testing.T) { // prepare the stream verifier - execPool := execpool.MakePool(t) - backlogQueueSize := 64 - verificationPool := execpool.MakeBacklog(execPool, backlogQueueSize, execpool.LowPriority, t) + verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) + defer verificationPool.Shutdown() errChan := make(chan error) var badSigResultCounter int var goodSigResultCounter int From 7705a78a9137126e3d140d23f17f386eaf071041 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 5 Dec 2022 13:36:28 -0500 Subject: [PATCH 127/156] test with the real backlog worker restarting txHandler --- data/transactions/verify/txn.go | 1 + data/txHandler_test.go | 61 +++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index a0b9300a47..0bd3d0a943 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -690,6 +690,7 @@ func (sv *StreamVerifier) Start(ctx context.Context) { go sv.batchingLoop() } +// WaitForStop waits until the batching loop terminates afer the ctx is cancled func (sv *StreamVerifier) WaitForStop() { sv.activeLoopWg.Wait() } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 7a2e163162..4130272f38 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -887,12 +887,16 @@ func TestMakeTxHandlerErrors(t *testing.T) { // get the leger return an error for returining the header of its latest round } -func TestTxHandlerRealBLWRestart(t *testing.T) { +func TestTxHandlerRestartWithBacklogAndTxPool(t *testing.T) { defer func() { // reset the counters transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) transactionMessagesTxnSigVerificationFailed = metrics.MakeCounter(metrics.TransactionMessagesTxnSigVerificationFailed) + transactionMessagesBacklogErr = metrics.MakeCounter(metrics.TransactionMessagesBacklogErr) + transactionMessagesAlreadyCommitted = metrics.MakeCounter(metrics.TransactionMessagesAlreadyCommitted) + transactionMessagesRemember = metrics.MakeCounter(metrics.TransactionMessagesRemember) + transactionMessagesHandled = metrics.MakeCounter(metrics.TransactionMessagesHandled) }() const numUsers = 100 @@ -901,6 +905,7 @@ func TestTxHandlerRealBLWRestart(t *testing.T) { addresses := make([]basics.Address, numUsers) secrets := make([]*crypto.SignatureSecrets, numUsers) + // avoid printing the warning messages origLevel := logging.Base().GetLevel() defer func() { logging.Base().SetLevel(origLevel) }() logging.Base().SetLevel(logging.Error) @@ -931,21 +936,19 @@ func TestTxHandlerRealBLWRestart(t *testing.T) { cfg.Archival = true ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) require.NoError(t, err) + defer ledger.Ledger.Close() - l := ledger - tp := pools.MakeTransactionPool(l.Ledger, cfg, logging.Base()) + tp := pools.MakeTransactionPool(ledger.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - handler, err := MakeTxHandler(tp, l, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + defer backlogPool.Shutdown() + handler, err := MakeTxHandler(tp, ledger, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) require.NoError(t, err) - // since Start is not called, set the context here - handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - defer handler.ctxCancel() // prepare the transactions - numTxns := 100 + numTxns := 3000 maxGroupSize := 1 - tps := 6000 - invalidRate := float32(0.2) + tps := 40000 + invalidRate := float32(0.5) rateAdjuster := time.Second / time.Duration(tps) signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(numTxns, numUsers, maxGroupSize, invalidRate, addresses, secrets) var encodedSignedTransactionGroups []network.IncomingMessage @@ -969,38 +972,52 @@ func TestTxHandlerRealBLWRestart(t *testing.T) { time.Sleep(rateAdjuster) } // stop in a loop to test for possible race conditions - for x := 0; x < 10; x++ { + for x := 0; x < 1000; x++ { handler.Stop() handler.Start() } handler.Stop() + // send the second half after stopping the txHandler for _, tg := range encodedSignedTransactionGroups[numTxns/2:] { handler.processIncomingTxn(tg) time.Sleep(rateAdjuster) } - dropped := transactionMessagesDroppedFromBacklog.GetUint64Value() - stuckInQueue := uint64(len(handler.backlogQueue)) + // check that all the incomming transactions are accounted for + droppeda, droppedb := getDropped() + dropped := droppeda + droppedb + stuckInBLQueue := uint64(len(handler.backlogQueue)) resultBadTxnCount := transactionMessagesTxnSigVerificationFailed.GetUint64Value() - resultGoodTxnCount := uint64(len(tp.PendingTxIDs())) -return - require.Equal(t, numTxns, int(dropped+resultGoodTxnCount+resultBadTxnCount+stuckInQueue)) - fmt.Println(int(dropped + resultGoodTxnCount + resultBadTxnCount)) + resultGoodTxnCount := transactionMessagesHandled.GetUint64Value() + shutdownDropCount := transactionMessagesBacklogErr.GetUint64Value() + require.Equal(t, numTxns, int(dropped+resultGoodTxnCount+resultBadTxnCount+stuckInBLQueue+shutdownDropCount)) + // start the handler again handler.Start() + defer handler.Stop() + + // no dpulicates are sent at this point + require.Equal(t, 0, int(transactionMessagesAlreadyCommitted.GetUint64Value())) + + // send the same set of transactions again for _, tg := range encodedSignedTransactionGroups { handler.processIncomingTxn(tg) time.Sleep(rateAdjuster) } - defer handler.Stop() + inputGoodTxnCount := len(signedTransactionGroups) - len(badTxnGroups) - resultBadTxnCount = transactionMessagesTxnSigVerificationFailed.GetUint64Value() - // inputGoodTxnCount := len(signedTransactionGroups) - len(badTxnGroups) - // require.Equal(t, len(badTxnGroups), int(resultBadTxnCount)) - //require.Equal(t, inputGoodTxnCount, len(tp.PendingTxIDs())) + // Wait untill all the expected transactions are in the pool + for x := 0; x < 100; x++ { + if len(tp.PendingTxGroups()) == inputGoodTxnCount { + break + } + time.Sleep(40 * time.Millisecond) + } + // check the couters and the accepted transactions + require.Equal(t, inputGoodTxnCount, len(tp.PendingTxGroups())) for _, txg := range tp.PendingTxGroups() { u, _ := binary.Uvarint(txg[0].Txn.Note) _, inBad := badTxnGroups[u] From 46cdd275d534c20bab70315302b8b6a71056c83a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 5 Dec 2022 14:14:56 -0500 Subject: [PATCH 128/156] CR comments --- agreement/asyncVoteVerifier_test.go | 2 +- data/ledger_test.go | 8 ------ data/transactions/verify/txn.go | 37 +++++++++++++--------------- data/transactions/verify/txn_test.go | 4 +-- util/execpool/backlog.go | 6 ++--- 5 files changed, 23 insertions(+), 34 deletions(-) diff --git a/agreement/asyncVoteVerifier_test.go b/agreement/asyncVoteVerifier_test.go index bd7aa60139..42378d506a 100644 --- a/agreement/asyncVoteVerifier_test.go +++ b/agreement/asyncVoteVerifier_test.go @@ -34,7 +34,7 @@ func (fp *expiredExecPool) EnqueueBacklog(enqueueCtx context.Context, t execpool // generate an error, to see if we correctly report that on the verifyVote() call. return context.Canceled } -func (fp *expiredExecPool) BufferLength() (length, capacity int) { +func (fp *expiredExecPool) BufferSize() (length, capacity int) { return } diff --git a/data/ledger_test.go b/data/ledger_test.go index fef8f840c0..064da60ea1 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -117,10 +117,6 @@ func testGenerateInitState(tb testing.TB, proto protocol.ConsensusVersion) (gene } func TestLedgerCirculation(t *testing.T) { - if testing.Short() { - t.Log("this is a long test and skipping for -short") - return - } partitiontest.PartitionTest(t) genesisInitState, keys := testGenerateInitState(t, protocol.ConsensusCurrentVersion) @@ -479,10 +475,6 @@ func (lm loggedMessages) Errorf(s string, args ...interface{}) { // The purpose here is to simulate the scenario where the catchup and the agreement compete to add blocks to the ledger. // The error messages reported can be excessive or unnecessary. This test evaluates what messages are generate and at what frequency. func TestLedgerErrorValidate(t *testing.T) { - if testing.Short() { - t.Log("this is a long test and skipping for -short") - return - } partitiontest.PartitionTest(t) var testPoolAddr = basics.Address{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 0bd3d0a943..9aedf013c8 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -48,9 +48,6 @@ var msigLsigLessOrEqual4 = metrics.MakeCounter(metrics.MetricName{Name: "algod_v var msigLsigLessOrEqual10 = metrics.MakeCounter(metrics.MetricName{Name: "algod_verify_msig_lsig_5_10", Description: "Total transaction scripts with 5-10 msigs"}) var msigLsigMore10 = metrics.MakeCounter(metrics.MetricName{Name: "algod_verify_msig_lsig_10", Description: "Total transaction scripts with 11+ msigs"}) -// ErrInvalidSignature is the error returned to report that at least one signature is invalid -var ErrInvalidSignature = errors.New("At least one signature didn't pass verification") - var errSignedTxnHasNoSig = errors.New("signedtxn has no sig") var errSignedTxnMaxOneSig = errors.New("signedtxn should only have one of Sig or Msig or LogicSig") var errShuttingDownError = errors.New("not verified, verifier is shutting down") @@ -710,8 +707,8 @@ func (sv *StreamVerifier) batchingLoop() { var added bool var numberOfSigsInCurrent uint64 var numberOfTimerResets uint64 - uelts := make([]*UnverifiedElement, 0) - defer func() { sv.cleanup(uelts) }() + ue := make([]*UnverifiedElement, 0) + defer func() { sv.cleanup(ue) }() for { select { case stx := <-sv.stxnChan: @@ -734,7 +731,7 @@ func (sv *StreamVerifier) batchingLoop() { // add this txngrp to the list of batchable txn groups numberOfSigsInCurrent = numberOfSigsInCurrent + numberOfBatchableSigsInGroup - uelts = append(uelts, stx) + ue = append(ue, stx) if numberOfSigsInCurrent > txnPerWorksetThreshold { // enough transaction in the batch to efficiently verify @@ -742,20 +739,20 @@ func (sv *StreamVerifier) batchingLoop() { // do not consider adding more txns to this batch. // bypass the exec pool situation and queue anyway // this is to prevent creation of very large batches - err := sv.addVerificationTaskToThePoolNow(uelts) + err := sv.addVerificationTaskToThePoolNow(ue) if err != nil { return } added = true } else { - added, err = sv.canAddVerificationTaskToThePool(uelts) + added, err = sv.canAddVerificationTaskToThePool(ue) if err != nil { return } } if added { numberOfSigsInCurrent = 0 - uelts = make([]*UnverifiedElement, 0) + ue = make([]*UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 @@ -784,17 +781,17 @@ func (sv *StreamVerifier) batchingLoop() { // bypass the exec pool situation and queue anyway // this is to prevent long delays in transaction propagation // at least one transaction here has waited 3 x waitForNextTxnDuration - err = sv.addVerificationTaskToThePoolNow(uelts) + err = sv.addVerificationTaskToThePoolNow(ue) added = true } else { - added, err = sv.canAddVerificationTaskToThePool(uelts) + added, err = sv.canAddVerificationTaskToThePool(ue) } if err != nil { return } if added { numberOfSigsInCurrent = 0 - uelts = make([]*UnverifiedElement, 0) + ue = make([]*UnverifiedElement, 0) // starting a new batch. Can wait long, since nothing is blocked timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 @@ -824,16 +821,16 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack } } -func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []*UnverifiedElement) (added bool, err error) { +func (sv *StreamVerifier) canAddVerificationTaskToThePool(ue []*UnverifiedElement) (added bool, err error) { // if the exec pool buffer is (half) full, can go back and collect // more signatures instead of waiting in the exec pool buffer // more signatures to the batch do not harm performance but introduce latency when delayed (see crypto.BenchmarkBatchVerifierBig) // if the buffer is full - if l, c := sv.verificationPool.BufferLength(); l == c { + if l, c := sv.verificationPool.BufferSize(); l == c { return false, nil } - err = sv.addVerificationTaskToThePoolNow(uelts) + err = sv.addVerificationTaskToThePoolNow(ue) if err != nil { // An error is returned when the context of the pool expires return false, err @@ -841,7 +838,7 @@ func (sv *StreamVerifier) canAddVerificationTaskToThePool(uelts []*UnverifiedEle return true, nil } -func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []*UnverifiedElement) error { +func (sv *StreamVerifier) addVerificationTaskToThePoolNow(ue []*UnverifiedElement) error { // if the context is canceled when the task is in the queue, it should be cancled // copy the ctx here so that when the StreamVerifier is started again, and a new context // is created, this task still gets cancled due to the ctx at the time of this task @@ -849,17 +846,17 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []*UnverifiedEle function := func(arg interface{}) interface{} { if taskCtx.Err() != nil { // ctx is canceled. the results will be returned - sv.cleanup(uelts) + sv.cleanup(ue) return nil } - uelts := arg.([]*UnverifiedElement) + ue := arg.([]*UnverifiedElement) batchVerifier := crypto.MakeBatchVerifier() bl := makeBatchLoad() // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here blockHeader := sv.nbw.getBlockHeader() - for _, ue := range uelts { + for _, ue := range ue { groupCtx, err := txnGroupBatchPrep(ue.TxnGroup, blockHeader, sv.ledger, batchVerifier) if err != nil { // verification failed, no need to add the sig to the batch, report the error @@ -911,7 +908,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(uelts []*UnverifiedEle } // EnqueueBacklog returns an error when the context is canceled - err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, uelts, nil) + err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, ue, nil) return err } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index d943cbb22b..aba75ba41e 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1208,7 +1208,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { // prepare the stream verifier numOfTxnGroups := len(txnGroups) verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) - _, buffLen := verificationPool.BufferLength() + _, buffLen := verificationPool.BufferSize() // make sure the pool is shut down and the buffer is full holdTasks := make(chan interface{}) @@ -1384,7 +1384,7 @@ func TestStreamVerifierBlockWatcher(t *testing.T) { func getSaturatedExecPool(t *testing.T) (execpool.BacklogPool, chan interface{}, execpool.BacklogPool) { verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) - _, buffLen := verificationPool.BufferLength() + _, buffLen := verificationPool.BufferSize() // make the buffer full to control when the tasks get executed holdTasks := make(chan interface{}) diff --git a/util/execpool/backlog.go b/util/execpool/backlog.go index a212f87909..125fb6f923 100644 --- a/util/execpool/backlog.go +++ b/util/execpool/backlog.go @@ -43,7 +43,7 @@ type backlogItemTask struct { type BacklogPool interface { ExecutionPool EnqueueBacklog(enqueueCtx context.Context, t ExecFunc, arg interface{}, out chan interface{}) error - BufferLength() (length, capacity int) + BufferSize() (length, capacity int) } // MakeBacklog creates a backlog @@ -95,8 +95,8 @@ func (b *backlog) Enqueue(enqueueCtx context.Context, t ExecFunc, arg interface{ } } -// BufferLength returns the length and the capacity of the buffer -func (b *backlog) BufferLength() (length, capacity int) { +// BufferSize returns the length and the capacity of the buffer +func (b *backlog) BufferSize() (length, capacity int) { return len(b.buffer), cap(b.buffer) } From c0d636f9f4cd25091852b41ddf7ddbc42a77f949 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 6 Dec 2022 00:09:49 -0500 Subject: [PATCH 129/156] CR: var renames, other movements --- data/txHandler.go | 22 ++++++++++++++-------- data/txHandler_test.go | 4 ++-- ledger/ledgercore/misc.go | 5 +++++ ledger/ledgercore/statedelta.go | 5 ----- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/data/txHandler.go b/data/txHandler.go index 866594ffc3..13f4ec55a3 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -65,6 +65,12 @@ var transactionGroupTxSyncHandled = metrics.MakeCounter(metrics.TransactionGroup var transactionGroupTxSyncRemember = metrics.MakeCounter(metrics.TransactionGroupTxSyncRemember) var transactionGroupTxSyncAlreadyCommitted = metrics.MakeCounter(metrics.TransactionGroupTxSyncAlreadyCommitted) +// ErrInvalidTxPool is reported when nil is passed for the tx pool +var ErrInvalidTxPool = errors.New("MakeTxHandler: txPool is nil on initialization") + +// ErrInvalidLedger is reported when nil is passed for the ledger +var ErrInvalidLedger = errors.New("MakeTxHandler: ledger is nil on initialization") + // The txBacklogMsg structure used to track a single incoming transaction from the gossip network, type txBacklogMsg struct { rawmsg *network.IncomingMessage // the raw message from the network @@ -93,11 +99,11 @@ type TxHandler struct { func MakeTxHandler(txPool *pools.TransactionPool, ledger *Ledger, net network.GossipNode, genesisID string, genesisHash crypto.Digest, executionPool execpool.BacklogPool) (*TxHandler, error) { if txPool == nil { - return nil, errors.New("MakeTxHandler: txPool is nil on initialization") + return nil, ErrInvalidTxPool } if ledger == nil { - return nil, errors.New("MakeTxHandler: ledger is nil on initialization") + return nil, ErrInvalidLedger } handler := &TxHandler{ @@ -177,9 +183,9 @@ func (handler *TxHandler) backlogWorker() { if !ok { return } - txBLMsg := wi.BacklogMessage.(*txBacklogMsg) - txBLMsg.verificationErr = wi.Err - handler.postProcessCheckedTxn(txBLMsg) + m := wi.BacklogMessage.(*txBacklogMsg) + m.verificationErr = wi.Err + handler.postProcessCheckedTxn(m) // restart the loop so that we could empty out the post verification queue. continue @@ -209,9 +215,9 @@ func (handler *TxHandler) backlogWorker() { // this is never happening since handler.postVerificationQueue is never closed return } - txBLMsg := wi.BacklogMessage.(*txBacklogMsg) - txBLMsg.verificationErr = wi.Err - handler.postProcessCheckedTxn(txBLMsg) + m := wi.BacklogMessage.(*txBacklogMsg) + m.verificationErr = wi.Err + handler.postProcessCheckedTxn(m) case <-handler.ctx.Done(): return diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 4130272f38..612159051c 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -878,10 +878,10 @@ func TestTxHandlerPostProcessErrorWithVerify(t *testing.T) { func TestMakeTxHandlerErrors(t *testing.T) { _, err := MakeTxHandler(nil, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) - require.Error(t, err, errors.New("MakeTxHandler: txPool is nil on initialization")) + require.Error(t, err, ErrInvalidTxPool) _, err = MakeTxHandler(&pools.TransactionPool{}, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) - require.Error(t, err, errors.New("MakeTxHandler: ledger is nil on initialization")) + require.Error(t, err, ErrInvalidLedger) // it is not possible to test MakeStreamVerifier returning an error, because it is not possible to // get the leger return an error for returining the header of its latest round diff --git a/ledger/ledgercore/misc.go b/ledger/ledgercore/misc.go index 5bc39cd1cc..22ea6b5e74 100644 --- a/ledger/ledgercore/misc.go +++ b/ledger/ledgercore/misc.go @@ -28,3 +28,8 @@ type InitState struct { Accounts map[basics.Address]basics.AccountData GenesisHash crypto.Digest } + +// BlockListener represents an object that needs to get notified on new blocks. +type BlockListener interface { + OnNewBlock(block bookkeeping.Block, delta StateDelta) +} diff --git a/ledger/ledgercore/statedelta.go b/ledger/ledgercore/statedelta.go index 887ab81c98..688c8d093b 100644 --- a/ledger/ledgercore/statedelta.go +++ b/ledger/ledgercore/statedelta.go @@ -32,11 +32,6 @@ const ( stateDeltaTargetOptimizationThreshold = uint64(50000000) ) -// BlockListener represents an object that needs to get notified on new blocks. -type BlockListener interface { - OnNewBlock(block bookkeeping.Block, delta StateDelta) -} - // ModifiedCreatable defines the changes to a single single creatable state type ModifiedCreatable struct { // Type of the creatable: app or asset From 18ce2711a1ca2d3722f593ec44b7ed0cd9e9b9f5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 6 Dec 2022 01:25:40 -0500 Subject: [PATCH 130/156] replace mutex with atomic in NewBlockWatcher --- data/transactions/verify/txn.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 9aedf013c8..1acd0e88d7 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -22,10 +22,9 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" - "github.com/algorand/go-deadlock" - "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" @@ -594,32 +593,27 @@ type StreamVerifier struct { // NewBlockWatcher is a struct used to provide a new block header to the // stream verifier type NewBlockWatcher struct { - blkHeader *bookkeeping.BlockHeader - mu deadlock.RWMutex + blkHeader atomic.Value } // MakeNewBlockWatcher construct a new block watcher with the initial blkHdr func MakeNewBlockWatcher(blkHdr bookkeeping.BlockHeader) (nbw *NewBlockWatcher) { - nbw = &NewBlockWatcher{ - blkHeader: &blkHdr, - } + nbw = &NewBlockWatcher{} + nbw.blkHeader.Store(&blkHdr) return nbw } // OnNewBlock implements the interface to subscribe to new block notifications from the ledger func (nbw *NewBlockWatcher) OnNewBlock(block bookkeeping.Block, delta ledgercore.StateDelta) { - if nbw.blkHeader.Round >= block.BlockHeader.Round { + bh := nbw.blkHeader.Load().(*bookkeeping.BlockHeader) + if bh.Round >= block.BlockHeader.Round { return } - nbw.mu.Lock() - defer nbw.mu.Unlock() - nbw.blkHeader = &block.BlockHeader + nbw.blkHeader.Store(&block.BlockHeader) } func (nbw *NewBlockWatcher) getBlockHeader() (bh *bookkeeping.BlockHeader) { - nbw.mu.RLock() - defer nbw.mu.RUnlock() - return nbw.blkHeader + return nbw.blkHeader.Load().(*bookkeeping.BlockHeader) } type batchLoad struct { From 30e0b142d6a6f3e82a57450fac4ba43c0b6c5790 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 6 Dec 2022 01:26:22 -0500 Subject: [PATCH 131/156] do not print warn/debug messages in these tests --- data/ledger_test.go | 4 ++-- data/txHandler_test.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/data/ledger_test.go b/data/ledger_test.go index 064da60ea1..05880b833d 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -125,7 +125,7 @@ func TestLedgerCirculation(t *testing.T) { cfg := config.GetDefaultLocal() cfg.Archival = true log := logging.TestingLog(t) - log.SetLevel(logging.Warn) + log.SetLevel(logging.Error) realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) require.NoError(t, err, "could not open ledger") defer realLedger.Close() @@ -517,7 +517,7 @@ func TestLedgerErrorValidate(t *testing.T) { defer realLedger.Close() l := Ledger{Ledger: realLedger, log: log} - l.log.SetLevel(logging.Debug) + l.log.SetLevel(logging.Warn) require.NotNil(t, &l) totalsRound, _, err := realLedger.LatestTotals() diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 612159051c..970730322a 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -298,6 +298,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t const numUsers = 100 log := logging.TestingLog(t) + log.SetLevel(logging.Warn) addresses := make([]basics.Address, numUsers) secrets := make([]*crypto.SignatureSecrets, numUsers) From 2b364d57f6de0561b21031edf8b394f8e67dff8b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 12 Dec 2022 10:24:16 -0500 Subject: [PATCH 132/156] CR fixes --- data/transactions/verify/txn.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 1acd0e88d7..64bb79bc30 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -739,7 +739,7 @@ func (sv *StreamVerifier) batchingLoop() { } added = true } else { - added, err = sv.canAddVerificationTaskToThePool(ue) + added, err = sv.tryAddVerificationTaskToThePool(ue) if err != nil { return } @@ -778,7 +778,7 @@ func (sv *StreamVerifier) batchingLoop() { err = sv.addVerificationTaskToThePoolNow(ue) added = true } else { - added, err = sv.canAddVerificationTaskToThePool(ue) + added, err = sv.tryAddVerificationTaskToThePool(ue) } if err != nil { return @@ -815,8 +815,8 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack } } -func (sv *StreamVerifier) canAddVerificationTaskToThePool(ue []*UnverifiedElement) (added bool, err error) { - // if the exec pool buffer is (half) full, can go back and collect +func (sv *StreamVerifier) tryAddVerificationTaskToThePool(ue []*UnverifiedElement) (added bool, err error) { + // if the exec pool buffer is full, can go back and collect // more signatures instead of waiting in the exec pool buffer // more signatures to the batch do not harm performance but introduce latency when delayed (see crypto.BenchmarkBatchVerifierBig) From a5754da45aea727038bb9d8b7963c1736103366b Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 12 Dec 2022 22:47:43 -0500 Subject: [PATCH 133/156] fix mege conflicts --- data/ledger_test.go | 2 +- data/transactions/verify/artifact_test.go | 4 +- data/transactions/verify/txn.go | 11 ++- data/transactions/verify/txn_test.go | 35 +++++++--- data/txHandler.go | 26 +++++-- data/txHandler_test.go | 85 ++++++++++++++++++----- 6 files changed, 119 insertions(+), 44 deletions(-) diff --git a/data/ledger_test.go b/data/ledger_test.go index 05880b833d..9469e578df 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -125,7 +125,7 @@ func TestLedgerCirculation(t *testing.T) { cfg := config.GetDefaultLocal() cfg.Archival = true log := logging.TestingLog(t) - log.SetLevel(logging.Error) + log.SetLevel(logging.Panic) realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) require.NoError(t, err, "could not open ledger") defer realLedger.Close() diff --git a/data/transactions/verify/artifact_test.go b/data/transactions/verify/artifact_test.go index 8444dfa353..8c4f44e601 100644 --- a/data/transactions/verify/artifact_test.go +++ b/data/transactions/verify/artifact_test.go @@ -77,7 +77,7 @@ func BenchmarkTinyMan(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := TxnGroup(stxnss[i], hdr, nil, &logic.NoHeaderLedger{}) + _, err := TxnGroup(stxnss[i], &hdr, nil, &logic.NoHeaderLedger{}) require.NoError(b, err) } }) @@ -93,7 +93,7 @@ func BenchmarkTinyMan(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := TxnGroup(stxns, hdr, nil, &logic.NoHeaderLedger{}) + _, err := TxnGroup(stxns, &hdr, nil, &logic.NoHeaderLedger{}) require.NoError(b, err) } }) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 4ace353501..56575e47ad 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -582,6 +582,7 @@ type VerificationResult struct { // results through the resultChan type StreamVerifier struct { resultChan chan<- *VerificationResult + droppedChan chan<- *UnverifiedElement stxnChan <-chan *UnverifiedElement verificationPool execpool.BacklogPool ctx context.Context @@ -589,7 +590,6 @@ type StreamVerifier struct { activeLoopWg sync.WaitGroup nbw *NewBlockWatcher ledger logic.LedgerForSignature - droppedFromPool *metrics.Counter } // NewBlockWatcher is a struct used to provide a new block header to the @@ -652,8 +652,8 @@ type LedgerForStreamVerifier interface { // MakeStreamVerifier creates a new stream verifier and returns the chans used to send txn groups // to it and obtain the txn signature verification result from func MakeStreamVerifier(stxnChan <-chan *UnverifiedElement, resultChan chan<- *VerificationResult, - ledger LedgerForStreamVerifier, verificationPool execpool.BacklogPool, - cache VerifiedTransactionCache, droppedFromPool *metrics.Counter) (*StreamVerifier, error) { + droppedChan chan<- *UnverifiedElement, ledger LedgerForStreamVerifier, + verificationPool execpool.BacklogPool, cache VerifiedTransactionCache) (*StreamVerifier, error) { latest := ledger.Latest() latestHdr, err := ledger.BlockHdr(latest) @@ -667,11 +667,11 @@ func MakeStreamVerifier(stxnChan <-chan *UnverifiedElement, resultChan chan<- *V return &StreamVerifier{ resultChan: resultChan, stxnChan: stxnChan, + droppedChan: droppedChan, verificationPool: verificationPool, cache: cache, nbw: nbw, ledger: ledger, - droppedFromPool: droppedFromPool, }, nil } @@ -812,8 +812,7 @@ func (sv *StreamVerifier) sendResult(veTxnGroup []transactions.SignedTxn, veBack }: default: // we failed to write to the output queue, since the queue was full. - // adding the metric here allows us to monitor how frequently it happens. - sv.droppedFromPool.Inc(nil) + sv.droppedChan <- &UnverifiedElement{veTxnGroup, veBacklogMessage} } } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index b0ecca65b9..eb08bb35ef 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -641,7 +641,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txn.Lsig.Args[0][0]-- } initCounter := logicCostTotal.GetUint64Value() - verifyGroup(t, txnGroups, blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") + verifyGroup(t, txnGroups, &blkHdr, breakSignatureFunc, restoreSignatureFunc, "rejected by logic") currentCounter := logicCostTotal.GetUint64Value() require.Greater(t, currentCounter, initCounter) } @@ -898,7 +898,8 @@ func streamVerifierTestCore(txnGroups [][]transactions.SignedTxn, badTxnGroups m stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + droppedChan := make(chan *UnverifiedElement) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) @@ -1245,7 +1246,8 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + droppedChan := make(chan *UnverifiedElement) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) @@ -1299,9 +1301,10 @@ func TestStreamVerifierRestart(t *testing.T) { stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) + droppedChan := make(chan *UnverifiedElement) ctx, cancel := context.WithCancel(context.Background()) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) @@ -1414,7 +1417,8 @@ func TestStreamVerifierCtxCancel(t *testing.T) { cache := MakeVerifiedTransactionCache(50) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + droppedChan := make(chan *UnverifiedElement) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) @@ -1462,7 +1466,8 @@ func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { cache := MakeVerifiedTransactionCache(50) stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + droppedChan := make(chan *UnverifiedElement) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) @@ -1515,9 +1520,17 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSizeMod) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + droppedChan := make(chan *UnverifiedElement) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) + defer close(droppedChan) + go func() { + for range droppedChan { + droppedFromPool.Inc(nil) + } + }() + // start the verifier sv.Start(ctx) overflow := 3 @@ -1532,7 +1545,7 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { var droppedPool uint64 // wait until overflow transactions are dropped for w := 0; w < 100; w++ { - droppedPool = sv.droppedFromPool.GetUint64Value() + droppedPool = droppedFromPool.GetUint64Value() if droppedPool >= uint64(overflow) { break } @@ -1572,7 +1585,7 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { } func TestStreamVerifierMakeStreamVerifierErr(t *testing.T) { - _, err := MakeStreamVerifier(nil, nil, &DummyLedgerForSignature{badHdr: true}, nil, nil, nil) + _, err := MakeStreamVerifier(nil, nil, nil, &DummyLedgerForSignature{badHdr: true}, nil, nil) require.Error(t, err) } @@ -1594,9 +1607,9 @@ func TestStreamVerifierCancelWhenPooled(t *testing.T) { stxnChan := make(chan *UnverifiedElement) resultChan := make(chan *VerificationResult, txBacklogSize) - + droppedChan := make(chan *UnverifiedElement) ctx, cancel := context.WithCancel(context.Background()) - sv, err := MakeStreamVerifier(stxnChan, resultChan, &DummyLedgerForSignature{}, verificationPool, cache, droppedFromPool) + sv, err := MakeStreamVerifier(stxnChan, resultChan, droppedChan, &DummyLedgerForSignature{}, verificationPool, cache) require.NoError(t, err) sv.Start(ctx) diff --git a/data/txHandler.go b/data/txHandler.go index 68ab5100a5..0ac81e5729 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -129,6 +129,7 @@ type TxHandler struct { ctxCancel context.CancelFunc streamVerifier *verify.StreamVerifier streamVerifierChan chan *verify.UnverifiedElement + streamVerifierDropped chan *verify.UnverifiedElement } // TxHandlerOpts is TxHandler configuration options @@ -151,11 +152,11 @@ type txHandlerConfig struct { // MakeTxHandler makes a new handler for transaction messages func MakeTxHandler(opts TxHandlerOpts) (*TxHandler, error) { - if opts.txPool == nil { + if opts.TxPool == nil { return nil, ErrInvalidTxPool } - if opts.ledger == nil { + if opts.Ledger == nil { return nil, ErrInvalidLedger } @@ -166,25 +167,40 @@ func MakeTxHandler(opts TxHandlerOpts) (*TxHandler, error) { ledger: opts.Ledger, txVerificationPool: opts.ExecutionPool, backlogQueue: make(chan *txBacklogMsg, txBacklogSize), - postVerificationQueue: make(chan *txBacklogMsg, txBacklogSize), + postVerificationQueue: make(chan *verify.VerificationResult, txBacklogSize), net: opts.Net, msgCache: makeSaltedCache(2 * txBacklogSize), txCanonicalCache: makeDigestCache(2 * txBacklogSize), cacheConfig: txHandlerConfig{opts.Config.TxFilterRawMsgEnabled(), opts.Config.TxFilterCanonicalEnabled()}, streamVerifierChan: make(chan *verify.UnverifiedElement), + streamVerifierDropped: make(chan *verify.UnverifiedElement), } // prepare the transaction stream verifer var err error handler.streamVerifier, err = verify.MakeStreamVerifier(handler.streamVerifierChan, - handler.postVerificationQueue, handler.ledger, handler.txVerificationPool, - handler.ledger.VerifiedTransactionCache(), transactionMessagesDroppedFromPool) + handler.postVerificationQueue, handler.streamVerifierDropped, handler.ledger, + handler.txVerificationPool, handler.ledger.VerifiedTransactionCache()) if err != nil { return nil, err } + go handler.droppedTxnWatcher() return handler, nil } +func (handler *TxHandler) droppedTxnWatcher() { + for unverified := range handler.streamVerifierDropped { + // we failed to write to the output queue, since the queue was full. + // adding the metric here allows us to monitor how frequently it happens. + transactionMessagesDroppedFromPool.Inc(nil) + + tx := unverified.BacklogMessage.(*txBacklogMsg) + + // delete from duplicate caches to give it a chance to be re-submitted + handler.deleteFromCaches(tx.rawmsgDataHash, tx.unverifiedTxGroupHash) + } +} + // Start enables the processing of incoming messages at the transaction handler func (handler *TxHandler) Start() { handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 54523bb0ef..a31d64c06d 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -42,7 +42,6 @@ import ( "github.com/algorand/go-algorand/data/pools" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/verify" - realledger "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" @@ -98,6 +97,8 @@ func BenchmarkTxHandlerProcessing(b *testing.B) { cfg.EnableProcessBlockStats = false txHandler, err := makeTestTxHandler(l, cfg) require.NoError(b, err) + defer txHandler.txVerificationPool.Shutdown() + defer close(txHandler.streamVerifierDropped) makeTxns := func(N int) [][]transactions.SignedTxn { ret := make([][]transactions.SignedTxn, 0, N) @@ -740,7 +741,7 @@ func makeTestTxHandlerOrphanedWithContext(ctx context.Context, backlogSize int, return handler } -func makeTestTxHandler(dl *Ledger, cfg config.Local) *TxHandler { +func makeTestTxHandler(dl *Ledger, cfg config.Local) (*TxHandler, error) { tp := pools.MakeTransactionPool(dl.Ledger, cfg, logging.Base()) backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) opts := TxHandlerOpts{ @@ -901,7 +902,6 @@ func TestTxHandlerProcessIncomingCacheRotation(t *testing.T) { // TestTxHandlerProcessIncomingCacheBacklogDrop checks if dropped messages are also removed from caches func TestTxHandlerProcessIncomingCacheBacklogDrop(t *testing.T) { partitiontest.PartitionTest(t) - t.Parallel() handler := makeTestTxHandlerOrphanedWithContext(context.Background(), 1, 20, 0) @@ -932,6 +932,7 @@ func TestTxHandlerProcessIncomingCacheTxPoolDrop(t *testing.T) { const numUsers = 100 log := logging.TestingLog(t) + log.SetLevel(logging.Panic) // prepare the accounts addresses, secrets, genesis := makeTestGenesisAccounts(t, numUsers) @@ -945,8 +946,20 @@ func TestTxHandlerProcessIncomingCacheTxPoolDrop(t *testing.T) { require.NoError(t, err) l := ledger - handler := makeTestTxHandler(l, cfg) - handler.postVerificationQueue = make(chan *txBacklogMsg) + handler, err := makeTestTxHandler(l, cfg) + require.NoError(t, err) + defer handler.txVerificationPool.Shutdown() + defer close(handler.streamVerifierDropped) + + // saturate the postVerificationQueue +loop: + for { + select { + case handler.postVerificationQueue <- &verify.VerificationResult{}: + default: + break loop + } + } makeTxns := func(sendIdx, recvIdx int) ([]transactions.SignedTxn, []byte) { tx := transactions.Transaction{ @@ -981,8 +994,21 @@ func TestTxHandlerProcessIncomingCacheTxPoolDrop(t *testing.T) { require.Equal(t, stxns, msg.unverifiedTxGroup) initialCount := transactionMessagesDroppedFromPool.GetUint64Value() - handler.asyncVerifySignature(msg) - currentCount := transactionMessagesDroppedFromPool.GetUint64Value() + + // emulate handler.Start() without the backlog + handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) + handler.streamVerifier.Start(handler.ctx) + defer handler.streamVerifier.WaitForStop() + defer handler.ctxCancel() + handler.streamVerifierChan <- &verify.UnverifiedElement{msg.unverifiedTxGroup, msg} + var currentCount uint64 + for x := 0; x < 1000; x++ { + currentCount = transactionMessagesDroppedFromPool.GetUint64Value() + if currentCount > 0 { + break + } + time.Sleep(10 * time.Millisecond) + } require.Equal(t, initialCount+1, currentCount) require.Equal(t, 0, handler.msgCache.Len()) require.Equal(t, 0, handler.txCanonicalCache.Len()) @@ -1085,6 +1111,8 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t l := ledger handler, err := makeTestTxHandler(l, cfg) require.NoError(t, err) + defer handler.txVerificationPool.Shutdown() + defer close(handler.streamVerifierDropped) // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) @@ -1354,7 +1382,9 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, l := ledger handler, err := makeTestTxHandler(l, cfg) - require.NoError(t, err) + require.NoError(b, err) + defer handler.txVerificationPool.Shutdown() + defer close(handler.streamVerifierDropped) // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) defer handler.ctxCancel() @@ -1663,7 +1693,6 @@ func (t *blockTicker) Wait() { func TestTxHandlerRememberReportErrorsWithTxPool(t *testing.T) { partitiontest.PartitionTest(t) - t.Parallel() result := map[string]float64{} checkResult := map[string]float64{} @@ -1715,7 +1744,10 @@ func TestTxHandlerRememberReportErrorsWithTxPool(t *testing.T) { ledger, err := LoadLedger(log, ledgerName, inMem, protocol.ConsensusCurrentVersion, genBal, genesisID, genesisHash, nil, cfg) require.NoError(t, err) - handler := makeTestTxHandler(ledger, cfg) + handler, err := makeTestTxHandler(ledger, cfg) + require.NoError(t, err) + defer handler.txVerificationPool.Shutdown() + defer close(handler.streamVerifierDropped) // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) defer handler.ctxCancel() @@ -1835,7 +1867,7 @@ func TestTxHandlerRememberReportErrorsWithTxPool(t *testing.T) { // make an invalid block to fail recompute pool and expose transactionMessageTxGroupRememberNoPendingEval metric blockTicker := &blockTicker{cond: *sync.NewCond(&deadlock.Mutex{})} - blockListeners := []realledger.BlockListener{ + blockListeners := []ledgercore.BlockListener{ handler.txPool, blockTicker, } @@ -1863,17 +1895,34 @@ func TestTxHandlerRememberReportErrorsWithTxPool(t *testing.T) { } func TestMakeTxHandlerErrors(t *testing.T) { - _, err := MakeTxHandler(nil, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) + opts := TxHandlerOpts{ + nil, nil, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, config.Local{}, + } + _, err := MakeTxHandler(opts) require.Error(t, err, ErrInvalidTxPool) - _, err = MakeTxHandler(&pools.TransactionPool{}, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, nil) + opts = TxHandlerOpts{ + &pools.TransactionPool{}, nil, nil, &mocks.MockNetwork{}, "", crypto.Digest{}, config.Local{}, + } + _, err = MakeTxHandler(opts) require.Error(t, err, ErrInvalidLedger) // it is not possible to test MakeStreamVerifier returning an error, because it is not possible to // get the leger return an error for returining the header of its latest round } +// TestTxHandlerRestartWithBacklogAndTxPool starts txHandler, sends transactions, +// stops, starts in a loop, sends more transactions, and makes sure all the transactions +// are accounted for. It uses the production backlog worker func TestTxHandlerRestartWithBacklogAndTxPool(t *testing.T) { + transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) + transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) + transactionMessagesTxnSigVerificationFailed = metrics.MakeCounter(metrics.TransactionMessagesTxnSigVerificationFailed) + transactionMessagesBacklogErr = metrics.MakeCounter(metrics.TransactionMessagesBacklogErr) + transactionMessagesAlreadyCommitted = metrics.MakeCounter(metrics.TransactionMessagesAlreadyCommitted) + transactionMessagesRemember = metrics.MakeCounter(metrics.TransactionMessagesRemember) + transactionMessagesHandled = metrics.MakeCounter(metrics.TransactionMessagesHandled) + defer func() { // reset the counters transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) @@ -1924,12 +1973,10 @@ func TestTxHandlerRestartWithBacklogAndTxPool(t *testing.T) { require.NoError(t, err) defer ledger.Ledger.Close() - tp := pools.MakeTransactionPool(ledger.Ledger, cfg, logging.Base()) - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - defer backlogPool.Shutdown() - handler, err := MakeTxHandler(tp, ledger, &mocks.MockNetwork{}, "", crypto.Digest{}, backlogPool) + handler, err := makeTestTxHandler(ledger, cfg) require.NoError(t, err) - + defer handler.txVerificationPool.Shutdown() + defer close(handler.streamVerifierDropped) // prepare the transactions numTxns := 3000 maxGroupSize := 1 @@ -1993,7 +2040,7 @@ func TestTxHandlerRestartWithBacklogAndTxPool(t *testing.T) { } inputGoodTxnCount := len(signedTransactionGroups) - len(badTxnGroups) - + tp := handler.txPool // Wait untill all the expected transactions are in the pool for x := 0; x < 100; x++ { if len(tp.PendingTxGroups()) == inputGoodTxnCount { From 3ff8cbf64ddb873019da54a6b9c1360470d2235e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 12 Dec 2022 23:18:50 -0500 Subject: [PATCH 134/156] add waitforstop functionality to txSaltedCache to avoid race condition on the ctx when txHandler restarts --- data/txDupCache.go | 7 +++++++ data/txHandler.go | 1 + 2 files changed, 8 insertions(+) diff --git a/data/txDupCache.go b/data/txDupCache.go index 026a168578..a70c3430ab 100644 --- a/data/txDupCache.go +++ b/data/txDupCache.go @@ -109,6 +109,7 @@ type txSaltedCache struct { curSalt [4]byte prevSalt [4]byte ctx context.Context + wg sync.WaitGroup } func makeSaltedCache(size int) *txSaltedCache { @@ -120,6 +121,7 @@ func makeSaltedCache(size int) *txSaltedCache { func (c *txSaltedCache) start(ctx context.Context, refreshInterval time.Duration) { c.ctx = ctx if refreshInterval != 0 { + c.wg.Add(1) go c.salter(refreshInterval) } @@ -128,9 +130,14 @@ func (c *txSaltedCache) start(ctx context.Context, refreshInterval time.Duration c.moreSalt() } +func (c *txSaltedCache) WaitForStop() { + c.wg.Wait() +} + // salter is a goroutine refreshing the cache by schedule func (c *txSaltedCache) salter(refreshInterval time.Duration) { ticker := time.NewTicker(refreshInterval) + defer c.wg.Done() defer ticker.Stop() for { select { diff --git a/data/txHandler.go b/data/txHandler.go index 0ac81e5729..2db226a210 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -219,6 +219,7 @@ func (handler *TxHandler) Stop() { handler.ctxCancel() handler.backlogWg.Wait() handler.streamVerifier.WaitForStop() + handler.msgCache.WaitForStop() } func reencode(stxns []transactions.SignedTxn) []byte { From 95923c208263a5ac4b37f123b11802000574de63 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 12 Dec 2022 23:31:10 -0500 Subject: [PATCH 135/156] fix lint --- data/txHandler_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index a31d64c06d..461fae1dd7 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1000,7 +1000,8 @@ loop: handler.streamVerifier.Start(handler.ctx) defer handler.streamVerifier.WaitForStop() defer handler.ctxCancel() - handler.streamVerifierChan <- &verify.UnverifiedElement{msg.unverifiedTxGroup, msg} + handler.streamVerifierChan <- &verify.UnverifiedElement{ + TxnGroup: msg.unverifiedTxGroup, BacklogMessage: msg} var currentCount uint64 for x := 0; x < 1000; x++ { currentCount = transactionMessagesDroppedFromPool.GetUint64Value() From fa5591f74ff0543aad21977d1cae3d6b6b3711c6 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 14 Dec 2022 14:08:15 -0500 Subject: [PATCH 136/156] msig benchmark first draft --- data/accountManager_test.go | 11 ++ data/ledger_test.go | 8 +- data/txHandler_test.go | 211 ++++++++++++++++++++++++++++-------- 3 files changed, 180 insertions(+), 50 deletions(-) diff --git a/data/accountManager_test.go b/data/accountManager_test.go index 1fcfe56bf7..b182582193 100644 --- a/data/accountManager_test.go +++ b/data/accountManager_test.go @@ -43,6 +43,11 @@ import ( func TestAccountManagerKeys(t *testing.T) { partitiontest.PartitionTest(t) + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + registry := &mocks.MockParticipationRegistry{} testAccountManagerKeys(t, registry, false) } @@ -85,12 +90,18 @@ func registryCloseTest(t testing.TB, registry account.ParticipationRegistry, dbf func TestAccountManagerKeysRegistry(t *testing.T) { partitiontest.PartitionTest(t) + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + registry, dbName := getRegistryImpl(t, false, true) defer registryCloseTest(t, registry, dbName) testAccountManagerKeys(t, registry, true) } func testAccountManagerKeys(t *testing.T, registry account.ParticipationRegistry, flushRegistry bool) { + log := logging.TestingLog(t) log.SetLevel(logging.Error) diff --git a/data/ledger_test.go b/data/ledger_test.go index 9b74ddc854..985e892714 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -125,7 +125,7 @@ func TestLedgerCirculation(t *testing.T) { cfg := config.GetDefaultLocal() cfg.Archival = true log := logging.TestingLog(t) - log.SetLevel(logging.Warn) + log.SetLevel(logging.Panic) realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) require.NoError(t, err, "could not open ledger") defer realLedger.Close() @@ -326,6 +326,10 @@ func TestLedgerSeed(t *testing.T) { func TestConsensusVersion(t *testing.T) { partitiontest.PartitionTest(t) + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } // find a consensus protocol that leads to ConsensusCurrentVersion var previousProtocol protocol.ConsensusVersion @@ -513,7 +517,7 @@ func TestLedgerErrorValidate(t *testing.T) { defer realLedger.Close() l := Ledger{Ledger: realLedger, log: log} - l.log.SetLevel(logging.Debug) + l.log.SetLevel(logging.Warn) require.NotNil(t, &l) totalsRound, _, err := realLedger.LatestTotals() diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 389eb6d2be..15511cf0a7 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1210,15 +1210,31 @@ func getDropped() (droppedBacklog, droppedPool uint64) { return } -// makeSignedTxnGroups prepares N transaction groups of random (maxGroupSize) sizes with random -// invalid signatures of a given probability (invalidProb) -func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, addresses []basics.Address, - secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, - badTxnGroups map[uint64]interface{}) { - badTxnGroups = make(map[uint64]interface{}) +func getTransaction(sender, receiver basics.Address, u int) transactions.Transaction { + noteField := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(noteField, uint64(u)) + + tx := transactions.Transaction{ + Type: protocol.PaymentTx, + Header: transactions.Header{ + Sender: sender, + Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, + FirstValid: 0, + LastValid: basics.Round(proto.MaxTxnLife), + GenesisHash: genesisHash, + Note: noteField, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: receiver, + Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, + }, + } + return tx +} +func getTransactionGroups(N, numUsers, maxGroupSize int, addresses []basics.Address) [][]transactions.Transaction { + txnGrps := make([][]transactions.Transaction, N) protoMaxGrpSize := proto.MaxTxGroupSize - ret = make([][]transactions.SignedTxn, 0, N) for u := 0; u < N; u++ { grpSize := rand.Intn(protoMaxGrpSize-1) + 1 if grpSize > maxGroupSize { @@ -1228,72 +1244,160 @@ func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, add txns := make([]transactions.Transaction, 0, grpSize) for g := 0; g < grpSize; g++ { // generate transactions - noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(u)) - tx := transactions.Transaction{ - Type: protocol.PaymentTx, - Header: transactions.Header{ - Sender: addresses[(u+g)%numUsers], - Fee: basics.MicroAlgos{Raw: proto.MinTxnFee * 2}, - FirstValid: 0, - LastValid: basics.Round(proto.MaxTxnLife), - GenesisHash: genesisHash, - Note: noteField, - }, - PaymentTxnFields: transactions.PaymentTxnFields{ - Receiver: addresses[(u+g+1)%numUsers], - Amount: basics.MicroAlgos{Raw: mockBalancesMinBalance + (rand.Uint64() % 10000)}, - }, - } + tx := getTransaction(addresses[(u+g)%numUsers], addresses[(u+g+1)%numUsers], u) if grpSize > 1 { txGroup.TxGroupHashes = append(txGroup.TxGroupHashes, crypto.Digest(tx.ID())) } txns = append(txns, tx) } - groupHash := crypto.HashObj(txGroup) - signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) - for g, txn := range txns { - if grpSize > 1 { - txn.Group = groupHash + if grpSize > 1 { + groupHash := crypto.HashObj(txGroup) + for t := range txns { + txns[t].Group = groupHash } - signedTx := txn.Sign(secrets[(u+g)%numUsers]) - signedTx.Txn = txn + } + txnGrps[u] = txns + } + return txnGrps +} + +func signTransactionGroups(txnGroups [][]transactions.Transaction, secrets []*crypto.SignatureSecrets, invalidProb float32) ( + ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}) { + numUsers := len(secrets) + badTxnGroups = make(map[uint64]interface{}) + for tg := range txnGroups { + grpSize := len(txnGroups[tg]) + signedTxGroup := make([]transactions.SignedTxn, 0, grpSize) + for t := range txnGroups[tg] { + signedTx := txnGroups[tg][t].Sign(secrets[(tg+t)%numUsers]) + signedTx.Txn = txnGroups[tg][t] signedTxGroup = append(signedTxGroup, signedTx) } // randomly make bad signatures if rand.Float32() < invalidProb { tinGrp := rand.Intn(grpSize) signedTxGroup[tinGrp].Sig[0] = signedTxGroup[tinGrp].Sig[0] + 1 - badTxnGroups[uint64(u)] = struct{}{} + badTxnGroups[uint64(tg)] = struct{}{} + } + ret = append(ret, signedTxGroup) + } + return +} + +func signMSigTransactionGroups(txnGroups [][]transactions.Transaction, secrets []*crypto.SignatureSecrets, + invalidProb float32, msigSize int) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}, err error) { + + numUsers := len(secrets) + badTxnGroups = make(map[uint64]interface{}) + for tg := range txnGroups { + msigVer := uint8(1) + msigTHld := uint8(msigSize) + pks := make([]crypto.PublicKey, msigSize) + for x := 0; x < msigSize; x++ { + pks[x] = secrets[(tg+x)%numUsers].SignatureVerifier + } + multiSigAddr, err := crypto.MultisigAddrGen(msigVer, msigTHld, pks) + if err != nil { + return ret, badTxnGroups, err + } + grpSize := len(txnGroups[tg]) + signedTxGroup := make([]transactions.SignedTxn, grpSize) + sigsForTxn := make([]crypto.MultisigSig, msigTHld) + + for t := range txnGroups[tg] { + txnGroups[tg][t].Sender = basics.Address(multiSigAddr) + for s := range sigsForTxn { + sig, err := crypto.MultisigSign(txnGroups[tg][t], crypto.Digest(multiSigAddr), msigVer, msigTHld, pks, *secrets[(tg+s)%numUsers]) + if err != nil { + return ret, badTxnGroups, err + } + sigsForTxn[s] = sig + } + msig, err := crypto.MultisigAssemble(sigsForTxn) + if err != nil { + return ret, badTxnGroups, err + } + signedTxGroup[t].Txn = txnGroups[tg][t] + signedTxGroup[t].Msig = msig + } + // randomly make bad signatures + if rand.Float32() < invalidProb { + tinGrp := rand.Intn(grpSize) + signedTxGroup[tinGrp].Msig.Threshold = signedTxGroup[tinGrp].Msig.Threshold - 1 + badTxnGroups[uint64(tg)] = struct{}{} } ret = append(ret, signedTxGroup) } return } +// makeSignedTxnGroups prepares N transaction groups of random (maxGroupSize) sizes with random +// invalid signatures of a given probability (invalidProb) +func makeSignedTxnGroups(N, numUsers, maxGroupSize int, invalidProb float32, addresses []basics.Address, + secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, + badTxnGroups map[uint64]interface{}) { + + txnGroups := getTransactionGroups(N, numUsers, maxGroupSize, addresses) + ret, badTxnGroups = signTransactionGroups(txnGroups, secrets, invalidProb) + return +} + +// makeMsigSignedTxnGroups prepares N transaction groups of random (maxGroupSize) sizes with random +// invalid signatures of a given probability (invalidProb) signed with multisig accounts of given size +func makeMsigSignedTxnGroups(N, numUsers, msigSize, maxGroupSize int, invalidProb float32, addresses []basics.Address, + secrets []*crypto.SignatureSecrets) (ret [][]transactions.SignedTxn, + badTxnGroups map[uint64]interface{}, err error) { + + txnGroups := getTransactionGroups(N, numUsers, maxGroupSize, addresses) + ret, badTxnGroups, err = signMSigTransactionGroups(txnGroups, secrets, invalidProb, msigSize) + return +} + // BenchmarkHandleTxns sends signed transactions directly to the verifier func BenchmarkHandleTxns(b *testing.B) { maxGroupSize := 1 - tpss := []int{6000000, 600000, 60000, 6000} invalidRates := []float32{0.5, 0.001} - for _, tps := range tpss { + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("inv_%.3f", ivr), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, 1, 0, ivr, b, false) + }) + } +} + +// BenchmarkHandleTxnGroups sends signed transaction groups directly to the verifier +func BenchmarkHandleTxnGroups(b *testing.B) { + maxGroupSize := proto.MaxTxGroupSize / 2 + invalidRates := []float32{0.5, 0.001} + for _, ivr := range invalidRates { + b.Run(fmt.Sprintf("inv_%.3f", ivr), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, 1, 0, ivr, b, false) + }) + } +} + +// BenchmarkHandleTxns sends signed transactions directly to the verifier +func BenchmarkHandleMsigTxns(b *testing.B) { + maxGroupSize := 1 + msigSizes := []int{255, 64, 16} + invalidRates := []float32{0.5, 0.001} + for _, msigSize := range msigSizes { for _, ivr := range invalidRates { - b.Run(fmt.Sprintf("tps_%d_inv_%.3f", tps, ivr), func(b *testing.B) { - runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, false) + b.Run(fmt.Sprintf("msigSize_%d_inv_%.3f", msigSize, ivr), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, 0, ivr, b, false) }) } } } // BenchmarkHandleTxnGroups sends signed transaction groups directly to the verifier -func BenchmarkHandleTxnGroups(b *testing.B) { +func BenchmarkHandleMsigTxnGroups(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 - tpss := []int{6000000, 600000, 60000, 6000} + msigSizes := []int{255, 64, 16} invalidRates := []float32{0.5, 0.001} - for _, tps := range tpss { + for _, msigSize := range msigSizes { for _, ivr := range invalidRates { - b.Run(fmt.Sprintf("tps_%d_inv_%.3f", tps, ivr), func(b *testing.B) { - runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, false) + b.Run(fmt.Sprintf("msigSize_%d_inv_%.3f", msigSize, ivr), func(b *testing.B) { + runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, 0, ivr, b, false) }) } } @@ -1308,7 +1412,7 @@ func BenchmarkHandleBLWTxns(b *testing.B) { for _, tps := range tpss { for _, ivr := range invalidRates { b.Run(fmt.Sprintf("tps_%d_inv_%.3f", tps, ivr), func(b *testing.B) { - runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, true) + runHandlerBenchmarkWithBacklog(maxGroupSize, 1, tps, ivr, b, true) }) } } @@ -1323,14 +1427,14 @@ func BenchmarkHandleBLWTxnGroups(b *testing.B) { for _, tps := range tpss { for _, ivr := range invalidRates { b.Run(fmt.Sprintf("tps_%d_inv_%.3f", tps, ivr), func(b *testing.B) { - runHandlerBenchmarkWithBacklog(maxGroupSize, tps, ivr, b, true) + runHandlerBenchmarkWithBacklog(maxGroupSize, 1, tps, ivr, b, true) }) } } } // runHandlerBenchmarkWithBacklog benchmarks the number of transactions verfied or dropped -func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, b *testing.B, useBacklogWorker bool) { +func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate float32, b *testing.B, useBacklogWorker bool) { defer func() { // reset the counters transactionMessagesDroppedFromBacklog = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromBacklog) @@ -1408,7 +1512,14 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, } // Prepare the transactions - signedTransactionGroups, badTxnGroups := makeSignedTxnGroups(b.N, numUsers, maxGroupSize, invalidRate, addresses, secrets) + var signedTransactionGroups [][]transactions.SignedTxn + var badTxnGroups map[uint64]interface{} + if msigSize == 1 { + signedTransactionGroups, badTxnGroups = makeSignedTxnGroups(b.N, numUsers, maxGroupSize, invalidRate, addresses, secrets) + } else { + signedTransactionGroups, badTxnGroups, err = makeMsigSignedTxnGroups(b.N, numUsers, msigSize, maxGroupSize, invalidRate, addresses, secrets) + require.NoError(b, err) + } var encodedSignedTransactionGroups []network.IncomingMessage if useBacklogWorker { encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, b.N) @@ -1424,7 +1535,10 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, var tt time.Time // Process the results and make sure they are correct - rateAdjuster := time.Second / time.Duration(tps) + var rateAdjuster time.Duration + if tps > 0 { + rateAdjuster = time.Second / time.Duration(tps) + } wg.Add(1) go func() { defer wg.Done() @@ -1434,7 +1548,9 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, defer func() { if groupCounter > 1 { droppedBacklog, droppedPool := getDropped() - b.Logf("Input T(grp)PS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + if tps > 0 { + b.Logf("Input T(grp)PS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) + } b.Logf("Verified TPS: %d", uint64(txnCounter)*uint64(time.Second)/uint64(time.Since(tt))) b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) @@ -1489,7 +1605,6 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, tps int, invalidRate float32, for _, stxngrp := range signedTransactionGroups { blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, &blm, nil) - time.Sleep(rateAdjuster) } } wg.Wait() From bc47cf7d145218b3a05980e938c75f88795ad9f2 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 00:47:41 -0500 Subject: [PATCH 137/156] - multisig prep in parallel - limit txn gen and reuse for longer runs - race protect the test --- data/txHandler_test.go | 164 ++++++++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 52 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 15511cf0a7..de4ab22ac0 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1286,48 +1286,81 @@ func signTransactionGroups(txnGroups [][]transactions.Transaction, secrets []*cr func signMSigTransactionGroups(txnGroups [][]transactions.Transaction, secrets []*crypto.SignatureSecrets, invalidProb float32, msigSize int) (ret [][]transactions.SignedTxn, badTxnGroups map[uint64]interface{}, err error) { - + ret = make([][]transactions.SignedTxn, len(txnGroups)) numUsers := len(secrets) badTxnGroups = make(map[uint64]interface{}) + badTxnGroupsMU := sync.Mutex{} + // process them using 6 threads + workers := make(chan interface{}, 6) + wg := sync.WaitGroup{} + errChan := make(chan error, 1) for tg := range txnGroups { - msigVer := uint8(1) - msigTHld := uint8(msigSize) - pks := make([]crypto.PublicKey, msigSize) - for x := 0; x < msigSize; x++ { - pks[x] = secrets[(tg+x)%numUsers].SignatureVerifier - } - multiSigAddr, err := crypto.MultisigAddrGen(msigVer, msigTHld, pks) - if err != nil { - return ret, badTxnGroups, err - } - grpSize := len(txnGroups[tg]) - signedTxGroup := make([]transactions.SignedTxn, grpSize) - sigsForTxn := make([]crypto.MultisigSig, msigTHld) - - for t := range txnGroups[tg] { - txnGroups[tg][t].Sender = basics.Address(multiSigAddr) - for s := range sigsForTxn { - sig, err := crypto.MultisigSign(txnGroups[tg][t], crypto.Digest(multiSigAddr), msigVer, msigTHld, pks, *secrets[(tg+s)%numUsers]) + wg.Add(1) + workers <- struct{}{} + go func(i int) { + defer func() { + wg.Done() + <-workers + }() + msigVer := uint8(1) + msigTHld := uint8(msigSize) + pks := make([]crypto.PublicKey, msigSize) + for x := 0; x < msigSize; x++ { + pks[x] = secrets[(i+x)%numUsers].SignatureVerifier + } + multiSigAddr, err := crypto.MultisigAddrGen(msigVer, msigTHld, pks) + if err != nil { + select { + case errChan <- err: + return + default: + return + } + } + grpSize := len(txnGroups[i]) + signedTxGroup := make([]transactions.SignedTxn, grpSize) + sigsForTxn := make([]crypto.MultisigSig, msigTHld) + + for t := range txnGroups[i] { + txnGroups[i][t].Sender = basics.Address(multiSigAddr) + for s := range sigsForTxn { + sig, err := crypto.MultisigSign(txnGroups[i][t], crypto.Digest(multiSigAddr), msigVer, msigTHld, pks, *secrets[(i+s)%numUsers]) + if err != nil { + select { + case errChan <- err: + return + default: + return + } + } + sigsForTxn[s] = sig + } + msig, err := crypto.MultisigAssemble(sigsForTxn) if err != nil { - return ret, badTxnGroups, err + select { + case errChan <- err: + return + default: + return + } } - sigsForTxn[s] = sig + signedTxGroup[t].Txn = txnGroups[i][t] + signedTxGroup[t].Msig = msig } - msig, err := crypto.MultisigAssemble(sigsForTxn) - if err != nil { - return ret, badTxnGroups, err + // randomly make bad signatures + if rand.Float32() < invalidProb { + tinGrp := rand.Intn(grpSize) + signedTxGroup[tinGrp].Msig.Threshold = signedTxGroup[tinGrp].Msig.Threshold - 1 + badTxnGroupsMU.Lock() + badTxnGroups[uint64(i)] = struct{}{} + badTxnGroupsMU.Unlock() } - signedTxGroup[t].Txn = txnGroups[tg][t] - signedTxGroup[t].Msig = msig - } - // randomly make bad signatures - if rand.Float32() < invalidProb { - tinGrp := rand.Intn(grpSize) - signedTxGroup[tinGrp].Msig.Threshold = signedTxGroup[tinGrp].Msig.Threshold - 1 - badTxnGroups[uint64(tg)] = struct{}{} - } - ret = append(ret, signedTxGroup) + ret[i] = signedTxGroup + }(tg) } + wg.Wait() + close(errChan) + err = <-errChan return } @@ -1441,7 +1474,7 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate transactionMessagesDroppedFromPool = metrics.MakeCounter(metrics.TransactionMessagesDroppedFromPool) }() - const numUsers = 100 + const numUsers = 512 log := logging.TestingLog(b) log.SetLevel(logging.Warn) @@ -1511,18 +1544,22 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate }() } - // Prepare the transactions + // Prepare 1000 transactions + genTCount := 1000 + if b.N < genTCount { + genTCount = b.N + } var signedTransactionGroups [][]transactions.SignedTxn var badTxnGroups map[uint64]interface{} if msigSize == 1 { - signedTransactionGroups, badTxnGroups = makeSignedTxnGroups(b.N, numUsers, maxGroupSize, invalidRate, addresses, secrets) + signedTransactionGroups, badTxnGroups = makeSignedTxnGroups(genTCount, numUsers, maxGroupSize, invalidRate, addresses, secrets) } else { - signedTransactionGroups, badTxnGroups, err = makeMsigSignedTxnGroups(b.N, numUsers, msigSize, maxGroupSize, invalidRate, addresses, secrets) + signedTransactionGroups, badTxnGroups, err = makeMsigSignedTxnGroups(genTCount, numUsers, msigSize, maxGroupSize, invalidRate, addresses, secrets) require.NoError(b, err) } var encodedSignedTransactionGroups []network.IncomingMessage if useBacklogWorker { - encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, b.N) + encodedSignedTransactionGroups = make([]network.IncomingMessage, 0, genTCount) for _, stxngrp := range signedTransactionGroups { data := make([]byte, 0) for _, stxn := range stxngrp { @@ -1547,23 +1584,31 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate invalidCounter := 0 defer func() { if groupCounter > 1 { + timeSinceStart := time.Since(tt) droppedBacklog, droppedPool := getDropped() if tps > 0 { b.Logf("Input T(grp)PS: %d (delay %f microsec)", tps, float64(rateAdjuster)/float64(time.Microsecond)) } - b.Logf("Verified TPS: %d", uint64(txnCounter)*uint64(time.Second)/uint64(time.Since(tt))) - b.Logf("Time/txn: %d(microsec)", uint64((time.Since(tt)/time.Microsecond))/txnCounter) + b.Logf("Verified TPS: %d T(grp)PS: %d", uint64(txnCounter)*uint64(time.Second)/uint64(timeSinceStart), + uint64(groupCounter)*uint64(time.Second)/uint64(timeSinceStart)) + b.Logf("Time/txn: %d(microsec)", uint64(timeSinceStart/time.Microsecond)/txnCounter) b.Logf("processed total: [%d groups (%d invalid)] [%d txns]", groupCounter, invalidCounter, txnCounter) b.Logf("dropped: [%d backlog] [%d pool]\n", droppedBacklog, droppedPool) } handler.Stop() // cancel the handler ctx }() + counterMutex := sync.Mutex{} stopChan := make(chan interface{}) + wg.Add(1) go func() { + defer wg.Done() for { time.Sleep(200 * time.Millisecond) droppedBacklog, droppedPool := getDropped() - if int(groupCounter+droppedBacklog+droppedPool) == len(signedTransactionGroups) { + counterMutex.Lock() + counters := groupCounter + droppedBacklog + droppedPool + counterMutex.Unlock() + if int(counters) == b.N { // all the benchmark txns processed close(stopChan) return @@ -1575,7 +1620,9 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate select { case wi := <-testResultChan: txnCounter = txnCounter + uint64(len(wi.unverifiedTxGroup)) + counterMutex.Lock() groupCounter++ + counterMutex.Unlock() u, _ := binary.Uvarint(wi.unverifiedTxGroup[0].Txn.Note) _, inBad := badTxnGroups[u] if wi.verificationErr == nil { @@ -1584,7 +1631,7 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate invalidCounter++ require.True(b, inBad, "Error for good signature") } - if groupCounter == uint64(len(signedTransactionGroups)) { + if groupCounter == uint64(b.N) { // all the benchmark txns processed return } @@ -1594,17 +1641,30 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate } }() + completed := false + c := 0 + ticker := &time.Ticker{} + if rateAdjuster > 0 { + ticker = time.NewTicker(rateAdjuster) + } + defer ticker.Stop() b.ResetTimer() tt = time.Now() - if useBacklogWorker { - for _, tg := range encodedSignedTransactionGroups { - handler.processIncomingTxn(tg) - time.Sleep(rateAdjuster) - } - } else { - for _, stxngrp := range signedTransactionGroups { - blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} - handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, &blm, nil) + for !completed { + for i, _ := range signedTransactionGroups { + if useBacklogWorker { + handler.processIncomingTxn(encodedSignedTransactionGroups[i]) + <-ticker.C + } else { + stxngrp := signedTransactionGroups[i] + blm := txBacklogMsg{rawmsg: nil, unverifiedTxGroup: stxngrp} + handler.txVerificationPool.EnqueueBacklog(handler.ctx, handler.asyncVerifySignature, &blm, nil) + } + c++ + if c == b.N { + completed = true + break + } } } wg.Wait() From ae64c6576cd81fec4e2a28c2a09e5d84036ae10a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 00:58:55 -0500 Subject: [PATCH 138/156] gofmt -s --- data/txHandler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index de4ab22ac0..562a86cfda 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1651,7 +1651,7 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate b.ResetTimer() tt = time.Now() for !completed { - for i, _ := range signedTransactionGroups { + for i := range signedTransactionGroups { if useBacklogWorker { handler.processIncomingTxn(encodedSignedTransactionGroups[i]) <-ticker.C From df65d12e10300f455f458f961a9bcf4bb3d56fe5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 01:12:59 -0500 Subject: [PATCH 139/156] msgp gen --- agreement/msgp_gen.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 3c396226b8..8c75e7034d 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3555,10 +3555,10 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // map header, size 8 // string "Deadline" o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).Deadline) + o = (*z).Deadline.MarshalMsg(o) // string "FastRecoveryDeadline" o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) + o = (*z).FastRecoveryDeadline.MarshalMsg(o) // string "LastConcluding" o = append(o, 0xae, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x69, 0x6e, 0x67) o = msgp.AppendUint64(o, uint64((*z).LastConcluding)) @@ -3644,7 +3644,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + bts, err = (*z).Deadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") return @@ -3660,7 +3660,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) + bts, err = (*z).FastRecoveryDeadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FastRecoveryDeadline") return @@ -3734,7 +3734,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { (*z).LastConcluding = step(zb0008) } case "Deadline": - (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) + bts, err = (*z).Deadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "Deadline") return @@ -3746,7 +3746,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "FastRecoveryDeadline": - (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) + bts, err = (*z).FastRecoveryDeadline.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "FastRecoveryDeadline") return @@ -3777,13 +3777,13 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + (*z).Deadline.Msgsize() + 8 + msgp.BoolSize + 21 + (*z).FastRecoveryDeadline.Msgsize() + 8 + (*z).Pending.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline.MsgIsZero()) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline.MsgIsZero()) && ((*z).Pending.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler From 7628495fcb1524cdd8c093f2a45b54e5546e2381 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 10:26:58 -0500 Subject: [PATCH 140/156] msgp --- agreement/msgp_gen.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/agreement/msgp_gen.go b/agreement/msgp_gen.go index 8c75e7034d..3c396226b8 100644 --- a/agreement/msgp_gen.go +++ b/agreement/msgp_gen.go @@ -3555,10 +3555,10 @@ func (z *player) MarshalMsg(b []byte) (o []byte) { // map header, size 8 // string "Deadline" o = append(o, 0x88, 0xa8, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = (*z).Deadline.MarshalMsg(o) + o = msgp.AppendDuration(o, (*z).Deadline) // string "FastRecoveryDeadline" o = append(o, 0xb4, 0x46, 0x61, 0x73, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x44, 0x65, 0x61, 0x64, 0x6c, 0x69, 0x6e, 0x65) - o = (*z).FastRecoveryDeadline.MarshalMsg(o) + o = msgp.AppendDuration(o, (*z).FastRecoveryDeadline) // string "LastConcluding" o = append(o, 0xae, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x69, 0x6e, 0x67) o = msgp.AppendUint64(o, uint64((*z).LastConcluding)) @@ -3644,7 +3644,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - bts, err = (*z).Deadline.UnmarshalMsg(bts) + (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "Deadline") return @@ -3660,7 +3660,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - bts, err = (*z).FastRecoveryDeadline.UnmarshalMsg(bts) + (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FastRecoveryDeadline") return @@ -3734,7 +3734,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { (*z).LastConcluding = step(zb0008) } case "Deadline": - bts, err = (*z).Deadline.UnmarshalMsg(bts) + (*z).Deadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "Deadline") return @@ -3746,7 +3746,7 @@ func (z *player) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "FastRecoveryDeadline": - bts, err = (*z).FastRecoveryDeadline.UnmarshalMsg(bts) + (*z).FastRecoveryDeadline, bts, err = msgp.ReadDurationBytes(bts) if err != nil { err = msgp.WrapError(err, "FastRecoveryDeadline") return @@ -3777,13 +3777,13 @@ func (_ *player) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *player) Msgsize() (s int) { - s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + (*z).Deadline.Msgsize() + 8 + msgp.BoolSize + 21 + (*z).FastRecoveryDeadline.Msgsize() + 8 + (*z).Pending.Msgsize() + s = 1 + 6 + (*z).Round.Msgsize() + 7 + msgp.Uint64Size + 5 + msgp.Uint64Size + 15 + msgp.Uint64Size + 9 + msgp.DurationSize + 8 + msgp.BoolSize + 21 + msgp.DurationSize + 8 + (*z).Pending.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *player) MsgIsZero() bool { - return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline.MsgIsZero()) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline.MsgIsZero()) && ((*z).Pending.MsgIsZero()) + return ((*z).Round.MsgIsZero()) && ((*z).Period == 0) && ((*z).Step == 0) && ((*z).LastConcluding == 0) && ((*z).Deadline == 0) && ((*z).Napping == false) && ((*z).FastRecoveryDeadline == 0) && ((*z).Pending.MsgIsZero()) } // MarshalMsg implements msgp.Marshaler From 0529bd98b92a6a406147d5311717a22ea25f0103 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 11:42:16 -0500 Subject: [PATCH 141/156] fix sig invalidation --- data/txHandler_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 562a86cfda..dd1b3800e6 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1350,7 +1350,8 @@ func signMSigTransactionGroups(txnGroups [][]transactions.Transaction, secrets [ // randomly make bad signatures if rand.Float32() < invalidProb { tinGrp := rand.Intn(grpSize) - signedTxGroup[tinGrp].Msig.Threshold = signedTxGroup[tinGrp].Msig.Threshold - 1 + tinMsig := rand.Intn(len(signedTxGroup[tinGrp].Msig.Subsigs)) + signedTxGroup[tinGrp].Msig.Subsigs[tinMsig].Sig[0] = signedTxGroup[tinGrp].Msig.Subsigs[tinMsig].Sig[0] + 1 badTxnGroupsMU.Lock() badTxnGroups[uint64(i)] = struct{}{} badTxnGroupsMU.Unlock() From 0fef8cc43ced455b96543d35191227a2ee40f621 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 12:02:28 -0500 Subject: [PATCH 142/156] use available cpus --- data/txHandler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 69387c5695..2c17487888 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1290,8 +1290,8 @@ func signMSigTransactionGroups(txnGroups [][]transactions.Transaction, secrets [ numUsers := len(secrets) badTxnGroups = make(map[uint64]interface{}) badTxnGroupsMU := sync.Mutex{} - // process them using 6 threads - workers := make(chan interface{}, 6) + // process them using multiple threads + workers := make(chan interface{}, runtime.NumCPU()-1) wg := sync.WaitGroup{} errChan := make(chan error, 1) for tg := range txnGroups { From 6dab0a0600f927ed174d45cbef2f086944cd7a47 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 15 Dec 2022 12:14:13 -0500 Subject: [PATCH 143/156] slow test treatment --- data/accountManager_test.go | 11 ++++++----- data/ledger_test.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/data/accountManager_test.go b/data/accountManager_test.go index d62a0490db..b182582193 100644 --- a/data/accountManager_test.go +++ b/data/accountManager_test.go @@ -43,6 +43,11 @@ import ( func TestAccountManagerKeys(t *testing.T) { partitiontest.PartitionTest(t) + if testing.Short() { + t.Log("this is a long test and skipping for -short") + return + } + registry := &mocks.MockParticipationRegistry{} testAccountManagerKeys(t, registry, false) } @@ -84,22 +89,18 @@ func registryCloseTest(t testing.TB, registry account.ParticipationRegistry, dbf } func TestAccountManagerKeysRegistry(t *testing.T) { + partitiontest.PartitionTest(t) if testing.Short() { t.Log("this is a long test and skipping for -short") return } - partitiontest.PartitionTest(t) registry, dbName := getRegistryImpl(t, false, true) defer registryCloseTest(t, registry, dbName) testAccountManagerKeys(t, registry, true) } func testAccountManagerKeys(t *testing.T, registry account.ParticipationRegistry, flushRegistry bool) { - if testing.Short() { - t.Log("this is a long test and skipping for -short") - return - } log := logging.TestingLog(t) log.SetLevel(logging.Error) diff --git a/data/ledger_test.go b/data/ledger_test.go index 9469e578df..985e892714 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -325,11 +325,11 @@ func TestLedgerSeed(t *testing.T) { } func TestConsensusVersion(t *testing.T) { + partitiontest.PartitionTest(t) if testing.Short() { t.Log("this is a long test and skipping for -short") return } - partitiontest.PartitionTest(t) // find a consensus protocol that leads to ConsensusCurrentVersion var previousProtocol protocol.ConsensusVersion From ba4550bb251ebf2cb85d6e0d652c47b73ef8812f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 16 Dec 2022 13:04:40 -0500 Subject: [PATCH 144/156] use deadlock mutex --- data/txHandler_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 1d629a2c21..cd022da210 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -33,6 +33,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/algorand/go-deadlock" + "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/components/mocks" "github.com/algorand/go-algorand/config" @@ -49,8 +51,6 @@ import ( "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/execpool" "github.com/algorand/go-algorand/util/metrics" - - "github.com/algorand/go-deadlock" ) func makeTestGenesisAccounts(tb require.TestingT, numUsers int) ([]basics.Address, []*crypto.SignatureSecrets, map[basics.Address]basics.AccountData) { @@ -1318,7 +1318,7 @@ func signMSigTransactionGroups(txnGroups [][]transactions.Transaction, secrets [ ret = make([][]transactions.SignedTxn, len(txnGroups)) numUsers := len(secrets) badTxnGroups = make(map[uint64]interface{}) - badTxnGroupsMU := sync.Mutex{} + badTxnGroupsMU := deadlock.Mutex{} // process them using multiple threads workers := make(chan interface{}, runtime.NumCPU()-1) wg := sync.WaitGroup{} @@ -1637,7 +1637,7 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate } handler.Stop() // cancel the handler ctx }() - counterMutex := sync.Mutex{} + counterMutex := deadlock.Mutex{} stopChan := make(chan interface{}) // monitor the counters to tell when everything is processed and the checker should stop wg.Add(1) From cb0f2b869d30aba6ce2bb893d2af74763f1b9391 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 16 Dec 2022 16:16:25 -0500 Subject: [PATCH 145/156] disable dedup when reuing txn with backlog worker --- data/txHandler_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/data/txHandler_test.go b/data/txHandler_test.go index cd022da210..fbfc277ec7 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1524,6 +1524,14 @@ func runHandlerBenchmarkWithBacklog(maxGroupSize, msigSize, tps int, invalidRate require.NoError(b, err) defer handler.txVerificationPool.Shutdown() defer close(handler.streamVerifierDropped) + + // The benchmark generates only 1000 txns, and reuses them. This is done for faster benchmark time and the + // ability to have long runs without being limited to the memory. The dedup will block the txns once the same + // ones are rotated again. If the purpose is to test dedup, then this can be changed by setting + // genTCount = b.N + handler.cacheConfig.enableFilteringRawMsg = false + handler.cacheConfig.enableFilteringCanonical = false + // since Start is not called, set the context here handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) defer handler.ctxCancel() From 2778ba6887aef7b57b6b54d88a224c36de25fd0c Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 16 Dec 2022 16:41:54 -0500 Subject: [PATCH 146/156] blank line --- data/accountManager_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/data/accountManager_test.go b/data/accountManager_test.go index b182582193..08a9cdb5ab 100644 --- a/data/accountManager_test.go +++ b/data/accountManager_test.go @@ -101,7 +101,6 @@ func TestAccountManagerKeysRegistry(t *testing.T) { } func testAccountManagerKeys(t *testing.T, registry account.ParticipationRegistry, flushRegistry bool) { - log := logging.TestingLog(t) log.SetLevel(logging.Error) From fc8c960fc54a2abd72426561b1a41c4a52dbaa72 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 22 Dec 2022 23:33:39 -0500 Subject: [PATCH 147/156] CR: simplify timer ticks --- data/transactions/verify/txn.go | 24 +++--------------------- data/txHandler_test.go | 7 +++---- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index f650001bec..701c24e572 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -72,9 +72,6 @@ const batchSizeBlockLimit = 1024 // for signature evaluation before waitForNextTxnDuration. const waitForNextTxnDuration = 5 * time.Millisecond -// waitForFirstTxnDuration is the time to wait for the first transaction in the batch -var waitForFirstTxnDuration = 2000 * time.Millisecond - // When the PaysetGroups is generating worksets, it enqueues up to concurrentWorksets entries to the execution pool. This serves several // purposes : // - if the verification task need to be aborted, there are only concurrentWorksets entries that are currently redundant on the execution pool queue. @@ -562,8 +559,8 @@ func (w *worksetBuilder) completed() bool { } // UnverifiedElement is the element passed to the Stream verifier -// BacklogMessage is a reference to the backlog message, which needs to be passed -// with the result +// BacklogMessage is a *txBacklogMsg from data/txHandler.go which needs to be +// passed back to that context type UnverifiedElement struct { TxnGroup []transactions.SignedTxn BacklogMessage interface{} @@ -698,7 +695,7 @@ func (sv *StreamVerifier) cleanup(pending []*UnverifiedElement) { func (sv *StreamVerifier) batchingLoop() { defer sv.activeLoopWg.Done() - timer := time.NewTicker(waitForFirstTxnDuration) + timer := time.NewTicker(waitForNextTxnDuration) defer timer.Stop() var added bool var numberOfSigsInCurrent uint64 @@ -708,7 +705,6 @@ func (sv *StreamVerifier) batchingLoop() { for { select { case stx := <-sv.stxnChan: - isFirstInBatch := numberOfSigsInCurrent == 0 numberOfBatchableSigsInGroup, err := getNumberOfBatchableSigsInGroup(stx.TxnGroup) if err != nil { // wrong number of signatures @@ -749,27 +745,16 @@ func (sv *StreamVerifier) batchingLoop() { if added { numberOfSigsInCurrent = 0 ue = make([]*UnverifiedElement, 0) - // starting a new batch. Can wait long, since nothing is blocked - timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length - timer.Reset(waitForNextTxnDuration) numberOfTimerResets++ } - } else { - if isFirstInBatch { - // an element is added and is waiting. shorten the waiting time - timer.Reset(waitForNextTxnDuration) - numberOfTimerResets = 0 - } } case <-timer.C: // timer ticked. it is time to send the batch even if it is not full if numberOfSigsInCurrent == 0 { // nothing batched yet... wait some more - timer.Reset(waitForFirstTxnDuration) - numberOfTimerResets = 0 continue } var err error @@ -788,12 +773,9 @@ func (sv *StreamVerifier) batchingLoop() { if added { numberOfSigsInCurrent = 0 ue = make([]*UnverifiedElement, 0) - // starting a new batch. Can wait long, since nothing is blocked - timer.Reset(waitForFirstTxnDuration) numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length. wait for some more txns - timer.Reset(waitForNextTxnDuration) numberOfTimerResets++ } case <-sv.ctx.Done(): diff --git a/data/txHandler_test.go b/data/txHandler_test.go index e4bff430c4..c2cb2cc364 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -1155,8 +1155,7 @@ func incomingTxHandlerProcessing(maxGroupSize, numberOfTransactionGroups int, t require.NoError(t, err) defer ledger.Close() - l := ledger - handler, err := makeTestTxHandler(l, cfg) + handler, err := makeTestTxHandler(ledger, cfg) require.NoError(t, err) defer handler.txVerificationPool.Shutdown() defer close(handler.streamVerifierDropped) @@ -1480,7 +1479,7 @@ func BenchmarkHandleTxnGroups(b *testing.B) { // BenchmarkHandleMsigTxns sends signed transactions directly to the verifier func BenchmarkHandleMsigTxns(b *testing.B) { maxGroupSize := 1 - msigSizes := []int{255, 64, 16} + msigSizes := []int{64, 16, 8, 4} invalidRates := []float32{0.5, 0.001} for _, msigSize := range msigSizes { for _, ivr := range invalidRates { @@ -1495,7 +1494,7 @@ func BenchmarkHandleMsigTxns(b *testing.B) { // BenchmarkHandleTxnGroups sends signed transaction groups directly to the verifier func BenchmarkHandleMsigTxnGroups(b *testing.B) { maxGroupSize := proto.MaxTxGroupSize / 2 - msigSizes := []int{255, 64, 16} + msigSizes := []int{64, 16, 8, 4} invalidRates := []float32{0.5, 0.001} for _, msigSize := range msigSizes { for _, ivr := range invalidRates { From 9f6a62c66257c726605d7d9a3148cd29091b22f2 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Dec 2022 00:15:55 -0500 Subject: [PATCH 148/156] update the test --- data/transactions/verify/txn_test.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index eb08bb35ef..efb7877d13 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1121,16 +1121,9 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= func TestStreamVerifierIdel(t *testing.T) { partitiontest.PartitionTest(t) - numOfTxns := 10 + numOfTxns := 1 txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) - origValue := waitForFirstTxnDuration - defer func() { - waitForFirstTxnDuration = origValue - }() - // set this value too small to hit the timeout first, then make sure can - // resume and process the incoming transactions - waitForFirstTxnDuration = 1 * time.Microsecond sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) sv.WaitForStop() } From 0c04cf790db98d855d2ab0906acc306970e9dd72 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Dec 2022 10:58:15 -0500 Subject: [PATCH 149/156] fix code merge issue --- ledger/simulation/simulator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledger/simulation/simulator.go b/ledger/simulation/simulator.go index a6b12b6832..b8a136a4b7 100644 --- a/ledger/simulation/simulator.go +++ b/ledger/simulation/simulator.go @@ -150,7 +150,7 @@ func (s Simulator) check(hdr bookkeeping.BlockHeader, txgroup []transactions.Sig } // Verify the signed transactions are well-formed and have valid signatures - _, err = verify.TxnGroup(txnsToVerify, hdr, nil, s.ledger) + _, err = verify.TxnGroup(txnsToVerify, &hdr, nil, s.ledger) if err != nil { return false, InvalidTxGroupError{SimulatorError{err}} } From 0d9a2339cf10f7041bacea31f2f14a23d407854a Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 23 Dec 2022 15:42:54 -0500 Subject: [PATCH 150/156] fix test stability issue --- data/transactions/verify/txn_test.go | 70 ++++++++++--------- .../verify/verifiedTxnCache_test.go | 12 ++-- 2 files changed, 43 insertions(+), 39 deletions(-) diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index efb7877d13..8440a4aa3b 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -170,7 +170,7 @@ func generateAccounts(numAccs int) ([]*crypto.SignatureSecrets, []basics.Address return secrets, addresses, pks } -func generateTestObjects(numTxs, numAccs int, blockRound basics.Round) ([]transactions.Transaction, []transactions.SignedTxn, []*crypto.SignatureSecrets, []basics.Address) { +func generateTestObjects(numTxs, numAccs, noteOffset int, blockRound basics.Round) ([]transactions.Transaction, []transactions.SignedTxn, []*crypto.SignatureSecrets, []basics.Address) { txs := make([]transactions.Transaction, numTxs) signed := make([]transactions.SignedTxn, numTxs) secrets, addresses, _ := generateAccounts(numAccs) @@ -192,7 +192,7 @@ func generateTestObjects(numTxs, numAccs int, blockRound basics.Round) ([]transa txs[i] = createPayTransaction(f, iss, exp, a, addresses[s], addresses[r]) noteField := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(noteField, uint64(i)) + binary.PutUvarint(noteField, uint64(i+noteOffset)) txs[i].Note = noteField signed[i] = txs[i].Sign(secrets[s]) @@ -207,7 +207,7 @@ func TestSignedPayment(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] - payments, stxns, secrets, addrs := generateTestObjects(1, 1, 0) + payments, stxns, secrets, addrs := generateTestObjects(1, 1, 0, 0) payment, stxn, secret, addr := payments[0], stxns[0], secrets[0], addrs[0] groupCtx, err := PrepareGroupContext(stxns, blockHeader, nil) @@ -228,7 +228,7 @@ func TestSignedPayment(t *testing.T) { func TestTxnValidationEncodeDecode(t *testing.T) { partitiontest.PartitionTest(t) - _, signed, _, _ := generateTestObjects(100, 50, 0) + _, signed, _, _ := generateTestObjects(100, 50, 0, 0) for _, txn := range signed { groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{txn}, blockHeader, nil) @@ -250,7 +250,7 @@ func TestTxnValidationEncodeDecode(t *testing.T) { func TestTxnValidationEmptySig(t *testing.T) { partitiontest.PartitionTest(t) - _, signed, _, _ := generateTestObjects(100, 50, 0) + _, signed, _, _ := generateTestObjects(100, 50, 0, 0) for _, txn := range signed { groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{txn}, blockHeader, nil) @@ -366,7 +366,7 @@ func TestPaysetGroups(t *testing.T) { return } - _, signedTxn, secrets, addrs := generateTestObjects(10000, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(10000, 20, 0, 50) blkHdr := createDummyBlockHeader() execPool := execpool.MakePool(t) @@ -393,7 +393,7 @@ func TestPaysetGroups(t *testing.T) { // we define a test that would take 10 seconds to execute, and try to abort at 1.5 seconds. txnCount := len(signedTxn) * 10 * int(time.Second/paysetGroupDuration) - _, signedTxn, secrets, addrs = generateTestObjects(txnCount, 20, 50) + _, signedTxn, secrets, addrs = generateTestObjects(txnCount, 20, 0, 50) txnGroups = generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) @@ -436,7 +436,7 @@ func BenchmarkPaysetGroups(b *testing.B) { if b.N < 2000 { b.N = 2000 } - _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 0, 50) blkHdr := createDummyBlockHeader() execPool := execpool.MakePool(b) @@ -455,7 +455,7 @@ func BenchmarkPaysetGroups(b *testing.B) { func TestTxnGroupMixedSignatures(t *testing.T) { partitiontest.PartitionTest(t) - _, signedTxn, secrets, addrs := generateTestObjects(1, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(1, 20, 0, 50) blkHdr := createDummyBlockHeader() // add a simple logic that verifies this condition: @@ -569,7 +569,7 @@ func generateTransactionGroups(maxGroupSize int, signedTxns []transactions.Signe func TestTxnGroupCacheUpdate(t *testing.T) { partitiontest.PartitionTest(t) - _, signedTxn, secrets, addrs := generateTestObjects(100, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(100, 20, 0, 50) blkHdr := createDummyBlockHeader() txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) @@ -609,7 +609,7 @@ func TestTxnGroupCacheUpdateMultiSig(t *testing.T) { func TestTxnGroupCacheUpdateFailLogic(t *testing.T) { partitiontest.PartitionTest(t) - _, signedTxn, _, _ := generateTestObjects(100, 20, 50) + _, signedTxn, _, _ := generateTestObjects(100, 20, 0, 50) blkHdr := createDummyBlockHeader() // sign the transaction with logic @@ -652,7 +652,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= func TestTxnGroupCacheUpdateLogicWithSig(t *testing.T) { partitiontest.PartitionTest(t) - _, signedTxn, secrets, addresses := generateTestObjects(100, 20, 50) + _, signedTxn, secrets, addresses := generateTestObjects(100, 20, 0, 50) blkHdr := createDummyBlockHeader() for i := 0; i < len(signedTxn); i++ { @@ -866,7 +866,7 @@ func BenchmarkTxn(b *testing.B) { if b.N < 2000 { b.N = 2000 } - _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(b.N, 20, 0, 50) blk := bookkeeping.Block{BlockHeader: createDummyBlockHeader()} txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) @@ -984,9 +984,9 @@ func verifyResults(txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64 require.Empty(t, badTxnGroups, "unverifiedGroups should have all the transactions with invalid sigs") } -func getSignedTransactions(numOfTxns, maxGrpSize int, badTxnProb float32) (txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}) { +func getSignedTransactions(numOfTxns, maxGrpSize, noteOffset int, badTxnProb float32) (txnGroups [][]transactions.SignedTxn, badTxnGroups map[uint64]struct{}) { - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, noteOffset, 50) txnGroups = generateTransactionGroups(maxGrpSize, signedTxn, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1009,7 +1009,7 @@ func TestStreamVerifier(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 4000 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0, 0.5) sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) sv.WaitForStop() @@ -1020,7 +1020,7 @@ func TestStreamVerifierCases(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0, 0) mod := 1 // txn with 0 sigs @@ -1031,7 +1031,7 @@ func TestStreamVerifierCases(t *testing.T) { sv.WaitForStop() mod++ - _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1046,7 +1046,7 @@ func TestStreamVerifierCases(t *testing.T) { sv.WaitForStop() mod++ - _, signedTxns, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + _, signedTxns, secrets, addrs = generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1068,7 +1068,7 @@ func TestStreamVerifierCases(t *testing.T) { sv.WaitForStop() mod++ - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1105,7 +1105,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= sv.WaitForStop() mod++ - _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) badTxnGroups = make(map[uint64]struct{}) @@ -1122,7 +1122,7 @@ func TestStreamVerifierIdel(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0, 0.5) sv := streamVerifierTestCore(txnGroups, badTxnGroups, nil, t) sv.WaitForStop() @@ -1132,7 +1132,7 @@ func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 10 - txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0) + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0, 0) mod := 1 // txn with 0 sigs @@ -1141,7 +1141,7 @@ func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { require.Error(t, err, errSignedTxnHasNoSig) mod++ - _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxns, secrets, addrs) batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[0]) require.NoError(t, err) @@ -1163,7 +1163,7 @@ func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { require.Equal(t, uint64(2), batchSigs) mod++ - _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs := generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) // logicsig @@ -1185,7 +1185,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= mod++ // txn with sig and msig - _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 50) + _, signedTxn, secrets, addrs = generateTestObjects(numOfTxns, 20, 0, 50) txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) txnGroups[mod][0].Msig = mSigTxn[0].Msig batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[mod]) @@ -1199,7 +1199,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { // only one transaction should be sufficient for the batch verifier // to realize the pool is terminated and to shut down numOfTxns := 1 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) @@ -1283,7 +1283,7 @@ func TestStreamVerifierRestart(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1000 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) @@ -1427,7 +1427,7 @@ func TestStreamVerifierCtxCancel(t *testing.T) { // send batchSizeBlockLimit after the exec pool buffer is full numOfTxns := 1 - txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0, 0.5) stxnChan <- &UnverifiedElement{TxnGroup: txnGroups[0], BacklogMessage: nil} // cancel the ctx before the sig is sent to the exec pool cancel() @@ -1476,7 +1476,7 @@ func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { // send batchSizeBlockLimit after the exec pool buffer is full numOfTxns := 1 - txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, _ := getSignedTransactions(numOfTxns, 1, 0, 0.5) stxnChan <- &UnverifiedElement{TxnGroup: txnGroups[0], BacklogMessage: nil} // cancel the ctx as the sig is not yet sent to the exec pool // the test might sporadically fail if between sending the txn above @@ -1529,7 +1529,7 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { overflow := 3 // send txBacklogSizeMod + 3 transactions to overflow the result buffer numOfTxns := txBacklogSizeMod + overflow - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0, 0.5) numOfTxnGroups := len(txnGroups) for _, tg := range txnGroups { stxnChan <- &UnverifiedElement{TxnGroup: tg, BacklogMessage: nil} @@ -1562,7 +1562,11 @@ func TestStreamVerifierPostVBlocked(t *testing.T) { wg.Add(1) // make sure the other results are fine - txnGroups, badTxnGroups = getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, badTxnGroups2 := getSignedTransactions(numOfTxns, 1, numOfTxns, 0.5) + // need to combine these, since left overs from the previous one could still come out + for b := range badTxnGroups2 { + badTxnGroups[b] = struct{}{} + } go processResults(ctx, errChan, resultChan, numOfTxnGroups, badTxnGroups, &badSigResultCounter, &goodSigResultCounter, &wg) for _, tg := range txnGroups { @@ -1587,7 +1591,7 @@ func TestStreamVerifierMakeStreamVerifierErr(t *testing.T) { func TestStreamVerifierCancelWhenPooled(t *testing.T) { partitiontest.PartitionTest(t) numOfTxns := 1000 - txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0.5) + txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, 1, 0, 0.5) // prepare the stream verifier numOfTxnGroups := len(txnGroups) diff --git a/data/transactions/verify/verifiedTxnCache_test.go b/data/transactions/verify/verifiedTxnCache_test.go index 2070f537aa..553eb33bdd 100644 --- a/data/transactions/verify/verifiedTxnCache_test.go +++ b/data/transactions/verify/verifiedTxnCache_test.go @@ -32,7 +32,7 @@ func TestAddingToCache(t *testing.T) { icache := MakeVerifiedTransactionCache(500) impl := icache.(*verifiedTransactionCache) - _, signedTxn, secrets, addrs := generateTestObjects(10, 5, 50) + _, signedTxn, secrets, addrs := generateTestObjects(10, 5, 0, 50) txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) groupCtx, err := PrepareGroupContext(txnGroups[0], blockHeader, nil) require.NoError(t, err) @@ -52,7 +52,7 @@ func TestBucketCycling(t *testing.T) { entriesPerBucket := 100 icache := MakeVerifiedTransactionCache(entriesPerBucket * (bucketCount - 1)) impl := icache.(*verifiedTransactionCache) - _, signedTxn, _, _ := generateTestObjects(entriesPerBucket*bucketCount*2, bucketCount, 0) + _, signedTxn, _, _ := generateTestObjects(entriesPerBucket*bucketCount*2, bucketCount, 0, 0) require.Equal(t, entriesPerBucket*bucketCount*2, len(signedTxn)) groupCtx, err := PrepareGroupContext([]transactions.SignedTxn{signedTxn[0]}, blockHeader, nil) @@ -82,7 +82,7 @@ func TestGetUnverifiedTransactionGroups50(t *testing.T) { size := 300 icache := MakeVerifiedTransactionCache(size * 2) impl := icache.(*verifiedTransactionCache) - _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10+size/1000, 0) + _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10+size/1000, 0, 0) txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) expectedUnverifiedGroups := make([][]transactions.SignedTxn, 0, len(txnGroups)/2) @@ -107,7 +107,7 @@ func BenchmarkGetUnverifiedTransactionGroups50(b *testing.B) { } icache := MakeVerifiedTransactionCache(b.N * 2) impl := icache.(*verifiedTransactionCache) - _, signedTxn, secrets, addrs := generateTestObjects(b.N*2, 10+b.N/1000, 0) + _, signedTxn, secrets, addrs := generateTestObjects(b.N*2, 10+b.N/1000, 0, 0) txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) queryTxnGroups := make([][]transactions.SignedTxn, 0, b.N) @@ -140,7 +140,7 @@ func TestUpdatePinned(t *testing.T) { size := 100 icache := MakeVerifiedTransactionCache(size * 10) impl := icache.(*verifiedTransactionCache) - _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0) + _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0, 0) txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) // insert some entries. @@ -169,7 +169,7 @@ func TestPinningTransactions(t *testing.T) { size := 100 icache := MakeVerifiedTransactionCache(size) impl := icache.(*verifiedTransactionCache) - _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0) + _, signedTxn, secrets, addrs := generateTestObjects(size*2, 10, 0, 0) txnGroups := generateTransactionGroups(protoMaxGroupSize, signedTxn, secrets, addrs) // insert half of the entries. From 74c4cb1673d181f3ecaabc83af4a4f95f28916a0 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 27 Dec 2022 17:45:49 -0500 Subject: [PATCH 151/156] CR: shorten waitForNextTxnDuration 5->2, slice init sizes --- data/ledger_test.go | 2 +- data/transactions/verify/txn.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data/ledger_test.go b/data/ledger_test.go index 03631f542a..82782d6b90 100644 --- a/data/ledger_test.go +++ b/data/ledger_test.go @@ -126,7 +126,7 @@ func TestLedgerCirculation(t *testing.T) { cfg := config.GetDefaultLocal() cfg.Archival = true log := logging.TestingLog(t) - log.SetLevel(logging.Panic) + log.SetLevel(logging.Warn) realLedger, err := ledger.OpenLedger(log, t.Name(), inMem, genesisInitState, cfg) require.NoError(t, err, "could not open ledger") defer realLedger.Close() diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 701c24e572..b0dcb4f62c 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -70,7 +70,7 @@ const batchSizeBlockLimit = 1024 // since every relay will go through this wait time before broadcasting the txn. // However, when the incoming txn rate is high, the batch will fill up quickly and will send // for signature evaluation before waitForNextTxnDuration. -const waitForNextTxnDuration = 5 * time.Millisecond +const waitForNextTxnDuration = 2 * time.Millisecond // When the PaysetGroups is generating worksets, it enqueues up to concurrentWorksets entries to the execution pool. This serves several // purposes : @@ -622,11 +622,11 @@ type batchLoad struct { messagesForTxn []int } -func makeBatchLoad() (bl batchLoad) { - bl.txnGroups = make([][]transactions.SignedTxn, 0) - bl.groupCtxs = make([]*GroupContext, 0) - bl.elementBacklogMessage = make([]interface{}, 0) - bl.messagesForTxn = make([]int, 0) +func makeBatchLoad(l int) (bl batchLoad) { + bl.txnGroups = make([][]transactions.SignedTxn, 0, l) + bl.groupCtxs = make([]*GroupContext, 0, l) + bl.elementBacklogMessage = make([]interface{}, 0, l) + bl.messagesForTxn = make([]int, 0, l) return bl } @@ -700,7 +700,7 @@ func (sv *StreamVerifier) batchingLoop() { var added bool var numberOfSigsInCurrent uint64 var numberOfTimerResets uint64 - ue := make([]*UnverifiedElement, 0) + ue := make([]*UnverifiedElement, 0, 8) defer func() { sv.cleanup(ue) }() for { select { @@ -744,7 +744,7 @@ func (sv *StreamVerifier) batchingLoop() { } if added { numberOfSigsInCurrent = 0 - ue = make([]*UnverifiedElement, 0) + ue = make([]*UnverifiedElement, 0, 8) numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length @@ -772,7 +772,7 @@ func (sv *StreamVerifier) batchingLoop() { } if added { numberOfSigsInCurrent = 0 - ue = make([]*UnverifiedElement, 0) + ue = make([]*UnverifiedElement, 0, 8) numberOfTimerResets = 0 } else { // was not added because of the exec pool buffer length. wait for some more txns @@ -830,7 +830,7 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(ue []*UnverifiedElemen ue := arg.([]*UnverifiedElement) batchVerifier := crypto.MakeBatchVerifier() - bl := makeBatchLoad() + bl := makeBatchLoad(len(ue)) // TODO: separate operations here, and get the sig verification inside the LogicSig to the batch here blockHeader := sv.nbw.getBlockHeader() for _, ue := range ue { From 964e560eff7870e7cd1e4786690d1f94e0b79fdd Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 28 Dec 2022 17:13:31 -0500 Subject: [PATCH 152/156] CR: refactor and other fixes --- data/transactions/verify/txn.go | 95 ++++++++++++---------------- data/transactions/verify/txn_test.go | 8 +-- data/txDupCache.go | 2 +- data/txHandler.go | 2 +- 4 files changed, 45 insertions(+), 62 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index b0dcb4f62c..ef3d2d302f 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -250,43 +250,46 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl return groupCtx, nil } -// stxnCoreChecks runs signatures validity checks and enqueues signature into batchVerifier for verification. -func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) *TxGroupError { - numSigs := 0 - hasSig := false - hasMsig := false - hasLogicSig := false +// checkTxnSigTypeCounts checks the number of signature types and reports an error in case of a violation +func checkTxnSigTypeCounts(s *transactions.SignedTxn) (isStateProofTxn bool, err *TxGroupError) { + numSigCategories := 0 if s.Sig != (crypto.Signature{}) { - numSigs++ - hasSig = true + numSigCategories++ } if !s.Msig.Blank() { - numSigs++ - hasMsig = true + numSigCategories++ } if !s.Lsig.Blank() { - numSigs++ - hasLogicSig = true + numSigCategories++ } - if numSigs == 0 { + if numSigCategories == 0 { // Special case: special sender address can issue special transaction // types (state proof txn) without any signature. The well-formed // check ensures that this transaction cannot pay any fee, and // cannot have any other interesting fields, except for the state proof payload. if s.Txn.Sender == transactions.StateProofSender && s.Txn.Type == protocol.StateProofTx { - return nil + return true, nil } - return &TxGroupError{err: errTxnSigHasNoSig, Reason: TxGroupErrorReasonHasNoSig} + return false, &TxGroupError{err: errTxnSigHasNoSig, Reason: TxGroupErrorReasonHasNoSig} } - if numSigs > 1 { - return &TxGroupError{err: errTxnSigNotWellFormed, Reason: TxGroupErrorReasonSigNotWellFormed} + if numSigCategories > 1 { + return false, &TxGroupError{err: errTxnSigNotWellFormed, Reason: TxGroupErrorReasonSigNotWellFormed} } + return false, nil +} - if hasSig { +// stxnCoreChecks runs signatures validity checks and enqueues signature into batchVerifier for verification. +func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) *TxGroupError { + isStateProofTxn, err := checkTxnSigTypeCounts(s) + if err != nil || isStateProofTxn { + return err + } + + if s.Sig != (crypto.Signature{}) { batchVerifier.EnqueueSignature(crypto.SignatureVerifier(s.Authorizer()), s.Txn, s.Sig) return nil } - if hasMsig { + if !s.Msig.Blank() { if err := crypto.MultisigBatchPrep(s.Txn, crypto.Digest(s.Authorizer()), s.Msig, batchVerifier); err != nil { return &TxGroupError{err: fmt.Errorf("multisig validation failed: %w", err), Reason: TxGroupErrorReasonMsigNotWellFormed} } @@ -305,7 +308,7 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex } return nil } - if hasLogicSig { + if !s.Lsig.Blank() { if err := logicSigVerify(s, txnIdx, groupCtx); err != nil { return &TxGroupError{err: err, Reason: TxGroupErrorReasonLogicSigFailed} } @@ -680,7 +683,7 @@ func (sv *StreamVerifier) Start(ctx context.Context) { go sv.batchingLoop() } -// WaitForStop waits until the batching loop terminates afer the ctx is cancled +// WaitForStop waits until the batching loop terminates afer the ctx is canceled func (sv *StreamVerifier) WaitForStop() { sv.activeLoopWg.Wait() } @@ -699,7 +702,7 @@ func (sv *StreamVerifier) batchingLoop() { defer timer.Stop() var added bool var numberOfSigsInCurrent uint64 - var numberOfTimerResets uint64 + var numberOfBatchAttempts uint64 ue := make([]*UnverifiedElement, 0, 8) defer func() { sv.cleanup(ue) }() for { @@ -745,10 +748,10 @@ func (sv *StreamVerifier) batchingLoop() { if added { numberOfSigsInCurrent = 0 ue = make([]*UnverifiedElement, 0, 8) - numberOfTimerResets = 0 + numberOfBatchAttempts = 0 } else { // was not added because of the exec pool buffer length - numberOfTimerResets++ + numberOfBatchAttempts++ } } case <-timer.C: @@ -758,7 +761,7 @@ func (sv *StreamVerifier) batchingLoop() { continue } var err error - if numberOfTimerResets > 1 { + if numberOfBatchAttempts > 1 { // bypass the exec pool situation and queue anyway // this is to prevent long delays in transaction propagation // at least one transaction here has waited 3 x waitForNextTxnDuration @@ -773,10 +776,10 @@ func (sv *StreamVerifier) batchingLoop() { if added { numberOfSigsInCurrent = 0 ue = make([]*UnverifiedElement, 0, 8) - numberOfTimerResets = 0 + numberOfBatchAttempts = 0 } else { // was not added because of the exec pool buffer length. wait for some more txns - numberOfTimerResets++ + numberOfBatchAttempts++ } case <-sv.ctx.Done(): return @@ -816,9 +819,9 @@ func (sv *StreamVerifier) tryAddVerificationTaskToThePool(ue []*UnverifiedElemen } func (sv *StreamVerifier) addVerificationTaskToThePoolNow(ue []*UnverifiedElement) error { - // if the context is canceled when the task is in the queue, it should be cancled + // if the context is canceled when the task is in the queue, it should be canceled // copy the ctx here so that when the StreamVerifier is started again, and a new context - // is created, this task still gets cancled due to the ctx at the time of this task + // is created, this task still gets canceled due to the ctx at the time of this task taskCtx := sv.ctx function := func(arg interface{}) interface{} { if taskCtx.Err() != nil { @@ -902,37 +905,17 @@ func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs u } func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) { - var hasSig, hasMsig bool - numSigCategories := 0 - if stx.Sig != (crypto.Signature{}) { - numSigCategories++ - hasSig = true - } - if !stx.Msig.Blank() { - numSigCategories++ - hasMsig = true - } - if !stx.Lsig.Blank() { - numSigCategories++ - } - - if numSigCategories == 0 { - // Special case: special sender address can issue special transaction - // types (state proof txn) without any signature. The well-formed - // check ensures that this transaction cannot pay any fee, and - // cannot have any other interesting fields, except for the state proof payload. - if stx.Txn.Sender == transactions.StateProofSender && stx.Txn.Type == protocol.StateProofTx { - return 0, nil + isStateProofTxn, err := checkTxnSigTypeCounts(stx) + if err != nil || isStateProofTxn { + if err != nil { + return 0, err } - return 0, errSignedTxnHasNoSig - } - if numSigCategories != 1 { - return 0, errSignedTxnMaxOneSig + return 0, nil } - if hasSig { + if stx.Sig != (crypto.Signature{}) { return 1, nil } - if hasMsig { + if !stx.Msig.Blank() { sig := stx.Msig batchSigs := uint64(0) for _, subsigi := range sig.Subsigs { diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index 8440a4aa3b..ff2cf5189a 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -944,15 +944,15 @@ func processResults(ctx context.Context, errChan chan<- error, resultChan <-chan u, _ := binary.Uvarint(result.TxnGroup[0].Txn.Note) if _, has := badTxnGroups[u]; has { (*badSigResultCounter)++ - // we expected an error, but it is not the general crypto error - if result.Err != crypto.ErrBatchHasFailedSigs { - errChan <- result.Err - } if result.Err == nil { err := fmt.Errorf("%dth (%d)transaction varified with a bad sig", x, u) errChan <- err return } + // we expected an error, but it is not the general crypto error + if result.Err != crypto.ErrBatchHasFailedSigs { + errChan <- result.Err + } } else { (*goodSigResultCounter)++ if result.Err != nil { diff --git a/data/txDupCache.go b/data/txDupCache.go index a70c3430ab..83429165cf 100644 --- a/data/txDupCache.go +++ b/data/txDupCache.go @@ -118,7 +118,7 @@ func makeSaltedCache(size int) *txSaltedCache { } } -func (c *txSaltedCache) start(ctx context.Context, refreshInterval time.Duration) { +func (c *txSaltedCache) Start(ctx context.Context, refreshInterval time.Duration) { c.ctx = ctx if refreshInterval != 0 { c.wg.Add(1) diff --git a/data/txHandler.go b/data/txHandler.go index 02d630ec66..78704f0f88 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -219,7 +219,7 @@ func (handler *TxHandler) droppedTxnWatcher() { // Start enables the processing of incoming messages at the transaction handler func (handler *TxHandler) Start() { handler.ctx, handler.ctxCancel = context.WithCancel(context.Background()) - handler.msgCache.start(handler.ctx, 60*time.Second) + handler.msgCache.Start(handler.ctx, 60*time.Second) handler.net.RegisterHandlers([]network.TaggedMessageHandler{ {Tag: protocol.TxnTag, MessageHandler: network.HandlerFunc(handler.processIncomingTxn)}, }) From c647731782b0feb18d76d93dcc4dd5e2f1607717 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 28 Dec 2022 17:37:38 -0500 Subject: [PATCH 153/156] CR: log execpool shutdown event and test for the log message --- data/transactions/verify/txn.go | 4 ++++ data/transactions/verify/txn_test.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index ef3d2d302f..697a63bccf 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -32,6 +32,7 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/execpool" "github.com/algorand/go-algorand/util/metrics" @@ -889,6 +890,9 @@ func (sv *StreamVerifier) addVerificationTaskToThePoolNow(ue []*UnverifiedElemen // EnqueueBacklog returns an error when the context is canceled err := sv.verificationPool.EnqueueBacklog(sv.ctx, function, ue, nil) + if err != nil { + logging.Base().Infof("addVerificationTaskToThePoolNow: EnqueueBacklog returned an error and StreamVerifier will stop: %v", err) + } return err } diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index ff2cf5189a..c664d90f74 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -17,6 +17,7 @@ package verify import ( + "bytes" "context" "encoding/binary" "errors" @@ -36,6 +37,7 @@ import ( "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/execpool" @@ -1201,6 +1203,12 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { numOfTxns := 1 txnGroups, badTxnGroups := getSignedTransactions(numOfTxns, protoMaxGroupSize, 0, 0.5) + // check the logged information + var logBuffer bytes.Buffer + log := logging.Base() + log.SetOutput(&logBuffer) + log.SetLevel(logging.Info) + // prepare the stream verifier numOfTxnGroups := len(txnGroups) verificationPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, t) @@ -1276,6 +1284,7 @@ func TestStreamVerifierPoolShutdown(t *testing.T) { for err := range errChan { require.ErrorIs(t, err, errShuttingDownError) } + require.Contains(t, logBuffer.String(), "addVerificationTaskToThePoolNow: EnqueueBacklog returned an error and StreamVerifier will stop: context canceled") } // TestStreamVerifierRestart tests what happens when the context is canceled @@ -1455,6 +1464,12 @@ func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { verificationPool, holdTasks, vp := getSaturatedExecPool(t) defer vp.Shutdown() + // check the logged information + var logBuffer bytes.Buffer + log := logging.Base() + log.SetOutput(&logBuffer) + log.SetLevel(logging.Info) + ctx, cancel := context.WithCancel(context.Background()) cache := MakeVerifiedTransactionCache(50) stxnChan := make(chan *UnverifiedElement) @@ -1492,6 +1507,7 @@ func TestStreamVerifierCtxCancelPoolQueue(t *testing.T) { wg.Wait() require.ErrorIs(t, result.Err, errShuttingDownError) + require.Contains(t, logBuffer.String(), "addVerificationTaskToThePoolNow: EnqueueBacklog returned an error and StreamVerifier will stop: context canceled") } // TestStreamVerifierPostVBlocked tests the behavior when the return channel (result chan) of verified From 736101075bd38f68b56d9fb313fe361315e1ac59 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 28 Dec 2022 19:52:04 -0500 Subject: [PATCH 154/156] fix rename in tests --- data/txDupCache_test.go | 8 ++++---- data/txHandler_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data/txDupCache_test.go b/data/txDupCache_test.go index e0bfac25a5..42c7659bfb 100644 --- a/data/txDupCache_test.go +++ b/data/txDupCache_test.go @@ -120,7 +120,7 @@ func TestTxHandlerSaltedCacheBasic(t *testing.T) { const size = 20 cache := makeSaltedCache(size) - cache.start(context.Background(), 0) + cache.Start(context.Background(), 0) require.Zero(t, cache.Len()) // add some unique random @@ -204,7 +204,7 @@ func TestTxHandlerSaltedCacheScheduled(t *testing.T) { const size = 20 updateInterval := 1000 * time.Microsecond cache := makeSaltedCache(size) - cache.start(context.Background(), updateInterval) + cache.Start(context.Background(), updateInterval) require.Zero(t, cache.Len()) // add some unique random @@ -229,7 +229,7 @@ func TestTxHandlerSaltedCacheManual(t *testing.T) { const size = 20 cache := makeSaltedCache(2 * size) - cache.start(context.Background(), 0) + cache.Start(context.Background(), 0) require.Zero(t, cache.Len()) // add some unique random @@ -294,7 +294,7 @@ func (m digestCacheMaker) make(size int) cachePusher { } func (m saltedCacheMaker) make(size int) cachePusher { scp := &saltedCachePusher{c: makeSaltedCache(size)} - scp.c.start(context.Background(), 0) + scp.c.Start(context.Background(), 0) return scp } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index 69e6499da0..0339ee2e4a 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -788,7 +788,7 @@ func makeTestTxHandlerOrphanedWithContext(ctx context.Context, backlogSize int, txCanonicalCache: makeDigestCache(cacheSize), cacheConfig: txHandlerConfig, } - handler.msgCache.start(ctx, refreshInterval) + handler.msgCache.Start(ctx, refreshInterval) return handler } From 2d4d8d9281bc69e512850bb95c7484d3cb78c9c8 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Wed, 28 Dec 2022 21:18:39 -0500 Subject: [PATCH 155/156] CR: do not recheck sig types --- data/transactions/verify/txn.go | 66 +++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 697a63bccf..95b78f07a9 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -251,17 +251,27 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl return groupCtx, nil } +type sigOrTxnType int + +const regularSig sigOrTxnType = 1 +const multiSig sigOrTxnType = 2 +const logicSig sigOrTxnType = 3 +const stateProofTxn sigOrTxnType = 4 + // checkTxnSigTypeCounts checks the number of signature types and reports an error in case of a violation -func checkTxnSigTypeCounts(s *transactions.SignedTxn) (isStateProofTxn bool, err *TxGroupError) { +func checkTxnSigTypeCounts(s *transactions.SignedTxn) (sigType sigOrTxnType, err *TxGroupError) { numSigCategories := 0 if s.Sig != (crypto.Signature{}) { numSigCategories++ + sigType = regularSig } if !s.Msig.Blank() { numSigCategories++ + sigType = multiSig } if !s.Lsig.Blank() { numSigCategories++ + sigType = logicSig } if numSigCategories == 0 { // Special case: special sender address can issue special transaction @@ -269,28 +279,28 @@ func checkTxnSigTypeCounts(s *transactions.SignedTxn) (isStateProofTxn bool, err // check ensures that this transaction cannot pay any fee, and // cannot have any other interesting fields, except for the state proof payload. if s.Txn.Sender == transactions.StateProofSender && s.Txn.Type == protocol.StateProofTx { - return true, nil + return stateProofTxn, nil } - return false, &TxGroupError{err: errTxnSigHasNoSig, Reason: TxGroupErrorReasonHasNoSig} + return 0, &TxGroupError{err: errTxnSigHasNoSig, Reason: TxGroupErrorReasonHasNoSig} } if numSigCategories > 1 { - return false, &TxGroupError{err: errTxnSigNotWellFormed, Reason: TxGroupErrorReasonSigNotWellFormed} + return 0, &TxGroupError{err: errTxnSigNotWellFormed, Reason: TxGroupErrorReasonSigNotWellFormed} } - return false, nil + return sigType, nil } // stxnCoreChecks runs signatures validity checks and enqueues signature into batchVerifier for verification. func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContext, batchVerifier *crypto.BatchVerifier) *TxGroupError { - isStateProofTxn, err := checkTxnSigTypeCounts(s) - if err != nil || isStateProofTxn { + sigType, err := checkTxnSigTypeCounts(s) + if err != nil { return err } - if s.Sig != (crypto.Signature{}) { + switch sigType { + case regularSig: batchVerifier.EnqueueSignature(crypto.SignatureVerifier(s.Authorizer()), s.Txn, s.Sig) return nil - } - if !s.Msig.Blank() { + case multiSig: if err := crypto.MultisigBatchPrep(s.Txn, crypto.Digest(s.Authorizer()), s.Msig, batchVerifier); err != nil { return &TxGroupError{err: fmt.Errorf("multisig validation failed: %w", err), Reason: TxGroupErrorReasonMsigNotWellFormed} } @@ -308,14 +318,19 @@ func stxnCoreChecks(s *transactions.SignedTxn, txnIdx int, groupCtx *GroupContex msigMore10.Inc(nil) } return nil - } - if !s.Lsig.Blank() { + + case logicSig: if err := logicSigVerify(s, txnIdx, groupCtx); err != nil { return &TxGroupError{err: err, Reason: TxGroupErrorReasonLogicSigFailed} } return nil + + case stateProofTxn: + return nil + + default: + return &TxGroupError{err: errUnknownSignature, Reason: TxGroupErrorReasonGeneric} } - return &TxGroupError{err: errUnknownSignature, Reason: TxGroupErrorReasonGeneric} } // LogicSigSanityCheck checks that the signature is valid and that the program is basically well formed. @@ -909,17 +924,14 @@ func getNumberOfBatchableSigsInGroup(stxs []transactions.SignedTxn) (batchSigs u } func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) { - isStateProofTxn, err := checkTxnSigTypeCounts(stx) - if err != nil || isStateProofTxn { - if err != nil { - return 0, err - } - return 0, nil + sigType, err := checkTxnSigTypeCounts(stx) + if err != nil { + return 0, err } - if stx.Sig != (crypto.Signature{}) { + switch sigType { + case regularSig: return 1, nil - } - if !stx.Msig.Blank() { + case multiSig: sig := stx.Msig batchSigs := uint64(0) for _, subsigi := range sig.Subsigs { @@ -928,7 +940,13 @@ func getNumberOfBatchableSigsInTxn(stx *transactions.SignedTxn) (uint64, error) } } return batchSigs, nil + case logicSig: + // Currently the sigs in here are not batched. Something to consider later. + return 0, nil + case stateProofTxn: + return 0, nil + default: + // this case is impossible + return 0, nil } - // This is the Lsig case. Currently the sigs in here are not batched. Something to consider later. - return 0, nil } From bab964baf2bd5f5a20f364134a8dd5b7abf6546f Mon Sep 17 00:00:00 2001 From: algonautshant Date: Thu, 29 Dec 2022 11:56:20 -0500 Subject: [PATCH 156/156] CR: remove unused err types and fix the test checking for the errors --- data/transactions/verify/txn.go | 2 -- data/transactions/verify/txn_test.go | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/data/transactions/verify/txn.go b/data/transactions/verify/txn.go index 95b78f07a9..ae06b5ddc5 100644 --- a/data/transactions/verify/txn.go +++ b/data/transactions/verify/txn.go @@ -49,8 +49,6 @@ var msigLsigLessOrEqual4 = metrics.MakeCounter(metrics.MetricName{Name: "algod_v var msigLsigLessOrEqual10 = metrics.MakeCounter(metrics.MetricName{Name: "algod_verify_msig_lsig_5_10", Description: "Total transaction scripts with 5-10 msigs"}) var msigLsigMore10 = metrics.MakeCounter(metrics.MetricName{Name: "algod_verify_msig_lsig_10", Description: "Total transaction scripts with 11+ msigs"}) -var errSignedTxnHasNoSig = errors.New("signedtxn has no sig") -var errSignedTxnMaxOneSig = errors.New("signedtxn should only have one of Sig or Msig or LogicSig") var errShuttingDownError = errors.New("not verified, verifier is shutting down") // The PaysetGroups is taking large set of transaction groups and attempt to verify their validity using multiple go-routines. diff --git a/data/transactions/verify/txn_test.go b/data/transactions/verify/txn_test.go index c664d90f74..cf3e0e037b 100644 --- a/data/transactions/verify/txn_test.go +++ b/data/transactions/verify/txn_test.go @@ -1029,7 +1029,7 @@ func TestStreamVerifierCases(t *testing.T) { txnGroups[mod][0].Sig = crypto.Signature{} u, _ := binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - sv := streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnHasNoSig, t) + sv := streamVerifierTestCore(txnGroups, badTxnGroups, errTxnSigHasNoSig, t) sv.WaitForStop() mod++ @@ -1115,7 +1115,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups[mod][0].Msig = mSigTxn[0].Msig u, _ = binary.Uvarint(txnGroups[mod][0].Txn.Note) badTxnGroups[u] = struct{}{} - sv = streamVerifierTestCore(txnGroups, badTxnGroups, errSignedTxnMaxOneSig, t) + sv = streamVerifierTestCore(txnGroups, badTxnGroups, errTxnSigNotWellFormed, t) sv.WaitForStop() } @@ -1140,7 +1140,7 @@ func TestGetNumberOfBatchableSigsInGroup(t *testing.T) { // txn with 0 sigs txnGroups[mod][0].Sig = crypto.Signature{} batchSigs, err := getNumberOfBatchableSigsInGroup(txnGroups[mod]) - require.Error(t, err, errSignedTxnHasNoSig) + require.ErrorIs(t, err, errTxnSigHasNoSig) mod++ _, signedTxns, secrets, addrs := generateTestObjects(numOfTxns, 20, 0, 50) @@ -1191,7 +1191,7 @@ byte base64 5rZMNsevs5sULO+54aN+OvU6lQ503z2X+SSYUABIx7E= txnGroups = generateTransactionGroups(1, signedTxn, secrets, addrs) txnGroups[mod][0].Msig = mSigTxn[0].Msig batchSigs, err = getNumberOfBatchableSigsInGroup(txnGroups[mod]) - require.Error(t, err, errSignedTxnMaxOneSig) + require.ErrorIs(t, err, errTxnSigNotWellFormed) } // TestStreamVerifierPoolShutdown tests what happens when the exec pool shuts down