optimize txtail memory consumption (algorand#2413)

The changes in this PR are as follows: The unused method Ledger. GetRoundTxIds was removed. As a result, the txTail. getRoundTxIds can be removed as well. This makes the txids map stored in the roundTxMembers structure redundant. In the cow.go, avoid adding empty leases to the cb.mods.Txleases map. Since we already not testing for empty leases, we can safely avoid storing them. Optimize the txTail. loadFromDisk to generate optimal lastValid map sizes. Optimize the txTail. loadFromDisk to avoid storing empty leases in the txleases map.
tsachiherman · Jul 6, 2021 · c4206f7 · c4206f7
1 parent 2169e05
commit c4206f7
Show file tree

Hide file tree

Showing 4 changed files with 133 additions and 41 deletions.
diff --git a/ledger/cow.go b/ledger/cow.go
@@ -220,7 +220,9 @@ func (cb *roundCowState) trackCreatable(creatableIndex basics.CreatableIndex) {
 
 func (cb *roundCowState) addTx(txn transactions.Transaction, txid transactions.Txid) {
 	cb.mods.Txids[txid] = txn.LastValid
-	cb.mods.Txleases[ledgercore.Txlease{Sender: txn.Sender, Lease: txn.Lease}] = txn.LastValid
+	if txn.Lease != [32]byte{} {
+		cb.mods.Txleases[ledgercore.Txlease{Sender: txn.Sender, Lease: txn.Lease}] = txn.LastValid
+	}
 }
 
 func (cb *roundCowState) setCompactCertNext(rnd basics.Round) {

diff --git a/ledger/ledger.go b/ledger/ledger.go
@@ -477,14 +477,6 @@ func (l *Ledger) CheckDup(currentProto config.ConsensusParams, current basics.Ro
 	return l.txTail.checkDup(currentProto, current, firstValid, lastValid, txid, txl.Txlease)
 }
 
-// GetRoundTxIds returns a map of the transactions ids that we have for the given round
-// this function is currently not being used, but remains here as it might be useful in the future.
-func (l *Ledger) GetRoundTxIds(rnd basics.Round) (txMap map[transactions.Txid]bool) {
-	l.trackerMu.RLock()
-	defer l.trackerMu.RUnlock()
-	return l.txTail.getRoundTxIds(rnd)
-}
-
 // Latest returns the latest known block round added to the ledger.
 func (l *Ledger) Latest() basics.Round {
 	return l.blockQ.latest()

diff --git a/ledger/txtail.go b/ledger/txtail.go
@@ -26,8 +26,9 @@ import (
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 )
 
+const initialLastValidArrayLen = 256
+
 type roundTxMembers struct {
-	txids    map[transactions.Txid]basics.Round
 	txleases map[ledgercore.Txlease]basics.Round // map of transaction lease to when it expires
 	proto    config.ConsensusParams
 }
@@ -60,6 +61,13 @@ func (t *txTail) loadFromDisk(l ledgerForTracker) error {
 	t.lastValid = make(map[basics.Round]map[transactions.Txid]struct{})
 
 	t.recent = make(map[basics.Round]roundTxMembers)
+
+	// the roundsLastValids is a temporary map used during the exection of
+	// loadFromDisk, allowing us to construct the lastValid maps in their
+	// optimal size. This would ensure that upon startup, we don't preallocate
+	// more memory than we truely need.
+	roundsLastValids := make(map[basics.Round][]transactions.Txid)
+
 	for ; old <= latest; old++ {
 		blk, err := l.Block(old)
 		if err != nil {
@@ -71,19 +79,44 @@ func (t *txTail) loadFromDisk(l ledgerForTracker) error {
 			return err
 		}
 
+		consensusParams := config.Consensus[blk.CurrentProtocol]
 		t.recent[old] = roundTxMembers{
-			txids:    make(map[transactions.Txid]basics.Round),
-			txleases: make(map[ledgercore.Txlease]basics.Round),
-			proto:    config.Consensus[blk.CurrentProtocol],
+			txleases: make(map[ledgercore.Txlease]basics.Round, len(payset)),
+			proto:    consensusParams,
 		}
+
 		for _, txad := range payset {
 			tx := txad.SignedTxn
-			t.recent[old].txids[tx.ID()] = tx.Txn.LastValid
-			t.recent[old].txleases[ledgercore.Txlease{Sender: tx.Txn.Sender, Lease: tx.Txn.Lease}] = tx.Txn.LastValid
-			t.putLV(tx.Txn.LastValid, tx.ID())
+			if consensusParams.SupportTransactionLeases && (tx.Txn.Lease != [32]byte{}) {
+				t.recent[old].txleases[ledgercore.Txlease{Sender: tx.Txn.Sender, Lease: tx.Txn.Lease}] = tx.Txn.LastValid
+			}
+			if tx.Txn.LastValid > t.lowWaterMark {
+				list := roundsLastValids[tx.Txn.LastValid]
+				// if the list reached capacity, resize.
+				if len(list) == cap(list) {
+					var newList []transactions.Txid
+					if cap(list) == 0 {
+						newList = make([]transactions.Txid, 0, initialLastValidArrayLen)
+					} else {
+						newList = make([]transactions.Txid, len(list), len(list)*2)
+					}
+					copy(newList[:], list[:])
+					list = newList
+				}
+				list = append(list, tx.ID())
+				roundsLastValids[tx.Txn.LastValid] = list
+			}
 		}
 	}
 
+	// add all the entries in roundsLastValids to their corresponding map entry in t.lastValid
+	for lastValid, list := range roundsLastValids {
+		lastValueMap := make(map[transactions.Txid]struct{}, len(list))
+		for _, id := range list {
+			lastValueMap[id] = struct{}{}
+		}
+		t.lastValid[lastValid] = lastValueMap
+	}
 	return nil
 }
 
@@ -93,13 +126,12 @@ func (t *txTail) close() {
 func (t *txTail) newBlock(blk bookkeeping.Block, delta ledgercore.StateDelta) {
 	rnd := blk.Round()
 
-	if t.recent[rnd].txids != nil {
+	if _, has := t.recent[rnd]; has {
 		// Repeat, ignore
 		return
 	}
 
 	t.recent[rnd] = roundTxMembers{
-		txids:    delta.Txids,
 		txleases: delta.Txleases,
 		proto:    config.Consensus[blk.CurrentProtocol],
 	}
@@ -162,15 +194,6 @@ func (t *txTail) checkDup(proto config.ConsensusParams, current basics.Round, fi
 	return nil
 }
 
-func (t *txTail) getRoundTxIds(rnd basics.Round) (txMap map[transactions.Txid]bool) {
-	rndtxs := t.recent[rnd].txids
-	txMap = make(map[transactions.Txid]bool, len(rndtxs))
-	for txid := range rndtxs {
-		txMap[txid] = true
-	}
-	return
-}
-
 func (t *txTail) putLV(lastValid basics.Round, id transactions.Txid) {
 	if _, ok := t.lastValid[lastValid]; !ok {
 		t.lastValid[lastValid] = make(map[transactions.Txid]struct{})

diff --git a/ledger/txtail_test.go b/ledger/txtail_test.go
@@ -94,22 +94,97 @@ func TestTxTailCheckdup(t *testing.T) {
 	}
 }
 
-func TestGetRoundTxIds(t *testing.T) {
-	r := basics.Round(100)
-	txids := make(map[transactions.Txid]basics.Round)
-	txids[transactions.Txid(crypto.Hash([]byte("a")))] = r
-	txids[transactions.Txid(crypto.Hash([]byte("b")))] = r
-	recent := make(map[basics.Round]roundTxMembers)
-	recent[r] = roundTxMembers{
-		txids: txids,
+type txTailTestLedger struct {
+	Ledger
+}
+
+const testTxTailValidityRange = 200
+const testTxTailTxnPerRound = 150
+
+func (t *txTailTestLedger) Latest() basics.Round {
+	return basics.Round(config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnLife + 10)
+}
+
+func (t *txTailTestLedger) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) {
+	return bookkeeping.BlockHeader{
+		UpgradeState: bookkeeping.UpgradeState{
+			CurrentProtocol: protocol.ConsensusCurrentVersion,
+		},
+	}, nil
+}
+
+func (t *txTailTestLedger) Block(r basics.Round) (bookkeeping.Block, error) {
+	blk := bookkeeping.Block{
+		BlockHeader: bookkeeping.BlockHeader{
+			UpgradeState: bookkeeping.UpgradeState{
+				CurrentProtocol: protocol.ConsensusCurrentVersion,
+			},
+			Round: r,
+		},
+		Payset: make(transactions.Payset, testTxTailTxnPerRound),
+	}
+	for i := range blk.Payset {
+		blk.Payset[i] = makeTxTailTestTransaction(r, i)
 	}
 
-	tt := txTail{
-		recent: recent,
+	return blk, nil
+}
+func makeTxTailTestTransaction(r basics.Round, txnIdx int) (txn transactions.SignedTxnInBlock) {
+	txn.Txn.FirstValid = r
+	txn.Txn.LastValid = r + testTxTailValidityRange
+	if txnIdx%5 == 0 {
+		digest := crypto.Hash([]byte{byte(r), byte(r >> 8), byte(r >> 16), byte(r >> 24), byte(r >> 32), byte(r >> 40), byte(r >> 48), byte(r >> 56)})
+		copy(txn.Txn.Lease[:], digest[:])
 	}
+	// use 7 different senders.
+	sender := uint64((int(r) + txnIdx) % 7)
+	senderDigest := crypto.Hash([]byte{byte(sender), byte(sender >> 8), byte(sender >> 16), byte(sender >> 24), byte(sender >> 32), byte(sender >> 40), byte(sender >> 48), byte(sender >> 56)})
+	copy(txn.Txn.Sender[:], senderDigest[:])
+	return txn
+}
+
+func TestTxTailLoadFromDisk(t *testing.T) {
+	var ledger txTailTestLedger
+	txtail := txTail{}
 
-	txMap := tt.getRoundTxIds(100)
-	require.Equal(t, 2, len(txMap))
-	require.True(t, txMap[transactions.Txid(crypto.Hash([]byte("a")))])
-	require.True(t, txMap[transactions.Txid(crypto.Hash([]byte("b")))])
+	err := txtail.loadFromDisk(&ledger)
+	require.NoError(t, err)
+	require.Equal(t, int(config.Consensus[protocol.ConsensusCurrentVersion].MaxTxnLife), len(txtail.recent))
+	require.Equal(t, testTxTailValidityRange, len(txtail.lastValid))
+	require.Equal(t, ledger.Latest(), txtail.lowWaterMark)
+
+	// do some fuzz testing for leases -
+	for i := 0; i < 5000; i++ {
+		r := basics.Round(crypto.RandUint64() % uint64(ledger.Latest()))
+		txIdx := int(crypto.RandUint64() % uint64(len(txtail.recent)))
+		txn := makeTxTailTestTransaction(r, txIdx)
+		if txn.Txn.Lease != [32]byte{} {
+			// transaction has a lease
+			txl := ledgercore.Txlease{Sender: txn.Txn.Sender, Lease: txn.Txn.Lease}
+			dupResult := txtail.checkDup(
+				config.Consensus[protocol.ConsensusCurrentVersion], ledger.Latest(),
+				txn.Txn.FirstValid, txn.Txn.LastValid, txn.Txn.ID(),
+				txl)
+			if r >= ledger.Latest()-testTxTailValidityRange {
+				require.Equal(t, ledgercore.MakeLeaseInLedgerError(txn.Txn.ID(), txl), dupResult)
+			} else {
+				require.Equal(t, &txtailMissingRound{round: txn.Txn.LastValid}, dupResult)
+			}
+		} else {
+			// transaction has no lease
+			dupResult := txtail.checkDup(
+				config.Consensus[protocol.ConsensusCurrentVersion], ledger.Latest(),
+				txn.Txn.FirstValid, txn.Txn.LastValid, txn.Txn.ID(),
+				ledgercore.Txlease{})
+			if r >= ledger.Latest()-testTxTailValidityRange {
+				if txn.Txn.LastValid > ledger.Latest() {
+					require.Equal(t, &ledgercore.TransactionInLedgerError{Txid: txn.Txn.ID()}, dupResult)
+				} else {
+					require.Nil(t, dupResult)
+				}
+			} else {
+				require.Equal(t, &txtailMissingRound{round: txn.Txn.LastValid}, dupResult)
+			}
+		}
+	}
 }