Convenient and High-fidelity Transaction Processing Tests (#2730)

* Convenient and High-fidelity Transaction Processing Tests

This refactors some of the `ledger` code so that it's convenient
to write short tests that operate on a fairly complete ledger,
including rewards payouts.  Previously, tests used genesis(), but
genesis() build the genesis block internally "by hand" rather
than using MakeGenesisBlock, so it missed some details (like
setting up RewardsState). Presumably, this was because
MakeGenesisBlock was in the `data` package, and could not be
imported.  That is the motivation behind moving it, and some
related code, to bookkeeping (where various Genesis related code
already existed).

The txntest packaged is motivated purely by a desire for more
concise tests.  It allows for the construction of
transaction.Transaction objects concisely, and we can add all
sort of conveneince routines here that would not make sense in
the production code (turning these into SignedTxns,
SignedTxnWithADs, TransactionGroups, etc).

* Converted a test, went from 130 lines to 68.  Same test.

* Docs and and another test converted

catchup/fetcher_test.go

@@ -60,7 +60,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger,
 	}
 
 	log := logging.TestingLog(t)
-	genBal := data.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
+	genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
 	genHash := crypto.Digest{0x42}
 	const inMem = true
 	cfg := config.GetDefaultLocal()

catchup/pref_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/algorand/go-algorand/data"
 	"github.com/algorand/go-algorand/data/account"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/datatest"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
@@ -81,7 +82,7 @@ func BenchmarkServiceFetchBlocks(b *testing.B) {
 }
 
 // one service
-func benchenv(t testing.TB, numAccounts, numBlocks int) (ledger, emptyLedger *data.Ledger, release func(), genesisBalances data.GenesisBalances) {
+func benchenv(t testing.TB, numAccounts, numBlocks int) (ledger, emptyLedger *data.Ledger, release func(), genesisBalances bookkeeping.GenesisBalances) {
 	P := numAccounts                                  // n accounts
 	maxMoneyAtStart := uint64(10 * defaultRewardUnit) // max money start
 	minMoneyAtStart := uint64(defaultRewardUnit)      // min money start
@@ -143,7 +144,7 @@ func benchenv(t testing.TB, numAccounts, numBlocks int) (ledger, emptyLedger *da
 	}
 
 	var err error
-	genesisBalances = data.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
+	genesisBalances = bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
 	const inMem = true
 	cfg := config.GetDefaultLocal()
 	cfg.Archival = true

daemon/algod/api/server/v2/test/helpers.go

@@ -266,7 +266,7 @@ func testingenv(t testing.TB, numAccounts, numTxs int, offlineAccounts bool) (*d
 	ad.AppLocalStates = map[basics.AppIndex]basics.AppLocalState{1: {}}
 	genesis[addr] = ad
 
-	bootstrap := data.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
+	bootstrap := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
 
 	// generate test transactions
 	const inMem = true

data/bookkeeping/genesis.go

@@ -17,9 +17,15 @@
 package bookkeeping
 
 import (
+	"fmt"
 	"io/ioutil"
+	"time"
 
+	"github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/committee"
+	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/protocol"
 )
 
@@ -115,3 +121,70 @@ type GenesisAllocation struct {
 func (genesis Genesis) ToBeHashed() (protocol.HashID, []byte) {
 	return protocol.Genesis, protocol.Encode(&genesis)
 }
+
+// GenesisBalances contains the information needed to generate a new ledger
+type GenesisBalances struct {
+	Balances    map[basics.Address]basics.AccountData
+	FeeSink     basics.Address
+	RewardsPool basics.Address
+	Timestamp   int64
+}
+
+// MakeGenesisBalances returns the information needed to bootstrap the ledger based on the current time
+func MakeGenesisBalances(balances map[basics.Address]basics.AccountData, feeSink, rewardsPool basics.Address) GenesisBalances {
+	return MakeTimestampedGenesisBalances(balances, feeSink, rewardsPool, time.Now().Unix())
+}
+
+// MakeTimestampedGenesisBalances returns the information needed to bootstrap the ledger based on a given time
+func MakeTimestampedGenesisBalances(balances map[basics.Address]basics.AccountData, feeSink, rewardsPool basics.Address, timestamp int64) GenesisBalances {
+	return GenesisBalances{Balances: balances, FeeSink: feeSink, RewardsPool: rewardsPool, Timestamp: timestamp}
+}
+
+// MakeGenesisBlock creates a genesis block, including setup of RewardsState.
+func MakeGenesisBlock(proto protocol.ConsensusVersion, genesisBal GenesisBalances, genesisID string, genesisHash crypto.Digest) (Block, error) {
+	params, ok := config.Consensus[proto]
+	if !ok {
+		return Block{}, fmt.Errorf("unsupported protocol %s", proto)
+	}
+
+	poolAddr := basics.Address(genesisBal.RewardsPool)
+	incentivePoolBalanceAtGenesis := genesisBal.Balances[poolAddr].MicroAlgos
+
+	genesisRewardsState := RewardsState{
+		FeeSink:                   genesisBal.FeeSink,
+		RewardsPool:               genesisBal.RewardsPool,
+		RewardsLevel:              0,
+		RewardsResidue:            0,
+		RewardsRecalculationRound: basics.Round(params.RewardsRateRefreshInterval),
+	}
+
+	if params.InitialRewardsRateCalculation {
+		genesisRewardsState.RewardsRate = basics.SubSaturate(incentivePoolBalanceAtGenesis.Raw, params.MinBalance) / uint64(params.RewardsRateRefreshInterval)
+	} else {
+		genesisRewardsState.RewardsRate = incentivePoolBalanceAtGenesis.Raw / uint64(params.RewardsRateRefreshInterval)
+	}
+
+	genesisProtoState := UpgradeState{
+		CurrentProtocol: proto,
+	}
+
+	blk := Block{
+		BlockHeader: BlockHeader{
+			Round:        0,
+			Branch:       BlockHash{},
+			Seed:         committee.Seed(genesisHash),
+			TxnRoot:      transactions.Payset{}.CommitGenesis(),
+			TimeStamp:    genesisBal.Timestamp,
+			GenesisID:    genesisID,
+			RewardsState: genesisRewardsState,
+			UpgradeState: genesisProtoState,
+			UpgradeVote:  UpgradeVote{},
+		},
+	}
+
+	if params.SupportGenesisHash {
+		blk.BlockHeader.GenesisHash = genesisHash
+	}
+
+	return blk, nil
+}

data/common_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/account"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
@@ -113,7 +114,7 @@ func testingenv(t testing.TB, numAccounts, numTxs int, offlineAccounts bool) (*L
 
 	genesis[poolAddr] = basics.MakeAccountData(basics.NotParticipating, basics.MicroAlgos{Raw: 100000 * uint64(proto.RewardsRateRefreshInterval)})
 
-	bootstrap := MakeGenesisBalances(genesis, poolAddr, sinkAddr)
+	bootstrap := bookkeeping.MakeGenesisBalances(genesis, poolAddr, sinkAddr)
 
 	// generate test transactions
 	const inMem = true
@@ -129,7 +130,7 @@ func testingenv(t testing.TB, numAccounts, numTxs int, offlineAccounts bool) (*L
 	if latest != 0 {
 		panic(fmt.Errorf("newly created ledger doesn't start on round 0"))
 	}
-	bal := bootstrap.balances
+	bal := bootstrap.Balances
 
 	for i := 0; i < TXs; i++ {
 		send := gen.Int() % P

data/datatest/fabricateLedger.go

@@ -25,6 +25,7 @@ import (
 	"github.com/algorand/go-algorand/data"
 	"github.com/algorand/go-algorand/data/account"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
 )
@@ -33,7 +34,7 @@ import (
 var roundDeadline = 0 * time.Second
 
 // FabricateLedger is a test-only helper to create a new in-memory Ledger and run the protocol through the specified Round with the given accounts
-func FabricateLedger(log logging.Logger, ledgerName string, accounts []account.Participation, genesis data.GenesisBalances, lastRound basics.Round) (*data.Ledger, error) {
+func FabricateLedger(log logging.Logger, ledgerName string, accounts []account.Participation, genesis bookkeeping.GenesisBalances, lastRound basics.Round) (*data.Ledger, error) {
 	const inMem = true
 	cfg := config.GetDefaultLocal()
 	cfg.Archival = true

data/ledger.go

@@ -17,7 +17,6 @@
 package data
 
 import (
-	"fmt"
 	"sync/atomic"
 	"time"
 
@@ -76,83 +75,35 @@ type roundSeed struct {
 	elements [2]roundSeedPair
 }
 
-func makeGenesisBlock(proto protocol.ConsensusVersion, genesisBal GenesisBalances, genesisID string, genesisHash crypto.Digest) (bookkeeping.Block, error) {
-	params, ok := config.Consensus[proto]
-	if !ok {
-		return bookkeeping.Block{}, fmt.Errorf("unsupported protocol %s", proto)
-	}
-
-	poolAddr := basics.Address(genesisBal.rewardsPool)
-	incentivePoolBalanceAtGenesis := genesisBal.balances[poolAddr].MicroAlgos
-
-	genesisRewardsState := bookkeeping.RewardsState{
-		FeeSink:                   genesisBal.feeSink,
-		RewardsPool:               genesisBal.rewardsPool,
-		RewardsLevel:              0,
-		RewardsResidue:            0,
-		RewardsRecalculationRound: basics.Round(params.RewardsRateRefreshInterval),
-	}
-
-	if params.InitialRewardsRateCalculation {
-		genesisRewardsState.RewardsRate = basics.SubSaturate(incentivePoolBalanceAtGenesis.Raw, params.MinBalance) / uint64(params.RewardsRateRefreshInterval)
-	} else {
-		genesisRewardsState.RewardsRate = incentivePoolBalanceAtGenesis.Raw / uint64(params.RewardsRateRefreshInterval)
-	}
-
-	genesisProtoState := bookkeeping.UpgradeState{
-		CurrentProtocol: proto,
-	}
-
-	blk := bookkeeping.Block{
-		BlockHeader: bookkeeping.BlockHeader{
-			Round:        0,
-			Branch:       bookkeeping.BlockHash{},
-			Seed:         committee.Seed(genesisHash),
-			TxnRoot:      transactions.Payset{}.CommitGenesis(),
-			TimeStamp:    genesisBal.timestamp,
-			GenesisID:    genesisID,
-			RewardsState: genesisRewardsState,
-			UpgradeState: genesisProtoState,
-			UpgradeVote:  bookkeeping.UpgradeVote{},
-		},
-	}
-
-	if params.SupportGenesisHash {
-		blk.BlockHeader.GenesisHash = genesisHash
-	}
-
-	return blk, nil
-}
-
 // LoadLedger creates a Ledger object to represent the ledger with the
 // specified database file prefix, initializing it if necessary.
 func LoadLedger(
 	log logging.Logger, dbFilenamePrefix string, memory bool,
-	genesisProto protocol.ConsensusVersion, genesisBal GenesisBalances, genesisID string, genesisHash crypto.Digest,
+	genesisProto protocol.ConsensusVersion, genesisBal bookkeeping.GenesisBalances, genesisID string, genesisHash crypto.Digest,
 	blockListeners []ledger.BlockListener, cfg config.Local,
 ) (*Ledger, error) {
-	if genesisBal.balances == nil {
-		genesisBal.balances = make(map[basics.Address]basics.AccountData)
+	if genesisBal.Balances == nil {
+		genesisBal.Balances = make(map[basics.Address]basics.AccountData)
 	}
-	genBlock, err := makeGenesisBlock(genesisProto, genesisBal, genesisID, genesisHash)
+	genBlock, err := bookkeeping.MakeGenesisBlock(genesisProto, genesisBal, genesisID, genesisHash)
 	if err != nil {
 		return nil, err
 	}
 
 	params := config.Consensus[genesisProto]
 	if params.ForceNonParticipatingFeeSink {
-		sinkAddr := genesisBal.feeSink
-		sinkData := genesisBal.balances[sinkAddr]
+		sinkAddr := genesisBal.FeeSink
+		sinkData := genesisBal.Balances[sinkAddr]
 		sinkData.Status = basics.NotParticipating
-		genesisBal.balances[sinkAddr] = sinkData
+		genesisBal.Balances[sinkAddr] = sinkData
 	}
 
 	l := &Ledger{
 		log: log,
 	}
 	genesisInitState := ledger.InitState{
 		Block:       genBlock,
-		Accounts:    genesisBal.balances,
+		Accounts:    genesisBal.Balances,
 		GenesisHash: genesisHash,
 	}
 	l.log.Debugf("Initializing Ledger(%s)", dbFilenamePrefix)

data/txHandler_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/pools"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/logging"
@@ -62,7 +63,7 @@ func BenchmarkTxHandlerProcessDecoded(b *testing.B) {
 	}
 
 	require.Equal(b, len(genesis), numUsers+1)
-	genBal := MakeGenesisBalances(genesis, sinkAddr, poolAddr)
+	genBal := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr)
 	ledgerName := fmt.Sprintf("%s-mem-%d", b.Name(), b.N)
 	const inMem = true
 	cfg := config.GetDefaultLocal()