Local Ledger (#1011)

* integrate block processor
algorand · Jun 8, 2022 · 12f8c2f · 12f8c2f
1 parent f26fca1
commit 12f8c2f
Show file tree

Hide file tree

Showing 21 changed files with 1,370 additions and 361 deletions.
diff --git a/README.md b/README.md
@@ -195,6 +195,7 @@ Settings can be provided from the command line, a configuration file, or an envi
 | default-balances-limit        |         | default-balances-limit        | INDEXER_DEFAULT_BALANCES_LIMIT        |
 | max-applications-limit        |         | max-applications-limit        | INDEXER_MAX_APPLICATIONS_LIMIT        |
 | default-applications-limit    |         | default-applications-limit    | INDEXER_DEFAULT_APPLICATIONS_LIMIT    |
+| data-dir                      | i       | data                          | INDEXER_DATA                          |
 
 ## Command line
 

diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go
@@ -13,15 +13,19 @@ import (
 	"time"
 
 	"github.com/algorand/go-algorand/crypto"
+	"github.com/algorand/go-algorand/crypto/merklesignature"
+	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/rpcs"
+	"github.com/algorand/indexer/processor"
+	"github.com/algorand/indexer/processor/blockprocessor"
 	"github.com/labstack/echo/v4"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/go-algorand-sdk/encoding/json"
-	"github.com/algorand/go-algorand/crypto/merklesignature"
-	"github.com/algorand/go-algorand/data/basics"
-	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/data/transactions"
 
 	"github.com/algorand/indexer/api/generated/v2"
@@ -56,7 +60,7 @@ func testServerImplementation(db idb.IndexerDb) *ServerImplementation {
 	return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts}
 }
 
-func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*postgres.IndexerDb /*db*/, func() /*shutdownFunc*/) {
+func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), processor.Processor, *ledger.Ledger) {
 	_, connStr, shutdownFunc := pgtest.SetupPostgres(t)
 
 	db, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil)
@@ -70,15 +74,16 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin
 	err = db.LoadGenesis(genesis)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&genesisBlock)
-	require.NoError(t, err)
-
-	return db, newShutdownFunc
+	l := test.MakeTestLedger("ledger")
+	proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock)
+	require.NoError(t, err, "failed to open ledger")
+	return db, newShutdownFunc, proc, l
 }
 
 func TestApplicationHandlers(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // A block containing an app call txn with ExtraProgramPages, that the creator and another account have opted into
@@ -112,8 +117,8 @@ func TestApplicationHandlers(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	//////////
 	// When // We query the app
@@ -218,8 +223,9 @@ func TestApplicationHandlers(t *testing.T) {
 }
 
 func TestAccountExcludeParameters(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // A block containing a creator of an app, an asset, who also holds and has opted-into those apps.
@@ -238,8 +244,8 @@ func TestAccountExcludeParameters(t *testing.T) {
 		&appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	//////////
 	// When // We look up the address using various exclude parameters.
@@ -389,8 +395,9 @@ type accountsErrorResponse struct {
 }
 
 func TestAccountMaxResultsLimit(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // A block containing an address that has created 10 apps, deleted 5 apps, and created 10 assets,
@@ -443,8 +450,8 @@ func TestAccountMaxResultsLimit(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	//////////
 	// When // We look up the address using a ServerImplementation with a maxAccountsAPIResults limit set,
@@ -768,8 +775,9 @@ func TestAccountMaxResultsLimit(t *testing.T) {
 }
 
 func TestBlockNotFound(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // An empty database.
@@ -833,8 +841,9 @@ func TestInnerTxn(t *testing.T) {
 		},
 	}
 
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // a DB with some inner txns in it.
@@ -845,8 +854,8 @@ func TestInnerTxn(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -881,8 +890,9 @@ func TestInnerTxn(t *testing.T) {
 // transaction group does not allow the root transaction to be returned on both
 // pages.
 func TestPagingRootTxnDeduplication(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // a DB with some inner txns in it.
@@ -897,8 +907,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	testcases := []struct {
 		name   string
@@ -1004,8 +1014,9 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
 }
 
 func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // A block containing a key reg txn with state proof key
@@ -1044,8 +1055,8 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	e := echo.New()
 	{
@@ -1098,8 +1109,9 @@ func TestVersion(t *testing.T) {
 	///////////
 	// Given // An API and context
 	///////////
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 	api := testServerImplementation(db)
 
 	e := echo.New()
@@ -1127,8 +1139,9 @@ func TestVersion(t *testing.T) {
 }
 
 func TestAccountClearsNonUTF8(t *testing.T) {
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // a DB with some inner txns in it.
@@ -1147,8 +1160,8 @@ func TestAccountClearsNonUTF8(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	verify := func(params generated.AssetParams) {
 		compareB64 := func(expected string, actual *[]byte) {
@@ -1257,8 +1270,9 @@ func TestLookupInnerLogs(t *testing.T) {
 		},
 	}
 
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // a DB with some inner txns in it.
@@ -1268,8 +1282,8 @@ func TestLookupInnerLogs(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1355,8 +1369,9 @@ func TestLookupMultiInnerLogs(t *testing.T) {
 		},
 	}
 
-	db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
+	db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()
+	defer l.Close()
 
 	///////////
 	// Given // a DB with some inner txns in it.
@@ -1366,8 +1381,8 @@ func TestLookupMultiInnerLogs(t *testing.T) {
 	block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
 	require.NoError(t, err)
 
-	err = db.AddBlock(&block)
-	require.NoError(t, err, "failed to commit")
+	err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
+	require.NoError(t, err)
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {