Improve fresh node startup time (algorand#2185)

## Summary

This PR contains three different changes that would allow the node to start up faster:
1. It disables the block 0 fix we previously deployed. This fix was intended to address catching up archival nodes, where we would calculate the block 0 hash incorrectly. The fix was long deployed, and the hash calculation is also fixed since.
2. The tracker and blocks databases are now being opened concurrently.
3. The accounts tracker database schema upgrade would now skip some of the steps that aren't required for a fresh database.

Combining all the above, the ledger startup of a new node is now about 2-3 times faster.

## Test Plan

This change has no functional enduser change. The existing tests provide sufficient coverage.

cmd/algod/main.go

@@ -58,7 +58,11 @@ var seed = flag.String("seed", "", "input to math/rand.Seed()")
 
 func main() {
 	flag.Parse()
+	exitCode := run()
+	os.Exit(exitCode)
+}
 
+func run() int {
 	dataDir := resolveDataDir()
 	absolutePath, absPathErr := filepath.Abs(dataDir)
 	config.UpdateVersionDataDir(absolutePath)
@@ -67,8 +71,7 @@ func main() {
 		seedVal, err := strconv.ParseInt(*seed, 10, 64)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "bad seed %#v: %s\n", *seed, err)
-			os.Exit(1)
-			return
+			return 1
 		}
 		rand.Seed(seedVal)
 	} else {
@@ -77,7 +80,7 @@ func main() {
 
 	if *versionCheck {
 		fmt.Println(config.FormatVersionAndLicense())
-		return
+		return 0
 	}
 
 	version := config.GetCurrentVersion()
@@ -90,52 +93,53 @@ func main() {
 
 	if *branchCheck {
 		fmt.Println(config.Branch)
-		return
+		return 0
 	}
 
 	if *channelCheck {
 		fmt.Println(config.Channel)
-		return
+		return 0
 	}
 
 	// Don't fallback anymore - if not specified, we want to panic to force us to update our tooling and/or processes
 	if len(dataDir) == 0 {
 		fmt.Fprintln(os.Stderr, "Data directory not specified.  Please use -d or set $ALGORAND_DATA in your environment.")
-		os.Exit(1)
+		return 1
 	}
 
 	if absPathErr != nil {
 		fmt.Fprintf(os.Stderr, "Can't convert data directory's path to absolute, %v\n", dataDir)
-		os.Exit(1)
+		return 1
 	}
 
-	if *genesisFile == "" {
-		*genesisFile = filepath.Join(dataDir, config.GenesisJSONFile)
+	genesisPath := *genesisFile
+	if genesisPath == "" {
+		genesisPath = filepath.Join(dataDir, config.GenesisJSONFile)
 	}
 
 	// Load genesis
-	genesisText, err := ioutil.ReadFile(*genesisFile)
+	genesisText, err := ioutil.ReadFile(genesisPath)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot read genesis file %s: %v\n", *genesisFile, err)
-		os.Exit(1)
+		fmt.Fprintf(os.Stderr, "Cannot read genesis file %s: %v\n", genesisPath, err)
+		return 1
 	}
 
 	var genesis bookkeeping.Genesis
 	err = protocol.DecodeJSON(genesisText, &genesis)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Cannot parse genesis file %s: %v\n", *genesisFile, err)
-		os.Exit(1)
+		fmt.Fprintf(os.Stderr, "Cannot parse genesis file %s: %v\n", genesisPath, err)
+		return 1
 	}
 
 	if *genesisPrint {
 		fmt.Println(genesis.ID())
-		return
+		return 0
 	}
 
 	// If data directory doesn't exist, we can't run. Don't bother trying.
 	if _, err := os.Stat(absolutePath); err != nil {
 		fmt.Fprintf(os.Stderr, "Data directory %s does not appear to be valid\n", dataDir)
-		os.Exit(1)
+		return 1
 	}
 
 	log := logging.Base()
@@ -146,11 +150,11 @@ func main() {
 	locked, err := fileLock.TryLock()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unexpected failure in establishing algod.lock: %s \n", err.Error())
-		os.Exit(1)
+		return 1
 	}
 	if !locked {
 		fmt.Fprintln(os.Stderr, "failed to lock algod.lock; is an instance of algod already running in this data directory?")
-		os.Exit(1)
+		return 1
 	}
 	defer fileLock.Unlock()
 
@@ -177,7 +181,7 @@ func main() {
 		}
 		if os.IsPermission(err) {
 			fmt.Fprintf(os.Stderr, "Permission error on accessing telemetry config: %v", err)
-			os.Exit(1)
+			return 1
 		}
 		fmt.Fprintf(os.Stdout, "Telemetry configured from '%s'\n", telemetryConfig.FilePath)
 
@@ -256,8 +260,7 @@ func main() {
 			url, err := network.ParseHostOrURL(peer)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Provided command line parameter '%s' is not a valid host:port pair\n", peer)
-				os.Exit(1)
-				return
+				return 1
 			}
 			peerOverrideArray[idx] = url.Host
 		}
@@ -293,12 +296,11 @@ func main() {
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		log.Error(err)
-		os.Exit(1)
-		return
+		return 1
 	}
 
 	if *initAndExit {
-		return
+		return 0
 	}
 
 	deadlockState := "enabled"
@@ -361,6 +363,7 @@ func main() {
 	}
 
 	s.Start()
+	return 0
 }
 
 func resolveDataDir() string {

cmd/algod/main_test.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2019-2021 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/algorand/go-algorand/config"
+)
+
+func BenchmarkAlgodStartup(b *testing.B) {
+	tmpDir, err := ioutil.TempDir(os.TempDir(), "BenchmarkAlgodStartup")
+	require.NoError(b, err)
+	defer os.RemoveAll(tmpDir)
+	genesisFile, err := ioutil.ReadFile("../../installer/genesis/devnet/genesis.json")
+	require.NoError(b, err)
+
+	dataDirectory = &tmpDir
+	bInitAndExit := true
+	initAndExit = &bInitAndExit
+	b.StartTimer()
+	for n := 0; n < b.N; n++ {
+		err := ioutil.WriteFile(filepath.Join(tmpDir, config.GenesisJSONFile), genesisFile, 0766)
+		require.NoError(b, err)
+		fmt.Printf("file %s was written\n", filepath.Join(tmpDir, config.GenesisJSONFile))
+		run()
+		os.RemoveAll(tmpDir)
+		os.Mkdir(tmpDir, 0766)
+	}
+}

ledger/accountdb.go

@@ -521,27 +521,27 @@ func getCatchpoint(tx *sql.Tx, round basics.Round) (fileName string, catchpoint
 //
 // accountsInit returns nil if either it has initialized the database
 // correctly, or if the database has already been initialized.
-func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) error {
+func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData, proto config.ConsensusParams) (newDatabase bool, err error) {
 	for _, tableCreate := range accountsSchema {
-		_, err := tx.Exec(tableCreate)
+		_, err = tx.Exec(tableCreate)
 		if err != nil {
-			return err
+			return
 		}
 	}
 
 	// Run creatables migration if it hasn't run yet
 	var creatableMigrated bool
-	err := tx.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated)
+	err = tx.QueryRow("SELECT 1 FROM pragma_table_info('assetcreators') WHERE name='ctype'").Scan(&creatableMigrated)
 	if err == sql.ErrNoRows {
 		// Run migration
 		for _, migrateCmd := range creatablesMigration {
 			_, err = tx.Exec(migrateCmd)
 			if err != nil {
-				return err
+				return
 			}
 		}
 	} else if err != nil {
-		return err
+		return
 	}
 
 	_, err = tx.Exec("INSERT INTO acctrounds (id, rnd) VALUES ('acctbase', 0)")
@@ -553,30 +553,32 @@ func accountsInit(tx *sql.Tx, initAccounts map[basics.Address]basics.AccountData
 			_, err = tx.Exec("INSERT INTO accountbase (address, data) VALUES (?, ?)",
 				addr[:], protocol.Encode(&data))
 			if err != nil {
-				return err
+				return true, err
 			}
 
 			totals.AddAccount(proto, data, &ot)
 		}
 
 		if ot.Overflowed {
-			return fmt.Errorf("overflow computing totals")
+			return true, fmt.Errorf("overflow computing totals")
 		}
 
 		err = accountsPutTotals(tx, totals, false)
 		if err != nil {
-			return err
+			return true, err
 		}
+		newDatabase = true
 	} else {
 		serr, ok := err.(sqlite3.Error)
 		// serr.Code is sqlite.ErrConstraint if the database has already been initialized;
 		// in that case, ignore the error and return nil.
 		if !ok || serr.Code != sqlite3.ErrConstraint {
-			return err
+			return
 		}
+
 	}
 
-	return nil
+	return newDatabase, nil
 }
 
 // accountsAddNormalizedBalance adds the normalizedonlinebalance column

ledger/accountdb_test.go

@@ -439,12 +439,14 @@ func TestAccountDBInit(t *testing.T) {
 	defer tx.Rollback()
 
 	accts := randomAccounts(20, true)
-	err = accountsInit(tx, accts, proto)
+	newDB, err := accountsInit(tx, accts, proto)
 	require.NoError(t, err)
+	require.True(t, newDB)
 	checkAccounts(t, tx, 0, accts)
 
-	err = accountsInit(tx, accts, proto)
+	newDB, err = accountsInit(tx, accts, proto)
 	require.NoError(t, err)
+	require.False(t, newDB)
 	checkAccounts(t, tx, 0, accts)
 }
 
@@ -528,7 +530,7 @@ func TestAccountDBRound(t *testing.T) {
 	defer tx.Rollback()
 
 	accts := randomAccounts(20, true)
-	err = accountsInit(tx, accts, proto)
+	_, err = accountsInit(tx, accts, proto)
 	require.NoError(t, err)
 	checkAccounts(t, tx, 0, accts)
 
@@ -735,7 +737,7 @@ func benchmarkInitBalances(b testing.TB, numAccounts int, dbs db.Pair, proto con
 
 	updates = generateRandomTestingAccountBalances(numAccounts)
 
-	err = accountsInit(tx, updates, proto)
+	_, err = accountsInit(tx, updates, proto)
 	require.NoError(b, err)
 	err = accountsAddNormalizedBalance(tx, proto)
 	require.NoError(b, err)
@@ -956,7 +958,7 @@ func TestAccountsReencoding(t *testing.T) {
 	pubVrfKey, _ := crypto.VrfKeygenFromSeed([32]byte{0, 1, 2, 3})
 
 	err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
-		err = accountsInit(tx, make(map[basics.Address]basics.AccountData), config.Consensus[protocol.ConsensusCurrentVersion])
+		_, err = accountsInit(tx, make(map[basics.Address]basics.AccountData), config.Consensus[protocol.ConsensusCurrentVersion])
 		if err != nil {
 			return err
 		}
@@ -1034,7 +1036,7 @@ func TestAccountsDbQueriesCreateClose(t *testing.T) {
 	defer dbs.Close()
 
 	err := dbs.Wdb.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
-		err = accountsInit(tx, make(map[basics.Address]basics.AccountData), config.Consensus[protocol.ConsensusCurrentVersion])
+		_, err = accountsInit(tx, make(map[basics.Address]basics.AccountData), config.Consensus[protocol.ConsensusCurrentVersion])
 		if err != nil {
 			return err
 		}