From 5cd9f4e95b0f60f34c61287691806253bc165ca1 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 7 Aug 2023 16:13:12 -0400 Subject: [PATCH 1/4] Flush ledger cache. --- tools/block-generator/generator/generate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 3a899a33b4..9e2ec7767d 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -350,6 +350,7 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return fmt.Errorf("failed to add validated block: %w", err) } + g.ledger.FlushCaches() cert.Block.Payset = vBlock.Block().Payset From 93a06927c606b3dd43066978469799354220b9e4 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Mon, 7 Aug 2023 16:57:53 -0400 Subject: [PATCH 2/4] Use wait for commit. --- tools/block-generator/generator/generate.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 9e2ec7767d..b9420da43f 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -350,7 +350,9 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return fmt.Errorf("failed to add validated block: %w", err) } - g.ledger.FlushCaches() + //g.ledger.FlushCaches() + //<-g.ledger.Wait(vBlock.Block().Round()) + g.ledger.WaitForCommit(vBlock.Block().Round()) cert.Block.Payset = vBlock.Block().Payset From 6e6663ae8e5045c944d42f63ec067e029e2ceb08 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 8 Aug 2023 12:23:27 -0400 Subject: [PATCH 3/4] precompile block-generator teal code. --- tools/block-generator/generator/daemon.go | 4 +- tools/block-generator/generator/generate.go | 38 ++++++++++++++++++- .../generator/make_transactions.go | 6 +-- tools/block-generator/generator/server.go | 4 +- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/tools/block-generator/generator/daemon.go b/tools/block-generator/generator/daemon.go index 36c49cac51..fb4f52bab3 100644 --- a/tools/block-generator/generator/daemon.go +++ b/tools/block-generator/generator/daemon.go @@ -31,13 +31,14 @@ func init() { var configFile string var port uint64 + var verbose bool DaemonCmd = &cobra.Command{ Use: "daemon", Short: "Start the generator daemon in standalone mode.", Run: func(cmd *cobra.Command, args []string) { addr := fmt.Sprintf(":%d", port) - srv, _ := MakeServer(configFile, addr) + srv, _ := MakeServer(configFile, addr, verbose) err := srv.ListenAndServe() if err != nil { panic(err) @@ -47,6 +48,7 @@ func init() { DaemonCmd.Flags().StringVarP(&configFile, "config", "c", "", "Specify the block configuration yaml file.") DaemonCmd.Flags().Uint64VarP(&port, "port", "p", 4010, "Port to start the server at.") + DaemonCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "If set the runner will print debugging information from the generator and ledger.") DaemonCmd.MarkFlagRequired("config") } diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index b9420da43f..7e5e284f05 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -32,25 +32,49 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" txn "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/tools/block-generator/util" ) // ---- templates ---- //go:embed teal/poap_boxes.teal var approvalBoxes string +var approvalBoxesBytes interface{} //go:embed teal/poap_clear.teal var clearBoxes string +var clearBoxesBytes interface{} //go:embed teal/swap_amm.teal var approvalSwap string +var approvalSwapBytes interface{} //go:embed teal/swap_clear.teal var clearSwap string +var clearSwapBytes interface{} + +func init() { + prog, err := logic.AssembleString(approvalBoxes) + util.MaybeFail(err, "failed to assemble approval program") + approvalBoxesBytes = prog.Program + + prog, err = logic.AssembleString(clearBoxes) + util.MaybeFail(err, "failed to assemble clear program") + clearBoxesBytes = prog.Program + + prog, err = logic.AssembleString(approvalSwap) + util.MaybeFail(err, "failed to assemble approvalSwap program") + approvalSwapBytes = prog.Program + + prog, err = logic.AssembleString(clearSwap) + util.MaybeFail(err, "failed to assemble clearSwap program") + clearSwapBytes = prog.Program +} // ---- constructors ---- @@ -305,8 +329,8 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { } return nil } - // round == nextRound case + // round == nextRound case err := g.startRound() if err != nil { return err @@ -321,6 +345,15 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { // we'll write genesis block / offset round for non-empty database cert.Block, _, _ = g.ledger.BlockCert(basics.Round(round - g.roundOffset)) } else { + start := time.Now() + var generated, evaluated, validated time.Time + if g.verbose { + defer func() { + fmt.Printf("block generation stats txn generation (%s), ledger eval (%s), ledger add block (%s)\n", + generated.Sub(start), evaluated.Sub(generated), validated.Sub(evaluated)) + }() + } + g.setBlockHeader(&cert) intra := uint64(0) @@ -337,6 +370,7 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { intra += numTxns } + generated = time.Now() vBlock, ledgerTxnCount, err := g.evaluateBlock(cert.Block.BlockHeader, txGroupsAD, int(intra)) if err != nil { @@ -345,6 +379,7 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if ledgerTxnCount != g.txnCounter+intra { return fmt.Errorf("evaluateBlock() txn count mismatches theoretical intra: %d != %d", ledgerTxnCount, g.txnCounter+intra) } + evaluated = time.Now() err = g.ledger.AddValidatedBlock(*vBlock, cert.Certificate) if err != nil { @@ -353,6 +388,7 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { //g.ledger.FlushCaches() //<-g.ledger.Wait(vBlock.Block().Round()) g.ledger.WaitForCommit(vBlock.Block().Round()) + validated = time.Now() cert.Block.Payset = vBlock.Block().Payset diff --git a/tools/block-generator/generator/make_transactions.go b/tools/block-generator/generator/make_transactions.go index e733d4339b..6caffe25b7 100644 --- a/tools/block-generator/generator/make_transactions.go +++ b/tools/block-generator/generator/make_transactions.go @@ -123,11 +123,11 @@ func (g *generator) makeAssetAcceptanceTxn(header txn.Header, index uint64) txn. // ---- application transactions ---- func (g *generator) makeAppCreateTxn(kind appKind, sender basics.Address, round, intra uint64, futureAppId uint64) []txn.SignedTxn { - var approval, clear string + var approval, clear interface{} if kind == appKindSwap { - approval, clear = approvalSwap, clearSwap + approval, clear = approvalSwapBytes, clearSwapBytes } else { - approval, clear = approvalBoxes, clearBoxes + approval, clear = approvalBoxesBytes, clearBoxesBytes } createTxn := g.makeTestTxn(sender, round, intra) diff --git a/tools/block-generator/generator/server.go b/tools/block-generator/generator/server.go index 4de0d08322..edfe470f3d 100644 --- a/tools/block-generator/generator/server.go +++ b/tools/block-generator/generator/server.go @@ -29,11 +29,11 @@ import ( ) // MakeServer configures http handlers. Returns the http server. -func MakeServer(configFile string, addr string) (*http.Server, Generator) { +func MakeServer(configFile string, addr string, verbose bool) (*http.Server, Generator) { noOp := func(next http.Handler) http.Handler { return next } - return MakeServerWithMiddleware(nil, 0, "", configFile, false, addr, noOp) + return MakeServerWithMiddleware(nil, 0, "", configFile, verbose, addr, noOp) } // BlocksMiddleware is a middleware for the blocks endpoint. From 5e31835cb62441ba574ec0a07b5ffeb7f2d505cf Mon Sep 17 00:00:00 2001 From: Will Winder Date: Tue, 8 Aug 2023 12:24:07 -0400 Subject: [PATCH 4/4] revert flush change. --- tools/block-generator/generator/generate.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/block-generator/generator/generate.go b/tools/block-generator/generator/generate.go index 7e5e284f05..22246e047b 100644 --- a/tools/block-generator/generator/generate.go +++ b/tools/block-generator/generator/generate.go @@ -385,9 +385,6 @@ func (g *generator) WriteBlock(output io.Writer, round uint64) error { if err != nil { return fmt.Errorf("failed to add validated block: %w", err) } - //g.ledger.FlushCaches() - //<-g.ledger.Wait(vBlock.Block().Round()) - g.ledger.WaitForCommit(vBlock.Block().Round()) validated = time.Now() cert.Block.Payset = vBlock.Block().Payset