Simplify dryrunning and debugging group txn (#1405)

Extend clerk dryrun so it can dump dryrun state object suitable for REST, tealdbg and python tests

cmd/goal/clerk.go

@@ -128,6 +128,9 @@ func init() {
 
 	dryrunCmd.Flags().StringVarP(&txFilename, "txfile", "t", "", "transaction or transaction-group to test")
 	dryrunCmd.Flags().StringVarP(&protoVersion, "proto", "P", "", "consensus protocol version id string")
+	dryrunCmd.Flags().BoolVar(&dumpForDryrun, "dryrun-dump", false, "Dump in dryrun format acceptable by dryrun REST api instead of running")
+	dryrunCmd.Flags().Var(&dumpForDryrunFormat, "dryrun-dump-format", "Dryrun dump format: "+dumpForDryrunFormat.AllowedString())
+	dryrunCmd.Flags().StringVarP(&outFilename, "outfile", "o", "", "Filename for writing dryrun state object")
 	dryrunCmd.MarkFlagRequired("txfile")
 
 	dryrunRemoteCmd.Flags().StringVarP(&txFilename, "dryrun-state", "D", "", "dryrun request object to run")
@@ -979,10 +982,22 @@ var dryrunCmd = &cobra.Command{
 		for i, st := range stxns {
 			txgroup[i] = st
 		}
+		proto, params := getProto(protoVersion)
+		if dumpForDryrun {
+			// Write dryrun data to file
+			dataDir := ensureSingleDataDir()
+			client := ensureFullClient(dataDir)
+			data, err := libgoal.MakeDryrunStateBytes(client, nil, txgroup, string(proto), dumpForDryrunFormat.String())
+			if err != nil {
+				reportErrorf(err.Error())
+			}
+			writeFile(outFilename, data, 0600)
+			return
+		}
+
 		if timeStamp <= 0 {
 			timeStamp = time.Now().Unix()
 		}
-		_, params := getProto(protoVersion)
 		for i, txn := range txgroup {
 			if txn.Lsig.Blank() {
 				continue

cmd/tealdbg/main.go

@@ -184,8 +184,8 @@ func debugLocal(args []string) {
 	// program can be set either directly
 	// or with SignedTxn.Lsig.Logic,
 	// or with BalanceRecord.AppParams.ApprovalProgram
-	if len(args) == 0 && (len(txnFile) == 0 || len(balanceFile) == 0) && len(ddrFile) == 0 {
-		log.Fatalln("No program to debug: must specify program(s), or transaction(s) and a balance record(s), or dryrun-req object")
+	if len(args) == 0 && len(txnFile) == 0 && len(ddrFile) == 0 {
+		log.Fatalln("No program to debug: must specify program(s), or transaction(s), or dryrun-req object")
 	}
 
 	if len(args) == 0 && groupIndex != 0 {

daemon/algod/api/server/v2/dryrun.go

@@ -411,12 +411,11 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response
 			GroupIndex: ti,
 			//Logger: nil, // TODO: capture logs, send them back
 		}
-		var result *generated.DryrunTxnResult
+		var result generated.DryrunTxnResult
 		if len(stxn.Lsig.Logic) > 0 {
 			var debug dryrunDebugReceiver
 			ep.Debugger = &debug
 			pass, err := logic.Eval(stxn.Lsig.Logic, ep)
-			result = new(generated.DryrunTxnResult)
 			var messages []string
 			result.Disassembly = debug.lines
 			result.LogicSigTrace = &debug.history
@@ -460,9 +459,6 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response
 				}
 			}
 			var messages []string
-			if result == nil {
-				result = new(generated.DryrunTxnResult)
-			}
 			if !ok {
 				messages = make([]string, 1)
 				messages[0] = fmt.Sprintf("uploaded state did not include app id %d referenced in txn[%d]", appIdx, ti)
@@ -507,7 +503,7 @@ func doDryrunRequest(dr *DryrunRequest, proto *config.ConsensusParams, response
 			}
 			result.AppCallMessages = &messages
 		}
-		response.Txns[ti] = *result
+		response.Txns[ti] = result
 	}
 }
 

libgoal/libgoal.go

@@ -926,7 +926,7 @@ func MakeDryrunStateBytes(client Client, txnOrStxn interface{}, other []transact
 	}
 }
 
-// MakeDryrunState function creates DryrunRequest data structure and serializes it into a file
+// MakeDryrunState function creates v2.DryrunRequest data structure
 func MakeDryrunState(client Client, txnOrStxn interface{}, other []transactions.SignedTxn, proto string) (dr v2.DryrunRequest, err error) {
 	gdr, err := MakeDryrunStateGenerated(client, txnOrStxn, other, proto)
 	if err != nil {
@@ -935,78 +935,81 @@ func MakeDryrunState(client Client, txnOrStxn interface{}, other []transactions.
 	return v2.DryrunRequestFromGenerated(&gdr)
 }
 
-// MakeDryrunStateGenerated function creates DryrunRequest data structure and serializes it into a file
+// MakeDryrunStateGenerated function creates generatedV2.DryrunRequest data structure
 func MakeDryrunStateGenerated(client Client, txnOrStxn interface{}, other []transactions.SignedTxn, proto string) (dr generatedV2.DryrunRequest, err error) {
 	var txns []transactions.SignedTxn
-	var tx *transactions.Transaction
-	if txn, ok := txnOrStxn.(transactions.Transaction); ok {
-		tx = &txn
+	if txnOrStxn == nil {
+		// empty input do nothing
+	} else if txn, ok := txnOrStxn.(transactions.Transaction); ok {
 		txns = append(txns, transactions.SignedTxn{Txn: txn})
 	} else if stxn, ok := txnOrStxn.(transactions.SignedTxn); ok {
-		tx = &stxn.Txn
 		txns = append(txns, stxn)
 	} else {
 		err = fmt.Errorf("unsupported txn type")
 		return
 	}
+
 	txns = append(txns, other...)
 	for i := range txns {
 		enc := protocol.EncodeJSON(&txns[i])
 		dr.Txns = append(dr.Txns, enc)
 	}
 
-	if tx.Type == protocol.ApplicationCallTx {
-		apps := []basics.AppIndex{tx.ApplicationID}
-		apps = append(apps, tx.ForeignApps...)
-		for _, appIdx := range apps {
-			var appParams generatedV2.ApplicationParams
-			if appIdx == 0 {
-				// if it is an app create txn then use params from the txn
-				appParams.ApprovalProgram = tx.ApprovalProgram
-				appParams.ClearStateProgram = tx.ClearStateProgram
-				appParams.GlobalStateSchema = &generatedV2.ApplicationStateSchema{
-					NumUint:      tx.GlobalStateSchema.NumUint,
-					NumByteSlice: tx.GlobalStateSchema.NumByteSlice,
-				}
-				appParams.LocalStateSchema = &generatedV2.ApplicationStateSchema{
-					NumUint:      tx.LocalStateSchema.NumUint,
-					NumByteSlice: tx.LocalStateSchema.NumByteSlice,
+	for _, txn := range txns {
+		tx := txn.Txn
+		if tx.Type == protocol.ApplicationCallTx {
+			apps := []basics.AppIndex{tx.ApplicationID}
+			apps = append(apps, tx.ForeignApps...)
+			for _, appIdx := range apps {
+				var appParams generatedV2.ApplicationParams
+				if appIdx == 0 {
+					// if it is an app create txn then use params from the txn
+					appParams.ApprovalProgram = tx.ApprovalProgram
+					appParams.ClearStateProgram = tx.ClearStateProgram
+					appParams.GlobalStateSchema = &generatedV2.ApplicationStateSchema{
+						NumUint:      tx.GlobalStateSchema.NumUint,
+						NumByteSlice: tx.GlobalStateSchema.NumByteSlice,
+					}
+					appParams.LocalStateSchema = &generatedV2.ApplicationStateSchema{
+						NumUint:      tx.LocalStateSchema.NumUint,
+						NumByteSlice: tx.LocalStateSchema.NumByteSlice,
+					}
+					appParams.Creator = tx.Sender.String()
+					// zero is not acceptable by ledger in dryrun/debugger
+					appIdx = defaultAppIdx
+				} else {
+					// otherwise need to fetch app state
+					var app generatedV2.Application
+					if app, err = client.ApplicationInformation(uint64(tx.ApplicationID)); err != nil {
+						return
+					}
+					appParams = app.Params
 				}
-				appParams.Creator = tx.Sender.String()
-				// zero is not acceptable by ledger in dryrun/debugger
-				appIdx = defaultAppIdx
-			} else {
-				// otherwise need to fetch app state
-				var app generatedV2.Application
-				if app, err = client.ApplicationInformation(uint64(tx.ApplicationID)); err != nil {
+				dr.Apps = append(dr.Apps, generatedV2.Application{
+					Id:     uint64(appIdx),
+					Params: appParams,
+				})
+			}
+
+			accounts := append(tx.Accounts, tx.Sender)
+			for _, acc := range accounts {
+				var info generatedV2.Account
+				if info, err = client.AccountInformationV2(acc.String()); err != nil {
 					return
 				}
-				appParams = app.Params
+				dr.Accounts = append(dr.Accounts, info)
 			}
-			dr.Apps = append(dr.Apps, generatedV2.Application{
-				Id:     uint64(appIdx),
-				Params: appParams,
-			})
-		}
 
-		accounts := append(tx.Accounts, tx.Sender)
-		for _, acc := range accounts {
-			var info generatedV2.Account
-			if info, err = client.AccountInformationV2(acc.String()); err != nil {
+			dr.ProtocolVersion = proto
+			if dr.Round, err = client.CurrentRound(); err != nil {
 				return
 			}
-			dr.Accounts = append(dr.Accounts, info)
-		}
-
-		dr.ProtocolVersion = proto
-		if dr.Round, err = client.CurrentRound(); err != nil {
-			return
-		}
-		var b v1.Block
-		if b, err = client.Block(dr.Round); err != nil {
-			return
+			var b v1.Block
+			if b, err = client.Block(dr.Round); err != nil {
+				return
+			}
+			dr.LatestTimestamp = uint64(b.Timestamp)
 		}
-		dr.LatestTimestamp = uint64(b.Timestamp)
 	}
 	return
 }

test/e2e-go/cli/goal/expect/goalDryrunRestTest.exp

@@ -101,6 +101,27 @@ if { [catch {
         timeout { ::AlgorandGoal::Abort "goal app create timeout" }
     }
 
+    # atomic transfer
+    set DRREQ_FILE_4 "$TEST_ROOT_DIR/atomic-tran-drreq.msgp"
+    set AT_TX1_FILE "$TEST_ROOT_DIR/atomic-tran-tx1.mspg"
+    set AT_TX2_FILE "$TEST_ROOT_DIR/atomic-tran-tx2.mspg"
+    set AT_COMBINED_FILE "$TEST_ROOT_DIR/atomic-tran-comb.mspg"
+    set AT_GROUPPED_FILE "$TEST_ROOT_DIR/atomic-tran-group.mspg"
+    spawn goal clerk send --from $PRIMARY_ACCOUNT_ADDRESS --to $PRIMARY_ACCOUNT_ADDRESS -a 1 --fee 1000 -d $TEST_PRIMARY_NODE_DIR -o $AT_TX1_FILE
+    expect {
+        timeout { ::AlgorandGoal::Abort "goal clerk send timeout" }
+    }
+    spawn goal app create --creator $PRIMARY_ACCOUNT_ADDRESS --approval-prog $TEAL_PROG_FILE --clear-prog $TEAL_PROG_FILE --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0 -d $TEST_PRIMARY_NODE_DIR -o $AT_TX2_FILE
+    expect {
+        timeout { ::AlgorandGoal::Abort "goal app create timeout" }
+    }
+    exec cat $AT_TX1_FILE $AT_TX2_FILE > $AT_COMBINED_FILE
+    exec goal clerk group -i $AT_COMBINED_FILE -o $AT_GROUPPED_FILE
+    spawn goal clerk dryrun -t $AT_GROUPPED_FILE -d $TEST_PRIMARY_NODE_DIR -o $DRREQ_FILE_4 --dryrun-dump --dryrun-dump-format=msgp
+    expect {
+        timeout { ::AlgorandGoal::Abort "goal clerk dryrun timeout" }
+    }
+
     # invalid app
     set INVALID_FILE_1 "$TEST_ROOT_DIR/invalid-app.json"
     set INVALID_FILE_1_ID [open $INVALID_FILE_1 "w"]
@@ -111,6 +132,8 @@ if { [catch {
     TestGoalDryrun $DRREQ_FILE_1 $TEST_PRIMARY_NODE_DIR
     TestGoalDryrun $DRREQ_FILE_2 $TEST_PRIMARY_NODE_DIR
     TestGoalDryrun $DRREQ_FILE_3 $TEST_PRIMARY_NODE_DIR
+    TestGoalDryrun $DRREQ_FILE_4 $TEST_PRIMARY_NODE_DIR
+
     TestGoalDryrunExitCode $DRREQ_FILE_3 $TEST_PRIMARY_NODE_DIR 0 "PASS"
     TestGoalDryrunExitCode "" $TEST_PRIMARY_NODE_DIR 1 "Cannot read file : open : no such file or directory"
     TestGoalDryrunExitCode $INVALID_FILE_1 $TEST_PRIMARY_NODE_DIR 1 "dryrun-remote: HTTP 400 Bad Request:"