Execution traces for call sequences failing tests (#105)

* Introduction of execution trace attaching and displaying for failed test call sequences
* Added helpers to extract Solidity errors/panics (now used by assertion test provider and execution tracer)
* Added fuzzer tests to verify execution trace output
* Updated README with required solidity version for unit tests
* Provided a wrapper for ExecuteCallSequenceIteratively, to simply re-execute a given call sequence without any custom checks
* Updated FuzzerWorker to re-execute the finalized shrunken sequence before performing the "finalized shrunken sequence" callback, so all providers can assume state is post-execution.
* Updated assertion test provider to attach an execution trace for the last call in a failing sequence
* Updated property test provider to attach execution traces for the last call in a failing sequence, and for its property test
* Updated the fuzzer worker so if a "trace all" flag is provided, it will attach execution traces to all finalized shrunken call sequence elements prior to returning the results to any test provider
* Updated TestChain to provide a method to get the post-execution state root hash for a given block number
* Cleaned up ABI error and event helpers by splitting them off to a compilation-related "abiutils" package.

---------

Co-authored-by: anishnaik <[email protected]>

README.md

@@ -83,7 +83,7 @@ This will use the `medusa.json` configuration in the current directory and begin
 
 ## Running Unit Tests
 
-First, install [crytic-compile](https://github.com/crytic/crytic-compile), [solc-select](https://github.com/crytic/solc-select), and ensure you have `solc`, `truffle`, and `hardhat` available on your system.
+First, install [crytic-compile](https://github.com/crytic/crytic-compile), [solc-select](https://github.com/crytic/solc-select), and ensure you have `solc` (version >=0.8.7), `truffle`, and `hardhat` available on your system.
 
 - From the root of the repository, invoke `go test -v ./...` on through command-line to run tests from all packages at or below the root.
   - Or enter each package directory to run `go test -v .` to test the immediate package.

chain/cheat_code_contract.go

@@ -7,9 +7,12 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 )
 
-// cheatCodeContract defines a struct which represents a pre-compiled contract with various methods that is
+// CheatCodeContract defines a struct which represents a pre-compiled contract with various methods that is
 // meant to act as a contract.
-type cheatCodeContract struct {
+type CheatCodeContract struct {
+	// The name of the cheat code contract.
+	name string
+
 	// address defines the address the cheat code contract should be installed at.
 	address common.Address
 
@@ -19,6 +22,9 @@ type cheatCodeContract struct {
 	// methodInfo describes a table of methodId (function selectors) to cheat code methods. This acts as a switch table
 	// for different methods in the contract.
 	methodInfo map[uint32]*cheatCodeMethod
+
+	// abi refers to the cheat code contract's ABI definition.
+	abi abi.ABI
 }
 
 // cheatCodeMethod defines the method information for a given precompiledContract.
@@ -48,11 +54,20 @@ type cheatCodeRawReturnData struct {
 
 // newCheatCodeContract returns a new precompiledContract which uses the attached cheatCodeTracer for execution
 // context.
-func newCheatCodeContract(tracer *cheatCodeTracer, address common.Address) *cheatCodeContract {
-	return &cheatCodeContract{
+func newCheatCodeContract(tracer *cheatCodeTracer, address common.Address, name string) *CheatCodeContract {
+	return &CheatCodeContract{
+		name:       name,
 		address:    address,
 		tracer:     tracer,
 		methodInfo: make(map[uint32]*cheatCodeMethod),
+		abi: abi.ABI{
+			Constructor: abi.Method{},
+			Methods:     make(map[string]abi.Method),
+			Events:      make(map[string]abi.Event),
+			Errors:      make(map[string]abi.Error),
+			Fallback:    abi.Method{},
+			Receive:     abi.Method{},
+		},
 	}
 }
 
@@ -65,9 +80,24 @@ func cheatCodeRevertData(returnData []byte) *cheatCodeRawReturnData {
 	}
 }
 
+// Name represents the name of the cheat code contract.
+func (c *CheatCodeContract) Name() string {
+	return c.name
+}
+
+// Address represents the address the cheat code contract is deployed at.
+func (c *CheatCodeContract) Address() common.Address {
+	return c.address
+}
+
+// Abi provides the cheat code contract interface.
+func (c *CheatCodeContract) Abi() *abi.ABI {
+	return &c.abi
+}
+
 // addMethod adds a new method to the precompiled contract.
 // Returns an error if one occurred.
-func (p *cheatCodeContract) addMethod(name string, inputs abi.Arguments, outputs abi.Arguments, handler cheatCodeMethodHandler) {
+func (c *CheatCodeContract) addMethod(name string, inputs abi.Arguments, outputs abi.Arguments, handler cheatCodeMethodHandler) {
 	// Verify a method name was provided
 	if name == "" {
 		panic("could not add method to precompiled cheatcode contract, empty method name provided")
@@ -81,31 +111,36 @@ func (p *cheatCodeContract) addMethod(name string, inputs abi.Arguments, outputs
 	// Set the method information in our method lookup
 	method := abi.NewMethod(name, name, abi.Function, "external", false, false, inputs, outputs)
 	methodId := binary.LittleEndian.Uint32(method.ID)
-	p.methodInfo[methodId] = &cheatCodeMethod{
+	c.methodInfo[methodId] = &cheatCodeMethod{
 		method:  method,
 		handler: handler,
 	}
+
+	// Add the method to the ABI.
+	// Note: Normally the key here should be the method name, not sig. But cheat code contracts have duplicate
+	// method names with different parameter types, so we use this so they don't override.
+	c.abi.Methods[method.Sig] = method
 }
 
 // RequiredGas determines the amount of gas necessary to execute the pre-compile with the given input data.
 // Returns the gas cost.
-func (p *cheatCodeContract) RequiredGas(input []byte) uint64 {
+func (c *CheatCodeContract) RequiredGas(input []byte) uint64 {
 	return 0
 }
 
 // Run executes the given pre-compile with the provided input data.
 // Returns the output data from execution, or an error if one occurred.
-func (p *cheatCodeContract) Run(input []byte) ([]byte, error) {
+func (c *CheatCodeContract) Run(input []byte) ([]byte, error) {
 	// Calling any method should require at least a signature
 	if len(input) < 4 {
 		return []byte{}, vm.ErrExecutionReverted
 	}
 
-	// Obtain the method identifier as a uint32
+	// Obtain the method identifier as an uint32
 	methodId := binary.LittleEndian.Uint32(input[:4])
 
 	// Ensure we have a method definition that matches our selector.
-	methodInfo, methodInfoExists := p.methodInfo[methodId]
+	methodInfo, methodInfoExists := c.methodInfo[methodId]
 	if !methodInfoExists || methodId != binary.LittleEndian.Uint32(methodInfo.method.ID) {
 		return []byte{}, vm.ErrExecutionReverted
 	}
@@ -117,7 +152,7 @@ func (p *cheatCodeContract) Run(input []byte) ([]byte, error) {
 	}
 
 	// Call the registered method handler.
-	outputValues, rawReturnData := methodInfo.handler(p.tracer, inputValues)
+	outputValues, rawReturnData := methodInfo.handler(c.tracer, inputValues)
 
 	// If we have raw return data, use that. Otherwise, proceed to unpack the returned output values.
 	if rawReturnData != nil {

chain/cheat_codes.go

@@ -14,11 +14,11 @@ import (
 	"strings"
 )
 
-// getCheatCodeProviders obtains a cheatCodeTracer (used to power cheat code analysis) and associated cheatCodeContract
+// getCheatCodeProviders obtains a cheatCodeTracer (used to power cheat code analysis) and associated CheatCodeContract
 // objects linked to the tracer (providing on-chain callable methods as an entry point). These objects are attached to
 // the TestChain to enable cheat code functionality.
 // Returns the tracer and associated pre-compile contracts, or an error, if one occurred.
-func getCheatCodeProviders() (*cheatCodeTracer, []*cheatCodeContract, error) {
+func getCheatCodeProviders() (*cheatCodeTracer, []*CheatCodeContract, error) {
 	// Create a cheat code tracer and attach it to the chain.
 	tracer := newCheatCodeTracer()
 
@@ -29,15 +29,15 @@ func getCheatCodeProviders() (*cheatCodeTracer, []*cheatCodeContract, error) {
 	}
 
 	// Return the tracer and precompiles
-	return tracer, []*cheatCodeContract{stdCheatCodeContract}, nil
+	return tracer, []*CheatCodeContract{stdCheatCodeContract}, nil
 }
 
-// getStandardCheatCodeContract obtains a cheatCodeContract which implements common cheat codes.
+// getStandardCheatCodeContract obtains a CheatCodeContract which implements common cheat codes.
 // Returns the precompiled contract, or an error if one occurs.
-func getStandardCheatCodeContract(tracer *cheatCodeTracer) (*cheatCodeContract, error) {
+func getStandardCheatCodeContract(tracer *cheatCodeTracer) (*CheatCodeContract, error) {
 	// Define our address for this precompile contract, then create a new precompile to add methods to.
 	contractAddress := common.HexToAddress("0x7109709ECfa91a80626fF3989D68f67F5b1DD12D")
-	contract := newCheatCodeContract(tracer, contractAddress)
+	contract := newCheatCodeContract(tracer, contractAddress, "StdCheats")
 
 	// Define some basic ABI argument types
 	typeAddress, err := abi.NewType("address", "", nil)

chain/test_chain.go

@@ -270,6 +270,22 @@ func (t *TestChain) State() *state.StateDB {
 	return t.state
 }
 
+// CheatCodeContracts returns all cheat code contracts which are installed in the chain.
+func (t *TestChain) CheatCodeContracts() map[common.Address]*CheatCodeContract {
+	// Create a map of cheat code contracts to store our results
+	contracts := make(map[common.Address]*CheatCodeContract, 0)
+
+	// Loop for each precompile, and try to see any which are of the "cheat code contract" type.
+	for address, precompile := range t.vmConfigExtensions.AdditionalPrecompiles {
+		if cheatCodeContract, ok := precompile.(*CheatCodeContract); ok {
+			contracts[address] = cheatCodeContract
+		}
+	}
+
+	// Return the results
+	return contracts
+}
+
 // CommittedBlocks returns the real blocks which were committed to the chain, where methods such as BlockFromNumber
 // return the simulated chain state with intermediate blocks injected for block number jumps, etc.
 func (t *TestChain) CommittedBlocks() []*chainTypes.Block {
@@ -411,23 +427,43 @@ func (t *TestChain) BlockHashFromNumber(blockNumber uint64) (common.Hash, error)
 	}
 }
 
-// StateAfterBlockNumber obtains the Ethereum world state after processing all transactions in the provided block
-// number. Returns the state, or an error if one occurs.
-func (t *TestChain) StateAfterBlockNumber(blockNumber uint64) (*state.StateDB, error) {
+// StateFromRoot obtains a state from a given state root hash.
+// Returns the state, or an error if one occurred.
+func (t *TestChain) StateFromRoot(root common.Hash) (*state.StateDB, error) {
+	// Load our state from the database
+	stateDB, err := state.New(root, t.stateDatabase, nil)
+	if err != nil {
+		return nil, err
+	}
+	return stateDB, nil
+}
+
+// StateRootAfterBlockNumber obtains the Ethereum world state root hash after processing all transactions in the
+// provided block number. Returns the state, or an error if one occurs.
+func (t *TestChain) StateRootAfterBlockNumber(blockNumber uint64) (common.Hash, error) {
 	// If our block number references something too new, return an error
 	if blockNumber > t.HeadBlockNumber() {
-		return nil, fmt.Errorf("could not obtain post-state for block number %d because it exceeds the current head block number %d", blockNumber, t.HeadBlockNumber())
+		return common.Hash{}, fmt.Errorf("could not obtain post-state for block number %d because it exceeds the current head block number %d", blockNumber, t.HeadBlockNumber())
 	}
 
 	// Obtain our closest internally committed block
 	_, closestBlock := t.fetchClosestInternalBlock(blockNumber)
 
-	// Load our state from the database
-	stateDB, err := state.New(closestBlock.Header.Root, t.stateDatabase, nil)
+	// Return our state root hash
+	return closestBlock.Header.Root, nil
+}
+
+// StateAfterBlockNumber obtains the Ethereum world state after processing all transactions in the provided block
+// number. Returns the state, or an error if one occurs.
+func (t *TestChain) StateAfterBlockNumber(blockNumber uint64) (*state.StateDB, error) {
+	// Obtain our block's post-execution state root hash
+	root, err := t.StateRootAfterBlockNumber(blockNumber)
 	if err != nil {
 		return nil, err
 	}
-	return stateDB, nil
+
+	// Load our state from the database
+	return t.StateFromRoot(root)
 }
 
 // RevertToBlockNumber sets the head of the chain to the block specified by the provided block number and reloads
@@ -486,24 +522,38 @@ func (t *TestChain) RevertToBlockNumber(blockNumber uint64) error {
 }
 
 // CallContract performs a message call over the current test chain state and obtains a core.ExecutionResult.
-// This is similar to the CallContract method provided by Ethereum for use in calling pure/view functions.
+// This is similar to the CallContract method provided by Ethereum for use in calling pure/view functions, as it
+// executed a transaction without committing any changes, instead discarding them.
+// It takes an optional state argument, which is the state to execute the message over. If not provided, the
+// current pending state (or committed state if none is pending) will be used instead.
 // The state executed over may be a pending block state.
-func (t *TestChain) CallContract(msg core.Message) (*core.ExecutionResult, error) {
-	// Obtain our state snapshot (note: this is different from the test chain snapshot)
-	snapshot := t.state.Snapshot()
+func (t *TestChain) CallContract(msg core.Message, state *state.StateDB, additionalTracers ...vm.EVMLogger) (*core.ExecutionResult, error) {
+	// If our provided state is nil, use our current chain state.
+	if state == nil {
+		state = t.state
+	}
+
+	// Obtain our state snapshot to revert any changes after our call
+	snapshot := state.Snapshot()
 
 	// Set infinite balance to the fake caller account
-	from := t.state.GetOrNewStateObject(msg.From())
+	from := state.GetOrNewStateObject(msg.From())
 	from.SetBalance(math.MaxBig256)
 
 	// Create our transaction and block contexts for the vm
 	txContext := core.NewEVMTxContext(msg)
 	blockContext := newTestChainBlockContext(t, t.Head().Header)
 
+	// Create a new call tracer router that incorporates any additional tracers provided just for this call, while
+	// still calling our internal tracers.
+	extendedTracerRouter := NewTestChainTracerRouter()
+	extendedTracerRouter.AddTracer(t.callTracerRouter)
+	extendedTracerRouter.AddTracers(additionalTracers...)
+
 	// Create our EVM instance.
-	evm := vm.NewEVM(blockContext, txContext, t.state, t.chainConfig, vm.Config{
+	evm := vm.NewEVM(blockContext, txContext, state, t.chainConfig, vm.Config{
 		Debug:            true,
-		Tracer:           t.callTracerRouter,
+		Tracer:           extendedTracerRouter,
 		NoBaseFee:        true,
 		ConfigExtensions: t.vmConfigExtensions,
 	})
@@ -515,7 +565,7 @@ func (t *TestChain) CallContract(msg core.Message) (*core.ExecutionResult, error
 	res, err := core.NewStateTransition(evm, msg, gasPool).TransitionDb()
 
 	// Revert to our state snapshot to undo any changes.
-	t.state.RevertToSnapshot(snapshot)
+	state.RevertToSnapshot(snapshot)
 
 	return res, err
 }
@@ -628,6 +678,9 @@ func (t *TestChain) PendingBlockAddTx(message core.Message) error {
 		return errors.New("could not add tx to the chain's pending block because no pending block was created")
 	}
 
+	// Obtain our state root hash prior to execution.
+	previousStateRoot := t.pendingBlock.Header.Root
+
 	// Create a gas pool indicating how much gas can be spent executing the transaction.
 	gasPool := new(core.GasPool).AddGas(t.pendingBlock.Header.GasLimit - t.pendingBlock.Header.GasUsed)
 
@@ -657,6 +710,8 @@ func (t *TestChain) PendingBlockAddTx(message core.Message) error {
 
 	// Create our message result
 	messageResult := &chainTypes.MessageResults{
+		PreStateRoot:      previousStateRoot,
+		PostStateRoot:     common.Hash{},
 		ExecutionResult:   executionResult,
 		Receipt:           receipt,
 		AdditionalResults: make(map[string]any, 0),
@@ -684,10 +739,11 @@ func (t *TestChain) PendingBlockAddTx(message core.Message) error {
 	// Update our block's bloom filter
 	t.pendingBlock.Header.Bloom.Add(receipt.Bloom.Bytes())
 
-	// Update the header's state root hash
+	// Update the header's state root hash, as well as our message result's
 	// Note: You could also retrieve the root without committing by using
 	// state.IntermediateRoot(config.IsEIP158(parentBlockNumber)).
 	t.pendingBlock.Header.Root = root
+	messageResult.PostStateRoot = root
 
 	// Update our block's transactions and results.
 	t.pendingBlock.Messages = append(t.pendingBlock.Messages, message)

chain/types/message_results.go

@@ -1,13 +1,20 @@
 package types
 
 import (
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
 // MessageResults represents metadata obtained from the execution of a CallMessage in a Block.
 // This contains results such as contracts deployed, and other variables tracked by a chain.TestChain.
 type MessageResults struct {
+	// PreStateRoot refers to the state root hash prior to the execution of this transaction.
+	PreStateRoot common.Hash
+
+	// PostStateRoot refers to the state root hash after the execution of this transaction.
+	PostStateRoot common.Hash
+
 	// ExecutionResult describes the core.ExecutionResult returned after processing a given call.
 	ExecutionResult *core.ExecutionResult
 

cmd/fuzz_flags.go

@@ -61,6 +61,9 @@ func addFuzzFlags() error {
 	fuzzCmd.Flags().Bool("assertion-mode", false,
 		fmt.Sprintf("enable assertion mode (unless a config file is provided, default is %t)", defaultConfig.Fuzzing.Testing.AssertionTesting.Enabled))
 
+	// Trace all
+	fuzzCmd.Flags().Bool("trace-all", false,
+		fmt.Sprintf("print the execution trace for every element in a shrunken call sequence instead of only the last element (unless a config file is provided, default is %t)", defaultConfig.Fuzzing.Testing.TraceAll))
 	return nil
 }
 
@@ -154,5 +157,12 @@ func updateProjectConfigWithFuzzFlags(cmd *cobra.Command, projectConfig *config.
 		}
 	}
 
+	// Update trace all enablement
+	if cmd.Flags().Changed("trace-all") {
+		projectConfig.Fuzzing.Testing.TraceAll, err = cmd.Flags().GetBool("trace-all")
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }