Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Execution traces for call sequences failing tests #105

Merged
merged 31 commits into from
Mar 22, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
f65bcb0
useless commit
anishnaik Mar 8, 2023
f046909
hacky way to get revert reason
anishnaik Mar 8, 2023
3960df8
Introduction of execution trace attaching and displaying for failed t…
Xenomega Mar 13, 2023
ac9b6be
* Added support for assert/reverts in ExecutionTrace
Xenomega Mar 13, 2023
7134493
Merge remote-tracking branch 'origin/master' into temp/event-logging
Xenomega Mar 13, 2023
600193e
Added event support to execution tracer and traces
Xenomega Mar 13, 2023
c59cd55
Removed old version of event emission test
Xenomega Mar 13, 2023
56d0f1d
Removed old call sequence event code
Xenomega Mar 13, 2023
b0a02a7
Improve indentation of execution traces for better readability
Xenomega Mar 13, 2023
b32085d
Fix lint issue
Xenomega Mar 13, 2023
30dfa43
Return data unpacking fix
Xenomega Mar 13, 2023
d904bc2
* Provided a wrapper for ExecuteCallSequenceIteratively, to simply re…
Xenomega Mar 15, 2023
7663797
Updated execution tracer to better record runtime bytecodes for "to" …
Xenomega Mar 15, 2023
863ed0f
Added self destruct test
Xenomega Mar 15, 2023
fc95b94
add traceAll config option
anishnaik Mar 15, 2023
a0c9e5f
updated formatting for call frames that do not execute code
anishnaik Mar 15, 2023
f26f976
fix bug in attachExecutionTraces
anishnaik Mar 15, 2023
45ecd9e
comment update
anishnaik Mar 15, 2023
4951cd6
lint updates
anishnaik Mar 15, 2023
8209b6a
another formatting update
anishnaik Mar 16, 2023
f7739b5
remove ChildCallFrames parameter and add as getter
anishnaik Mar 16, 2023
9b3e407
First pass at cheat code contract support in execution traces
Xenomega Mar 18, 2023
555f6c6
Merge branch 'master' into dev/execution-tracing
Xenomega Mar 18, 2023
77b2760
Added support for custom solidity errors (TODO: cleanup of execution …
Xenomega Mar 19, 2023
2e9044c
Cleaned up ABI error and event helpers by splitting them off to a com…
Xenomega Mar 19, 2023
704bcf2
* More cleanup of execution tracing code
Xenomega Mar 19, 2023
10c291a
Added a ConstructorArgsData field to execution tracing's CallFrame, t…
Xenomega Mar 19, 2023
d593592
Fix lint issue
Xenomega Mar 19, 2023
1365540
Add a test case for call argument resolution
Xenomega Mar 19, 2023
837b88b
Added support for proxy calls to execution tracer + tests
Xenomega Mar 21, 2023
56caf88
Updated doc comments
Xenomega Mar 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions chain/test_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -422,19 +422,32 @@ func (t *TestChain) StateFromRoot(root common.Hash) (*state.StateDB, error) {
return stateDB, nil
}

// StateAfterBlockNumber obtains the Ethereum world state after processing all transactions in the provided block
// number. Returns the state, or an error if one occurs.
func (t *TestChain) StateAfterBlockNumber(blockNumber uint64) (*state.StateDB, error) {
// StateRootAfterBlockNumber obtains the Ethereum world state root hash after processing all transactions in the
// provided block number. Returns the state, or an error if one occurs.
func (t *TestChain) StateRootAfterBlockNumber(blockNumber uint64) (common.Hash, error) {
// If our block number references something too new, return an error
if blockNumber > t.HeadBlockNumber() {
return nil, fmt.Errorf("could not obtain post-state for block number %d because it exceeds the current head block number %d", blockNumber, t.HeadBlockNumber())
return common.Hash{}, fmt.Errorf("could not obtain post-state for block number %d because it exceeds the current head block number %d", blockNumber, t.HeadBlockNumber())
}

// Obtain our closest internally committed block
_, closestBlock := t.fetchClosestInternalBlock(blockNumber)

// Return our state root hash
return closestBlock.Header.Root, nil
}

// StateAfterBlockNumber obtains the Ethereum world state after processing all transactions in the provided block
// number. Returns the state, or an error if one occurs.
func (t *TestChain) StateAfterBlockNumber(blockNumber uint64) (*state.StateDB, error) {
// Obtain our block's post-execution state root hash
root, err := t.StateRootAfterBlockNumber(blockNumber)
if err != nil {
return nil, err
}

// Load our state from the database
return t.StateFromRoot(closestBlock.Header.Root)
return t.StateFromRoot(root)
}

// RevertToBlockNumber sets the head of the chain to the block specified by the provided block number and reloads
Expand Down
51 changes: 30 additions & 21 deletions fuzzing/calls/call_sequence.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,31 +21,16 @@ import (
type CallSequence []*CallSequenceElement

// AttachExecutionTraces takes a given chain which executed the call sequence, and a list of contract definitions,
// and it replays each call of the sequence with an execution tracer attached to
// and it replays each call of the sequence with an execution tracer attached to it, it then sets each
// CallSequenceElement.ExecutionTrace to the resulting trace.
// Returns an error if one occurred.
func (cs CallSequence) AttachExecutionTraces(chain *chain.TestChain, contractDefinitions fuzzingTypes.Contracts) error {
// For each call sequence element, attach an execution trace.
for _, cse := range cs {
// Verify the element has been executed before.
if cse.ChainReference == nil {
return fmt.Errorf("failed to resolve execution trace as the chain reference is nil, indicating the call sequence element has never been executed")
}

// Obtain the state prior to executing this transaction.
state, err := chain.StateFromRoot(cse.ChainReference.MessageResults().PreStateRoot)
err := cse.AttachExecutionTrace(chain, contractDefinitions)
if err != nil {
return fmt.Errorf("failed to resolve execution trace due to error loading root hash from database: %v", err)
return nil
}

// Create an execution tracer
executionTracer := executiontracer.NewExecutionTracer(contractDefinitions)

// Perform a call with our execution tracer.
_, err = chain.CallContract(cse.Call, state, executionTracer)
if err != nil {
return fmt.Errorf("failed to resolve execution trace due to error replaying the call: %v", err)
}

// Set the resulting trace
cse.ExecutionTrace = executionTracer.Trace()
}
return nil
}
Expand Down Expand Up @@ -245,6 +230,30 @@ func (cse *CallSequenceElement) String() string {
)
}

// AttachExecutionTrace takes a given chain which executed the call sequence element, and a list of contract definitions,
// and it replays the call with an execution tracer attached to it, it then sets CallSequenceElement.ExecutionTrace to
// the resulting trace.
// Returns an error if one occurred.
anishnaik marked this conversation as resolved.
Show resolved Hide resolved
func (cse *CallSequenceElement) AttachExecutionTrace(chain *chain.TestChain, contractDefinitions fuzzingTypes.Contracts) error {
// Verify the element has been executed before.
if cse.ChainReference == nil {
return fmt.Errorf("failed to resolve execution trace as the chain reference is nil, indicating the call sequence element has never been executed")
}

// Obtain the state prior to executing this transaction.
state, err := chain.StateFromRoot(cse.ChainReference.MessageResults().PreStateRoot)
if err != nil {
return fmt.Errorf("failed to resolve execution trace due to error loading root hash from database: %v", err)
}

// Perform our call with the given trace
_, cse.ExecutionTrace, err = executiontracer.CallWithExecutionTrace(chain, contractDefinitions, cse.Call, state)
if err != nil {
return fmt.Errorf("failed to resolve execution trace due to error replaying the call: %v", err)
}
return nil
}

// CallSequenceElementChainReference references the inclusion of a CallSequenceElement's underlying call being
// included in a block as a transaction.
type CallSequenceElementChainReference struct {
Expand Down
16 changes: 16 additions & 0 deletions fuzzing/calls/call_sequence_execution.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,19 @@ func ExecuteCallSequenceIteratively(chain *chain.TestChain, fetchElementFunc Exe
}
return callSequenceExecuted, nil
}

// ExecuteCallSequence executes a provided CallSequence on the provided chain.
// It returns the slice of the call sequence which was tested, and an error if one occurred.
// If no error occurred, it can be expected that the returned call sequence contains all elements originally provided.
func ExecuteCallSequence(chain *chain.TestChain, callSequence CallSequence) (CallSequence, error) {
// Execute our sequence with a simple fetch operation provided to obtain each element.
fetchElementFunc := func(currentIndex int) (*CallSequenceElement, error) {
if currentIndex < len(callSequence) {
return callSequence[currentIndex], nil
}
return nil, nil
}

// Execute our provided call sequence iteratively.
return ExecuteCallSequenceIteratively(chain, fetchElementFunc, nil)
}
44 changes: 17 additions & 27 deletions fuzzing/executiontracer/execution_tracer.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,33 @@ package executiontracer

import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/trailofbits/medusa/chain/types"
"github.com/trailofbits/medusa/chain"
"github.com/trailofbits/medusa/fuzzing/contracts"
"golang.org/x/exp/slices"
"math/big"
)

// executionTracerResultsKey describes the key to use when storing tracer results in call message results, or when
// querying them.
const executionTracerResultsKey = "ExecutionTracerResults"

// GetExecutionTracerResults obtains an ExecutionTrace stored by a ExecutionTracer from message results. This is nil if
// no ExecutionTrace was recorded by a tracer (e.g. ExecutionTracer was not attached during this message execution).
func GetExecutionTracerResults(messageResults *types.MessageResults) *ExecutionTrace {
// Try to obtain the results the tracer should've stored.
if genericResult, ok := messageResults.AdditionalResults[executionTracerResultsKey]; ok {
if castedResult, ok := genericResult.(*ExecutionTrace); ok {
return castedResult
}
// CallWithExecutionTrace obtains an execution trace for a given call, on the provided chain, using the state
// provided. If a nil state is provided, the current chain state will be used.
// Returns the ExecutionTrace for the call or an error if one occurs.
func CallWithExecutionTrace(chain *chain.TestChain, contractDefinitions contracts.Contracts, msg core.Message, state *state.StateDB) (*core.ExecutionResult, *ExecutionTrace, error) {
// Create an execution tracer
executionTracer := NewExecutionTracer(contractDefinitions)

// Call the contract on our chain with the provided state.
executionResult, err := chain.CallContract(msg, state, executionTracer)
if err != nil {
return nil, nil, err
}

// If we could not obtain them, return nil.
return nil
}
// Obtain our trace
trace := executionTracer.Trace()

// RemoveExecutionTracerResults removes an ExecutionTrace stored by an ExecutionTracer, from message results.
func RemoveExecutionTracerResults(messageResults *types.MessageResults) {
delete(messageResults.AdditionalResults, executionTracerResultsKey)
// Return the trace
return executionResult, trace, nil
}

// ExecutionTracer records execution information into an ExecutionTrace, containing information about each call
Expand Down Expand Up @@ -229,11 +227,3 @@ func (t *ExecutionTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64
func (t *ExecutionTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) {

}

// CaptureTxEndSetAdditionalResults can be used to set additional results captured from execution tracing. If this
// tracer is used during transaction execution (block creation), the results can later be queried from the block.
// This method will only be called on the added tracer if it implements the extended TestChainTracer interface.
func (t *ExecutionTracer) CaptureTxEndSetAdditionalResults(results *types.MessageResults) {
// Store our tracer results.
results.AdditionalResults[executionTracerResultsKey] = t.trace
}
15 changes: 11 additions & 4 deletions fuzzing/fuzzer_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -398,10 +398,17 @@ func (fw *FuzzerWorker) shrinkCallSequence(callSequence calls.CallSequence, shri
}
}

// Shrinking is complete, next we attach execution traces by re-running the sequence via calls at every previous
// state before the call, but with an execution tracer attached.
// TODO: Only perform this action if the config specified some `--verbose` flag
if true {
// We have a finalized call sequence, re-execute it, so our current chain state is representative of post-execution.
_, err = calls.ExecuteCallSequence(fw.chain, optimizedSequence)
if err != nil {
return nil, err
}

// Shrinking is complete. If our config specified we want all result sequences to have execution traces attached,
// attach them now to each element in the sequence. Otherwise, call sequences will only have traces that the
// test providers choose to attach themselves.
verbose := false
if verbose {
err = optimizedSequence.AttachExecutionTraces(fw.chain, fw.fuzzer.contractDefinitions)
if err != nil {
return nil, err
Expand Down
41 changes: 20 additions & 21 deletions fuzzing/test_case_assertion_provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"sync"

"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/core"
"github.com/trailofbits/medusa/fuzzing/contracts"
)

Expand Down Expand Up @@ -51,23 +50,9 @@ func (t *AssertionTestCaseProvider) isTestableMethod(method abi.Method) bool {
return !method.IsConstant() || t.fuzzer.config.Fuzzing.Testing.AssertionTesting.TestViewMethods
}

// isAssertionVMError indicates whether a provided execution returned from the EVM is due to a failed assert(...)
// statement.
func (t *AssertionTestCaseProvider) isAssertionVMError(result *core.ExecutionResult) bool {
// Try to unpack our error and return data for a panic code and verify it matches the "assert failed" panic code.
// Solidity >0.8.0 introduced asserts failing as reverts but with special return data. But we indicate we also
// want to be backwards compatible with older Solidity which simply hit an invalid opcode and did not actually
// have a panic code.
panicCode := types.GetSolidityPanicCode(result.Err, result.ReturnData, true)
if panicCode != nil && panicCode.Uint64() == types.PanicCodeAssertFailed {
return true
}
return false
}

// checkAssertionFailures checks the results of the last call for assertion failures.
// Returns the method ID, a boolean indicating if an assertion test failed, or an error if one occurs.
func (t *AssertionTestCaseProvider) checkAssertionFailures(worker *FuzzerWorker, callSequence calls.CallSequence) (*contracts.ContractMethodID, bool, error) {
func (t *AssertionTestCaseProvider) checkAssertionFailures(callSequence calls.CallSequence) (*contracts.ContractMethodID, bool, error) {
// If we have an empty call sequence, we cannot have an assertion failure
if len(callSequence) == 0 {
return nil, false, nil
Expand All @@ -82,9 +67,15 @@ func (t *AssertionTestCaseProvider) checkAssertionFailures(worker *FuzzerWorker,
methodId := contracts.GetContractMethodID(lastCall.Contract, lastCallMethod)

// Check if we encountered an assertion error.
encounteredAssertionVMError := t.isAssertionVMError(lastCall.ChainReference.MessageResults().ExecutionResult)
// Try to unpack our error and return data for a panic code and verify it matches the "assert failed" panic code.
// Solidity >0.8.0 introduced asserts failing as reverts but with special return data. But we indicate we also
// want to be backwards compatible with older Solidity which simply hit an invalid opcode and did not actually
// have a panic code.
lastExecutionResult := lastCall.ChainReference.MessageResults().ExecutionResult
panicCode := types.GetSolidityPanicCode(lastExecutionResult.Err, lastExecutionResult.ReturnData, true)
encounteredAssertionFailure := panicCode != nil && panicCode.Uint64() == types.PanicCodeAssertFailed

return &methodId, encounteredAssertionVMError, nil
return &methodId, encounteredAssertionFailure, nil
}

// onFuzzerStarting is the event handler triggered when the Fuzzer is starting a fuzzing campaign. It creates test cases
Expand Down Expand Up @@ -184,7 +175,7 @@ func (t *AssertionTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorke
shrinkRequests := make([]ShrinkCallSequenceRequest, 0)

// Obtain the method ID for the last call and check if it encountered assertion failures.
methodId, testFailed, err := t.checkAssertionFailures(worker, callSequence)
methodId, testFailed, err := t.checkAssertionFailures(callSequence)
if err != nil {
return nil, err
}
Expand All @@ -211,7 +202,7 @@ func (t *AssertionTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorke
shrinkRequest := ShrinkCallSequenceRequest{
VerifierFunction: func(worker *FuzzerWorker, shrunkenCallSequence calls.CallSequence) (bool, error) {
// Obtain the method ID for the last call and check if it encountered assertion failures.
shrunkSeqMethodId, shrunkSeqTestFailed, err := t.checkAssertionFailures(worker, shrunkenCallSequence)
shrunkSeqMethodId, shrunkSeqTestFailed, err := t.checkAssertionFailures(shrunkenCallSequence)
if err != nil {
return false, err
}
Expand All @@ -220,7 +211,15 @@ func (t *AssertionTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorke
return shrunkSeqTestFailed && *methodId == *shrunkSeqMethodId, nil
},
FinishedCallback: func(worker *FuzzerWorker, shrunkenCallSequence calls.CallSequence) error {
// When we're finished shrinking, update our test state and report it finalized.
// When we're finished shrinking, attach an execution trace to the last call
if len(shrunkenCallSequence) > 0 {
err = shrunkenCallSequence[len(shrunkenCallSequence)-1].AttachExecutionTrace(worker.chain, worker.fuzzer.contractDefinitions)
if err != nil {
return err
}
}

// Update our test state and report it finalized.
testCase.status = TestCaseStatusFailed
testCase.callSequence = &shrunkenCallSequence
worker.Fuzzer().ReportTestCaseFinished(testCase)
Expand Down
16 changes: 11 additions & 5 deletions fuzzing/test_case_property.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,17 @@ import (
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/trailofbits/medusa/fuzzing/calls"
fuzzerTypes "github.com/trailofbits/medusa/fuzzing/contracts"
"github.com/trailofbits/medusa/fuzzing/executiontracer"
"strings"
)

// PropertyTestCase describes a test being run by a PropertyTestCaseProvider.
type PropertyTestCase struct {
status TestCaseStatus
targetContract *fuzzerTypes.Contract
targetMethod abi.Method
callSequence *calls.CallSequence
status TestCaseStatus
targetContract *fuzzerTypes.Contract
targetMethod abi.Method
callSequence *calls.CallSequence
propertyTestTrace *executiontracer.ExecutionTrace
}

// Status describes the TestCaseStatus used to define the current state of the test.
Expand All @@ -37,10 +39,14 @@ func (t *PropertyTestCase) Message() string {
// If the test failed, return a failure message.
if t.Status() == TestCaseStatusFailed {
return fmt.Sprintf(
"Test \"%s.%s\" failed after the following call sequence:\n%s",
"Test \"%s.%s\" failed after the following call sequence:\n%s\n"+
"Property test \"%s.%s\" execution:\n%s",
t.targetContract.Name(),
t.targetMethod.Sig,
t.CallSequence().String(),
t.targetContract.Name(),
t.targetMethod.Sig,
t.propertyTestTrace.String(),
)
}
return ""
Expand Down
Loading