From d02de26f250d936ba1ef1aa04f53a36039b2d427 Mon Sep 17 00:00:00 2001 From: Andrew Low Date: Thu, 8 Feb 2024 20:20:36 -0500 Subject: [PATCH 1/3] tmp analyzer to re-parse all runtime events --- analyzer/evmabibackfill/evm_abi_backfill.go | 347 ++++++++++---------- analyzer/queries/queries.go | 10 + 2 files changed, 180 insertions(+), 177 deletions(-) diff --git a/analyzer/evmabibackfill/evm_abi_backfill.go b/analyzer/evmabibackfill/evm_abi_backfill.go index 5b89e6089..0b18b26a9 100644 --- a/analyzer/evmabibackfill/evm_abi_backfill.go +++ b/analyzer/evmabibackfill/evm_abi_backfill.go @@ -1,22 +1,20 @@ package evmabibackfill import ( - "bytes" "context" - "encoding/base64" "encoding/json" "fmt" - "strings" + "math/big" - "github.com/ethereum/go-ethereum/accounts/abi" ethCommon "github.com/ethereum/go-ethereum/common" sdkEVM "github.com/oasisprotocol/oasis-sdk/client-sdk/go/modules/evm" "github.com/oasisprotocol/nexus/analyzer" + "github.com/oasisprotocol/nexus/analyzer/evmabi" "github.com/oasisprotocol/nexus/analyzer/item" "github.com/oasisprotocol/nexus/analyzer/queries" "github.com/oasisprotocol/nexus/analyzer/runtime" - "github.com/oasisprotocol/nexus/analyzer/runtime/abiparse" + apiTypes "github.com/oasisprotocol/nexus/api/v1/types" "github.com/oasisprotocol/nexus/common" "github.com/oasisprotocol/nexus/config" "github.com/oasisprotocol/nexus/log" @@ -81,51 +79,14 @@ func NewAnalyzer( ) } -// Transaction data is canonically represented as a byte array. However, -// the transaction body is stored as a JSONB column in postgres, which -// causes the tx body->>data to be returned as a base64-encoded string -// enclosed by escaped double quote characters. -func cleanTxData(raw string) ([]byte, error) { - s := strings.TrimPrefix(strings.TrimSuffix(raw, "\""), "\"") - return base64.StdEncoding.DecodeString(s) -} - func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*abiEncodedItem, error) { // There are two types of data we process using a contract abi: transactions and events. // Within a transaction, we process the call data and the revert reason. Since they are // colocated in the same table we can fetch them using a single query. var items []*abiEncodedItem - txRows, err := p.target.Query(ctx, queries.RuntimeEvmVerifiedContractTxs, p.runtime, limit) - if err != nil { - return nil, fmt.Errorf("querying verified contract txs: %w", err) - } - defer txRows.Close() - for txRows.Next() { - var rawTxData string - var tx abiEncodedTx - var item abiEncodedItem - item.Tx = &tx - if err = txRows.Scan( - &item.ContractAddr, - &item.Abi, - &tx.TxHash, - &rawTxData, - &tx.TxRevertReason, - ); err != nil { - return nil, fmt.Errorf("scanning verified contract tx: %w", err) - } - if tx.TxData, err = cleanTxData(rawTxData); err != nil { - return nil, fmt.Errorf("error decoding tx data from db: %w", err) - } - items = append(items, &item) - } - // Short circuit. - if len(items) == int(limit) { - return items, nil - } - eventRows, err := p.target.Query(ctx, queries.RuntimeEvmVerifiedContractEvents, p.runtime, int(limit)-len(items)) + eventRows, err := p.target.Query(ctx, queries.RuntimeEvmEvents, p.runtime, limit) if err != nil { - return nil, fmt.Errorf("querying verified contract evs: %w", err) + return nil, fmt.Errorf("querying evs: %w", err) } defer eventRows.Close() for eventRows.Next() { @@ -133,8 +94,6 @@ func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*abiEncodedIt var item abiEncodedItem item.Event = &ev if err = eventRows.Scan( - &item.ContractAddr, - &item.Abi, &ev.Round, &ev.TxIndex, &ev.EventBody, @@ -146,104 +105,180 @@ func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*abiEncodedIt return items, nil } -// Transaction revert reasons for failed evm transactions have been encoded -// differently over the course of Oasis history. Older transaction revert -// reasons were returned as one of -// - "reverted: Incorrect premium amount" -// - "reverted: base64(up to 1024 bytes of revert data)" -// -// Note that if the revert reason was longer than 1024 bytes it was truncated. -// Newer transaction revert reasons are returned as -// - "reverted: base64(revert data)" -// -// In all cases the revert reason has the "reverted: " prefix, which we first -// strip. We then attempt to base64-decode the remaining string to recover -// the error message. It should be noted that if the b64 decoding fails, it's -// likely an older error message. -// -// See the docstring of tryParseErrorMessage in analyzer/runtime/extract.go -// for more info. -func cleanTxRevertReason(raw string) ([]byte, error) { - s := strings.TrimPrefix(raw, runtime.TxRevertErrPrefix) - return base64.StdEncoding.DecodeString(s) -} - // Attempts to parse the raw event body into the event name, args, and signature // as defined by the abi of the contract that emitted this event. -func (p *processor) parseEvent(ev *abiEncodedEvent, contractAbi abi.ABI) (*string, []*abiEncodedArg, *ethCommon.Hash) { - abiEvent, abiEventArgs, err := abiparse.ParseEvent(ev.EventBody.Topics, ev.EventBody.Data, &contractAbi) - if err != nil { - p.logger.Warn("error processing event using abi", "err", err) - return nil, nil, nil - } - eventArgs, err := marshalArgs(abiEvent.Inputs, abiEventArgs) - if err != nil { - p.logger.Warn("error processing event args using abi", "err", err) +func (p *processor) parseEvent(ev *abiEncodedEvent) (*string, []*abiEncodedArg, *ethCommon.Hash) { + var evmLogName *string + var evmLogSignature *ethCommon.Hash + var evmLogParams []*abiEncodedArg + if err := runtime.VisitEVMEvent(&ev.EventBody, &runtime.EVMEventHandler{ + ERC20Transfer: func(fromECAddr ethCommon.Address, toECAddr ethCommon.Address, value *big.Int) error { + evmLogName = common.Ptr(apiTypes.Erc20Transfer) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "from", + EvmType: "address", + Value: fromECAddr, + }, + { + Name: "to", + EvmType: "address", + Value: toECAddr, + }, + { + Name: "value", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: value.String(), + }, + } + return nil + }, + ERC20Approval: func(ownerECAddr ethCommon.Address, spenderECAddr ethCommon.Address, value *big.Int) error { + evmLogName = common.Ptr(apiTypes.Erc20Approval) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "owner", + EvmType: "address", + Value: ownerECAddr, + }, + { + Name: "spender", + EvmType: "address", + Value: spenderECAddr, + }, + { + Name: "value", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: value.String(), + }, + } + return nil + }, + ERC721Transfer: func(fromECAddr ethCommon.Address, toECAddr ethCommon.Address, tokenID *big.Int) error { + evmLogName = common.Ptr(evmabi.ERC721.Events["Transfer"].Name) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "from", + EvmType: "address", + Value: fromECAddr, + }, + { + Name: "to", + EvmType: "address", + Value: toECAddr, + }, + { + Name: "tokenID", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: tokenID.String(), + }, + } + return nil + }, + ERC721Approval: func(ownerECAddr ethCommon.Address, approvedECAddr ethCommon.Address, tokenID *big.Int) error { + evmLogName = common.Ptr(evmabi.ERC721.Events["Approval"].Name) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "owner", + EvmType: "address", + Value: ownerECAddr, + }, + { + Name: "approved", + EvmType: "address", + Value: approvedECAddr, + }, + { + Name: "tokenID", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: tokenID.String(), + }, + } + return nil + }, + ERC721ApprovalForAll: func(ownerECAddr ethCommon.Address, operatorECAddr ethCommon.Address, approved bool) error { + evmLogName = common.Ptr(evmabi.ERC721.Events["ApprovalForAll"].Name) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "owner", + EvmType: "address", + Value: ownerECAddr, + }, + { + Name: "operator", + EvmType: "address", + Value: operatorECAddr, + }, + { + Name: "approved", + EvmType: "bool", + Value: approved, + }, + } + return nil + }, + WROSEDeposit: func(ownerECAddr ethCommon.Address, amount *big.Int) error { + evmLogName = common.Ptr(evmabi.WROSE.Events["Deposit"].Name) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "dst", + EvmType: "address", + Value: ownerECAddr, + }, + { + Name: "wad", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: amount.String(), + }, + } + return nil + }, + WROSEWithdrawal: func(ownerECAddr ethCommon.Address, amount *big.Int) error { + evmLogName = common.Ptr(evmabi.WROSE.Events["Withdrawal"].Name) + evmLogSignature = common.Ptr(ethCommon.BytesToHash(ev.EventBody.Topics[0])) + evmLogParams = []*abiEncodedArg{ + { + Name: "src", + EvmType: "address", + Value: ownerECAddr, + }, + { + Name: "wad", + EvmType: "uint256", + // JSON supports encoding big integers, but many clients (javascript, jq, etc.) + // will incorrectly parse them as floats. So we encode uint256 as a string instead. + Value: amount.String(), + }, + } + return nil + }, + }); err != nil { + p.logger.Error("error processing event", "round", ev.Round, "tx_index", ev.TxIndex, "err", err) return nil, nil, nil } - return &abiEvent.Name, eventArgs, &abiEvent.ID -} - -// Attempts to parse the raw evm.Call transaction data into the transaction -// method name and arguments as defined by the abi of the contract that was called. -func (p *processor) parseTxCall(tx *abiEncodedTx, contractAbi abi.ABI) (*string, []*abiEncodedArg) { - method, abiTxArgs, err := abiparse.ParseData(tx.TxData, &contractAbi) - if err != nil { - p.logger.Warn("error processing tx using abi", "err", err) - return nil, nil - } - txArgs, err := marshalArgs(method.Inputs, abiTxArgs) - if err != nil { - p.logger.Warn("error processing tx args using abi", "err", err) - return nil, nil - } - - return &method.RawName, txArgs -} - -// Attempts to parse the transaction revert reason into the error name and args -// as defined by the abi of the contract that was called. -func (p *processor) parseTxErr(tx *abiEncodedTx, contractAbi abi.ABI) (*string, []*abiEncodedArg) { - var abiErrMsg string - var abiErr *abi.Error - var abiErrArgs []interface{} - var errArgs []*abiEncodedArg - if tx.TxRevertReason != nil { - txrr, err := cleanTxRevertReason(*tx.TxRevertReason) - if err != nil { - // This is most likely an older tx with a plaintext revert reason, such - // as "reverted: Ownable: caller is not the owner". In this case, we do - // not parse the error with the abi. - p.logger.Info("encountered likely old-style reverted transaction", "revert reason", tx.TxRevertReason, "tx hash", tx.TxHash, "err", err) - return nil, nil - } - abiErr, abiErrArgs, err = abiparse.ParseError(txrr, &contractAbi) - if err != nil || abiErr == nil { - p.logger.Warn("error processing tx error using abi", "contract address", "err", err) - return nil, nil - } - abiErrMsg = runtime.TxRevertErrPrefix + abiErr.Name + prettyPrintArgs(abiErrArgs) - errArgs, err = marshalArgs(abiErr.Inputs, abiErrArgs) - if err != nil { - p.logger.Warn("error processing tx error args", "err", err) - return nil, nil - } - } - - return &abiErrMsg, errArgs + return evmLogName, evmLogParams, evmLogSignature } func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch, item *abiEncodedItem) error { - // Unmarshal abi - contractAbi, err := abi.JSON(bytes.NewReader(item.Abi)) - if err != nil { - return fmt.Errorf("error unmarshalling abi: %w", err) - } - // Parse data - p.logger.Debug("processing item using abi", "contract_address", item.ContractAddr) if item.Event != nil { - eventName, eventArgs, eventSig := p.parseEvent(item.Event, contractAbi) + eventName, eventArgs, eventSig := p.parseEvent(item.Event) batch.Queue( queries.RuntimeEventEvmParsedFieldsUpdate, p.runtime, @@ -254,53 +289,11 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch, eventArgs, eventSig, ) - } else if item.Tx != nil { - methodName, methodArgs := p.parseTxCall(item.Tx, contractAbi) - errMsg, errArgs := p.parseTxErr(item.Tx, contractAbi) - batch.Queue( - queries.RuntimeTransactionEvmParsedFieldsUpdate, - p.runtime, - item.Tx.TxHash, - methodName, - methodArgs, - errMsg, - errArgs, - ) } return nil } -func marshalArgs(abiArgs abi.Arguments, argVals []interface{}) ([]*abiEncodedArg, error) { - if len(abiArgs) != len(argVals) { - return nil, fmt.Errorf("number of args does not match abi specification") - } - args := []*abiEncodedArg{} - for i, v := range argVals { - args = append(args, &abiEncodedArg{ - Name: abiArgs[i].Name, - EvmType: abiArgs[i].Type.String(), - Value: v, - }) - } - - return args, nil -} - -func prettyPrintArgs(argVals []interface{}) string { - var sb strings.Builder - sb.WriteString("(") - for i, v := range argVals { - if i == len(argVals)-1 { - sb.WriteString(fmt.Sprintf("%v", v)) - } else { - sb.WriteString(fmt.Sprintf("%v,", v)) - } - } - sb.WriteString(")") - return sb.String() -} - func (p *processor) QueueLength(ctx context.Context) (int, error) { var txQueueLength int if err := p.target.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM (%s) subquery", queries.RuntimeEvmVerifiedContractTxs), p.runtime, 1000).Scan(&txQueueLength); err != nil { diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index 2d3f0766f..af16a2436 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -953,4 +953,14 @@ var ( WHERE evs.abi_parsed_at IS NULL LIMIT $2` + + RuntimeEvmEvents = ` + SELECT + round, tx_index, body + FROM chain.runtime_events + WHERE + runtime = $1 AND + type = 'evm.log' AND + abi_parsed_at IS NULL + LIMIT $2` ) From d71f9b870fe5cf74c2895a150580fd5dad6e537e Mon Sep 17 00:00:00 2001 From: Andrew Low Date: Thu, 8 Feb 2024 20:42:21 -0500 Subject: [PATCH 2/3] always overwrite --- analyzer/queries/queries.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index af16a2436..80aafc0c5 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -477,9 +477,9 @@ var ( RuntimeEventEvmParsedFieldsUpdate = ` UPDATE chain.runtime_events SET - evm_log_name = COALESCE($5, evm_log_name), - evm_log_params = COALESCE($6, evm_log_params), - evm_log_signature = COALESCE($7, evm_log_signature), + evm_log_name = $5, + evm_log_params = $6, + evm_log_signature = $7, abi_parsed_at = CURRENT_TIMESTAMP WHERE runtime = $1 AND From ad682eccd877435afc29ba01f0cae6a24fc1ccf5 Mon Sep 17 00:00:00 2001 From: Andrew Low Date: Mon, 12 Feb 2024 17:33:24 -0500 Subject: [PATCH 3/3] optimize query --- analyzer/evmabibackfill/evm_abi_backfill.go | 18 +++++++++++------- analyzer/queries/queries.go | 4 +++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/analyzer/evmabibackfill/evm_abi_backfill.go b/analyzer/evmabibackfill/evm_abi_backfill.go index 0b18b26a9..61e415d90 100644 --- a/analyzer/evmabibackfill/evm_abi_backfill.go +++ b/analyzer/evmabibackfill/evm_abi_backfill.go @@ -29,6 +29,8 @@ type processor struct { runtime common.Runtime target storage.TargetStorage logger *log.Logger + + currRound uint64 } var _ item.ItemProcessor[*abiEncodedItem] = (*processor)(nil) @@ -69,6 +71,7 @@ func NewAnalyzer( runtime, target, logger, + 0, } return item.NewAnalyzer[*abiEncodedItem]( evmAbiAnalyzerPrefix+string(runtime), @@ -84,11 +87,12 @@ func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*abiEncodedIt // Within a transaction, we process the call data and the revert reason. Since they are // colocated in the same table we can fetch them using a single query. var items []*abiEncodedItem - eventRows, err := p.target.Query(ctx, queries.RuntimeEvmEvents, p.runtime, limit) + eventRows, err := p.target.Query(ctx, queries.RuntimeEvmEvents, p.runtime, p.currRound, limit) if err != nil { return nil, fmt.Errorf("querying evs: %w", err) } defer eventRows.Close() + firstEv := true for eventRows.Next() { var ev abiEncodedEvent var item abiEncodedItem @@ -100,6 +104,10 @@ func (p *processor) GetItems(ctx context.Context, limit uint64) ([]*abiEncodedIt ); err != nil { return nil, fmt.Errorf("scanning verified contract event: %w", err) } + if firstEv { + p.currRound = ev.Round - 100 // margin of safety + firstEv = false + } items = append(items, &item) } return items, nil @@ -295,14 +303,10 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch, } func (p *processor) QueueLength(ctx context.Context) (int, error) { - var txQueueLength int - if err := p.target.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM (%s) subquery", queries.RuntimeEvmVerifiedContractTxs), p.runtime, 1000).Scan(&txQueueLength); err != nil { - return 0, fmt.Errorf("querying number of verified abi txs: %w", err) - } var evQueueLength int // We limit the event count for performance reasons since the query requires a join of the transactions and events tables. - if err := p.target.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM (%s) subquery", queries.RuntimeEvmVerifiedContractEvents), p.runtime, 1000).Scan(&evQueueLength); err != nil { + if err := p.target.QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*) FROM (%s) subquery", queries.RuntimeEvmEvents), p.runtime, p.currRound, 1000).Scan(&evQueueLength); err != nil { return 0, fmt.Errorf("querying number of verified abi events: %w", err) } - return txQueueLength + evQueueLength, nil + return evQueueLength, nil } diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index 80aafc0c5..9499afc6b 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -960,7 +960,9 @@ var ( FROM chain.runtime_events WHERE runtime = $1 AND + round >= $2 AND type = 'evm.log' AND abi_parsed_at IS NULL - LIMIT $2` + ORDER BY round ASC + LIMIT $3` )