#755 indexer catch inner instruction call (#756)

* Add types

* In Indexer catch neon trnasactions called from other solana programs

* simplify logic

* check result separately

* check for transaction before decoding result and event

* fix typo

.buildkite/steps/build-image.sh

@@ -11,6 +11,8 @@ docker pull neonlabsorg/solana:${SOLANA_REVISION}
 
 # Refreshing neonlabsorg/evm_loader:latest image is required to run .buildkite/steps/build-image.sh locally
 docker pull neonlabsorg/evm_loader:${NEON_EVM_COMMIT}
+# Refreshing neonlabsorg/evm_loader:ci-proxy-caller-program image is required to run .buildkite/steps/build-image.sh locally
+docker pull neonlabsorg/evm_loader:ci-proxy-caller-program
 
 docker build -t neonlabsorg/proxy:${REVISION} \
     --build-arg SOLANA_REVISION=${SOLANA_REVISION} \

.buildkite/steps/deploy-test.sh

@@ -43,6 +43,7 @@ function cleanup_docker {
 
     if docker logs solana >solana.log 2>&1; then echo "solana logs saved"; fi
     if docker logs evm_loader >evm_loader.log 2>&1; then echo "evm_loader logs saved"; fi
+    if docker logs proxy_program_loader >proxy_program_loader.log 2>&1; then echo "proxy_program_loader logs saved"; fi
     if docker logs dbcreation >dbcreation.log 2>&1; then echo "dbcreation logs saved"; fi
     if docker logs faucet >faucet.log 2>&1; then echo "faucet logs saved"; fi
     if docker logs airdropper >airdropper.log 2>&1; then echo "airdropper logs saved"; fi

Dockerfile

@@ -4,6 +4,7 @@ ARG NEON_EVM_COMMIT=v0.7.7
 FROM neonlabsorg/solana:${SOLANA_REVISION} AS cli
 
 FROM neonlabsorg/evm_loader:${NEON_EVM_COMMIT} AS spl
+FROM neonlabsorg/evm_loader:ci-proxy-caller-program AS proxy_program
 
 FROM ubuntu:20.04
 
@@ -39,6 +40,7 @@ COPY --from=spl /opt/solana_utils.py \
                 /opt/eth_tx_utils.py \
                 /spl/bin/
 COPY --from=spl /opt/neon-cli /spl/bin/emulator
+COPY --from=proxy_program /opt/proxy_program-keypair.json /spl/bin/
 
 COPY proxy/operator-keypairs/id.json /root/.config/solana/
 

proxy/common_neon/utils/utils.py

@@ -7,6 +7,8 @@
 from enum import Enum
 from eth_utils import big_endian_to_int
 
+from proxy.indexer.utils import SolanaIxSignInfo
+
 #TODO: move it out from here
 from ...environment import EVM_LOADER_ID, LOG_FULL_OBJECT_INFO
 
@@ -81,11 +83,8 @@ def is_empty(self) -> bool:
 
 
 class NeonTxResultInfo:
-    def __init__(self, neon_sign='', tx=None, ix_idx=-1):
-        if not isinstance(tx, dict):
-            self._set_defaults()
-        else:
-            self.decode(neon_sign, tx, ix_idx)
+    def __init__(self):
+        self._set_defaults()
 
     def __str__(self):
         return str_fmt_object(self)
@@ -130,6 +129,12 @@ def _decode_event(self, neon_sign, log, tx_idx):
         }
         self.logs.append(rec)
 
+    def append_record(self, rec):
+        log_idx = len(self.logs)
+        rec['transactionLogIndex'] = hex(log_idx)
+        rec['logIndex'] = hex(log_idx)
+        self.logs.append(rec)
+
     def _decode_return(self, log: bytes, ix_idx: int, tx: {}):
         self.status = '0x1' if log[1] < 0xd0 else '0x0'
         self.gas_used = hex(int.from_bytes(log[2:10], 'little'))
@@ -138,6 +143,14 @@ def _decode_return(self, log: bytes, ix_idx: int, tx: {}):
         self.slot = tx['slot']
         self.idx = ix_idx
 
+    def set_result(self, sign: SolanaIxSignInfo, status, gas_used, return_value):
+        self.status = status
+        self.gas_used = gas_used
+        self.return_value = return_value
+        self.sol_sign = sign.sign
+        self.slot = sign.slot
+        self.idx = sign.idx
+
     def fill_block_info(self, block: SolanaBlockInfo):
         self.slot = block.slot
         self.block_hash = block.hash

proxy/deploy-test.sh

@@ -20,6 +20,7 @@ solana balance
 set ${TESTNAME:=*}
 
 export ETH_TOKEN_MINT=$NEON_TOKEN_MINT
+export TEST_PROGRAM=$(solana address -k /spl/bin/proxy_program-keypair.json)
 
 # python3 -m unittest discover -v -p "test_${TESTNAME}.py"
 find . -name "test_${TESTNAME}.py" -printf "%f\n" | sort | parallel --halt now,fail=1 --jobs 4 python3 -m unittest discover -v -p {}

proxy/docker-compose-test.yml

@@ -36,6 +36,20 @@ services:
         condition: service_healthy
     command: bash -c "create-test-accounts.sh 1 && deploy-evm.sh"
 
+  proxy_program_loader:
+    container_name: proxy_program_loader
+    image: neonlabsorg/evm_loader:ci-proxy-caller-program
+    environment:
+      - SOLANA_URL=http://solana:8899
+    networks:
+      - net
+    depends_on:
+      solana:
+        condition: service_healthy
+      evm_loader:
+        condition: service_completed_successfully
+    command: bash -c "deploy-proxy_program.sh"
+
   postgres:
     container_name: postgres
     image: postgres:14.0

proxy/indexer/indexer.py

@@ -1,5 +1,5 @@
 import copy
-from typing import Iterator, Optional, List
+from typing import Iterator, List, Optional, Dict
 
 import base58
 import time
@@ -25,7 +25,7 @@
 
 @logged_group("neon.Indexer")
 class SolanaIxInfo:
-    def __init__(self, sign: str, slot: int, tx: {}):
+    def __init__(self, sign: str, slot: int, tx: Dict):
         self.sign = SolanaIxSignInfo(sign=sign, slot=slot, idx=-1)
         self.cost_info = CostInfo(sign, tx, EVM_LOADER_ID)
         self.tx = tx
@@ -38,6 +38,7 @@ def __str__(self):
 
     def _set_defaults(self):
         self.ix = {}
+        self.neon_obj = None
         self.evm_ix = 0xFF
         self.ix_data = None
 
@@ -52,6 +53,17 @@ def _decode_ixdata(self) -> bool:
             self.ix_data = None
         return False
 
+    def _get_neon_instruction(self) -> bool:
+        accounts = self._msg['accountKeys']
+        if 'programIdIndex' not in self.ix:
+            self.debug(f'{self} error: fail to get program id')
+            return False
+        if accounts[self.ix['programIdIndex']] != EVM_LOADER_ID:
+            return False
+        if not self._decode_ixdata():
+            return False
+        return True
+
     def clear(self):
         self._set_defaults()
 
@@ -60,23 +72,23 @@ def iter_ixs(self):
             return
 
         self._set_defaults()
-        accounts = self._msg['accountKeys']
         tx_ixs = enumerate(self._msg['instructions'])
 
         evm_ix_idx = -1
         for ix_idx, self.ix in tx_ixs:
             # Make a new object to keep values in existing
             self.sign = SolanaIxSignInfo(sign=self.sign.sign, slot=self.sign.slot, idx=ix_idx)
-            if 'programIdIndex' not in self.ix:
-                self.debug(f'{self} error: fail to get program id')
-                continue
-            if accounts[self.ix['programIdIndex']] != EVM_LOADER_ID:
-                continue
-            if not self._decode_ixdata():
-                continue
 
-            evm_ix_idx += 1
-            yield evm_ix_idx
+            if self._get_neon_instruction():
+                evm_ix_idx += 1
+                yield evm_ix_idx
+
+            for inner_tx in self.tx['meta']['innerInstructions']:
+                if inner_tx['index'] == ix_idx:
+                    for self.ix in inner_tx['instructions']:
+                        if self._get_neon_instruction():
+                            evm_ix_idx += 1
+                            yield evm_ix_idx
 
         self._set_defaults()
 
@@ -94,7 +106,7 @@ def get_account(self, idx: int) -> str:
             return msg_keys[ix_accounts[idx]]
         return ''
 
-    def get_account_list(self, start: int) -> [str]:
+    def get_account_list(self, start: int) -> List[str]:
         assert self._is_valid
 
         msg_keys = self._msg['accountKeys']
@@ -450,10 +462,7 @@ def _decode_tx(self, tx: NeonTxResult):
         The transaction can already have results, because parser waits all ixs in the slot, because
         the parsing order can be other than the execution order
         """
-        if not tx.neon_res.is_valid():
-            tx.neon_res.decode(tx.neon_tx.sign, self.ix.tx, self.ix.sign.idx)
-            if tx.neon_res.is_valid():
-                return self._decoding_done(tx, 'found Neon results')
+        self.ix.neon_obj = tx
         return self._decoding_success(tx, 'mark ix used')
 
     def _init_tx_from_holder(self,
@@ -653,12 +662,65 @@ def execute(self) -> bool:
         if neon_tx.error:
             return self._decoding_skip(f'Neon tx rlp error "{neon_tx.error}"')
 
-        neon_res = NeonTxResultInfo(neon_tx.sign, self.ix.tx, self.ix.sign.idx)
         tx = NeonTxResult('')
         tx.neon_tx = neon_tx
-        tx.neon_res = neon_res
 
-        return self._decoding_done(tx, 'call success')
+        return self._decode_tx(tx)
+
+
+class OnResultIxDecoder(DummyIxDecoder):
+    def __init__(self, state: ReceiptsParserState):
+        DummyIxDecoder.__init__(self, 'OnResult', state)
+
+    def execute(self) -> bool:
+        self._decoding_start()
+
+        if self.ix.neon_obj is None:
+            return self._decoding_skip('no transaction to add result')
+
+        log = self.ix.ix_data
+
+        status = '0x1' if log[1] < 0xd0 else '0x0'
+        gas_used = hex(int.from_bytes(log[2:10], 'little'))
+        return_value = log[10:].hex()
+
+        self.ix.neon_obj.neon_res.set_result(self.ix.sign, status, gas_used, return_value)
+        return self._decoding_done(self.ix.neon_obj, 'found Neon results')
+
+
+class OnEventIxDecoder(DummyIxDecoder):
+    def __init__(self, state: ReceiptsParserState):
+        DummyIxDecoder.__init__(self, 'OnEvent', state)
+
+    def execute(self) -> bool:
+        self._decoding_start()
+
+        if self.ix.neon_obj is None:
+            return self._decoding_skip('no transaction to add events')
+
+        log = self.ix.ix_data
+
+        address = log[1:21]
+        count_topics = int().from_bytes(log[21:29], 'little')
+        topics = []
+        pos = 29
+        for _ in range(count_topics):
+            topic_bin = log[pos:pos + 32]
+            topics.append('0x' + topic_bin.hex())
+            pos += 32
+        data = log[pos:]
+        rec = {
+            'address': '0x' + address.hex(),
+            'topics': topics,
+            'data': '0x' + data.hex(),
+            'transactionIndex': hex(self.ix.sign.idx),
+            'transactionHash': self.ix.neon_obj.neon_tx.sign,
+            # 'blockNumber': block_number, # set when transaction found
+            # 'blockHash': block_hash # set when transaction found
+        }
+
+        self.ix.neon_obj.neon_res.append_record(rec)
+        return True
 
 
 class PartialCallIxDecoder(DummyIxDecoder):
@@ -877,8 +939,8 @@ def __init__(self, solana_url, indexer_user: IIndexerUser):
             0x03: DummyIxDecoder('Call', self.state),
             0x04: DummyIxDecoder('CreateAccountWithSeed', self.state),
             0x05: CallFromRawIxDecoder(self.state),
-            0x06: DummyIxDecoder('OnEvent', self.state),
-            0x07: DummyIxDecoder('OnResult', self.state),
+            0x06: OnResultIxDecoder(self.state),
+            0x07: OnEventIxDecoder(self.state),
             0x09: PartialCallIxDecoder(self.state),
             0x0a: ContinueIxDecoder(self.state),
             0x0b: ExecuteTrxFromAccountIxDecoder(self.state),

proxy/indexer/indexer_base.py

@@ -163,7 +163,7 @@ def gather_unknown_transactions(self):
             latest_params={"maximum_tx": maximum_tx, "maximum_slot": maximum_slot}
         )
 
-    def _get_signatures(self, before: Optional[str], limit: int) -> []:
+    def _get_signatures(self, before: Optional[str], limit: int) -> List:
         response = self.solana.get_signatures_for_address(before, limit, FINALIZED)
         error = response.get('error')
         result = response.get('result', [])
@@ -208,14 +208,7 @@ def _get_tx_receipts(self, full_list: List[Tuple[str, int, int]]) -> None:
 
     def _add_tx(self, sol_sign, tx, slot, tx_idx):
         if tx is not None:
-            add = False
-            msg = tx['transaction']['message']
-            for instruction in msg['instructions']:
-                if msg["accountKeys"][instruction["programIdIndex"]] == EVM_LOADER_ID:
-                    add = True
-            if add:
-                self.debug(f'{(slot, tx_idx, sol_sign)}')
-                self.transaction_receipts.add_tx(slot, tx_idx, sol_sign, tx)
+            self.debug(f'{(slot, tx_idx, sol_sign)}')
+            self.transaction_receipts.add_tx(slot, tx_idx, sol_sign, tx)
         else:
             self.debug(f"trx is None {sol_sign}")
-