#447 save instruction statistics (#573)

* #447 save instruction statistics

* #447 add token info

* #447 add table for tx costs

* #447 fix

* #447 remove obsolete fields

* #447 fix merge

* #447 add heap info and move step info  to different table

* #447 fix

* #447 fix PR issues

* fix PR issues

* Use re for getting bpf and heap info

proxy/db/scheme.sql

@@ -21,13 +21,14 @@
     );
 
     CREATE TABLE IF NOT EXISTS neon_accounts (
-        neon_account CHAR(42),
-        pda_account VARCHAR(50),
-        code_account VARCHAR(50),
+        neon_address CHAR(42),
+        pda_address VARCHAR(50),
+        code_address VARCHAR(50),
         slot BIGINT,
         code TEXT,
+        sol_sign CHAR(88),
 
-        UNIQUE(pda_account, code_account)
+        UNIQUE(pda_address, code_address)
     );
 
     CREATE TABLE IF NOT EXISTS failed_airdrop_attempts (
@@ -79,6 +80,7 @@
         neon_sign CHAR(66),
         slot BIGINT,
         idx INT,
+        neon_steps INT,
 
         UNIQUE(sol_sign, neon_sign, idx),
         UNIQUE(neon_sign, sol_sign, idx)
@@ -118,6 +120,17 @@
 
     ALTER TABLE neon_transactions ADD COLUMN IF NOT EXISTS tx_idx INT DEFAULT 0;
 
+    CREATE TABLE IF NOT EXISTS solana_neon_transactions_costs (
+        sol_sign CHAR(88) UNIQUE,
+        operator VARCHAR(50),
+
+        heap_size INT,
+        bpf_instructions INT,
+
+        sol_cost BIGINT,
+        neon_income BIGINT
+    );
+
     CREATE TABLE IF NOT EXISTS solana_transaction_receipts (
         slot        BIGINT,
         tx_idx      INT,

proxy/indexer/accounts_db.py

@@ -1,46 +1,62 @@
+from dataclasses import astuple, dataclass
 from ..indexer.base_db import BaseDB, DBQuery
 from ..common_neon.utils import str_fmt_object
 
 
+@dataclass
 class NeonAccountInfo:
-    def __init__(self, neon_account: str = None, pda_account: str = None, code_account: str = None, slot: int = 0, code: str = None):
-        self.neon_account = neon_account
-        self.pda_account = pda_account
-        self.code_account = code_account
-        self.slot = slot
-        self.code = code
+    neon_address: str = None
+    pda_address: str = None
+    code_address: str = None
+    slot: int = 0
+    code: str = None
+    sol_sign: str = None
 
     def __str__(self):
         return str_fmt_object(self)
 
+    def __iter__(self):
+        return iter(astuple(self))
+
 
 class NeonAccountDB(BaseDB):
     def __init__(self):
         BaseDB.__init__(self, 'neon_accounts')
 
-    def set_acc_by_request(self, neon_account: str, pda_account: str, code_account: str, code: str):
+    def set_acc_by_request(self, neon_account: NeonAccountInfo):
         with self._conn.cursor() as cursor:
             cursor.execute(f'''
-                INSERT INTO {self._table_name}(neon_account, pda_account, code_account, slot, code)
-                VALUES(%s, %s, %s, %s, %s)
-                ON CONFLICT (pda_account, code_account) DO UPDATE
+                INSERT INTO neon_accounts (neon_address, pda_address, code_address, slot, code, sol_sign)
+                VALUES(%s, %s, %s, %s, %s, %s)
+                ON CONFLICT (pda_address, code_address) DO UPDATE
                 SET
                     code=EXCLUDED.code
                 ;
                 ''',
-                (neon_account, pda_account, code_account, 0, code))
+                tuple(neon_account))
 
-    def set_acc_indexer(self, neon_account: str, pda_account: str, code_account: str, slot: int):
+    def set_acc_indexer(self, neon_account: NeonAccountInfo):
+        if not self.fill_neon_address_if_missing(neon_account):
+            return
         with self._conn.cursor() as cursor:
             cursor.execute(f'''
-                INSERT INTO {self._table_name}(neon_account, pda_account, code_account, slot)
-                VALUES(%s, %s, %s, %s)
-                ON CONFLICT (pda_account, code_account) DO UPDATE
+                INSERT INTO neon_accounts (neon_address, pda_address, code_address, slot,  code, sol_sign)
+                VALUES(%s, %s, %s, %s, %s, %s)
+                ON CONFLICT (pda_address, code_address) DO UPDATE
                 SET
                     slot=EXCLUDED.slot
                 ;
                 ''',
-                (neon_account, pda_account, code_account, slot))
+                tuple(neon_account))
+
+    def fill_neon_address_if_missing(self, neon_account: NeonAccountInfo) -> bool:
+        if neon_account.neon_address is None:
+            value = self.get_account_info_by_pda_address(neon_account.pda_address)
+            if not value.neon_address:
+                self.error(f"Not found account for pda_address: {neon_account.pda_address}")
+                return False
+            neon_account.neon_address = value.neon_address
+        return True
 
     def _acc_from_value(self, value) -> NeonAccountInfo:
         self.debug(f"accounts db returned {value}")
@@ -49,18 +65,28 @@ def _acc_from_value(self, value) -> NeonAccountInfo:
             return NeonAccountInfo()
 
         return NeonAccountInfo(
-            neon_account=value[0],
-            pda_account=value[1],
-            code_account=value[2],
+            neon_address=value[0],
+            pda_address=value[1],
+            code_address=value[2],
             slot=value[3],
-            code=value[4]
+            code=value[4],
+            sol_sign=value[5]
+        )
+
+    def get_account_info_by_neon_address(self, neon_address) -> NeonAccountInfo:
+        return self._acc_from_value(
+            self._fetchone(DBQuery(
+                column_list=['neon_address', 'pda_address', 'code_address', 'slot', 'code', 'sol_sign'],
+                key_list=[('neon_address', neon_address)],
+                order_list=['slot desc']
+            ))
         )
 
-    def get_account_info(self, account) -> NeonAccountInfo:
+    def get_account_info_by_pda_address(self, pda_address) -> NeonAccountInfo:
         return self._acc_from_value(
             self._fetchone(DBQuery(
-                column_list=['neon_account', 'pda_account', 'code_account', 'slot', 'code'],
-                key_list=[('neon_account', account)],
+                column_list=['neon_address', 'pda_address', 'code_address', 'slot', 'code', 'sol_sign'],
+                key_list=[('pda_address', pda_address)],
                 order_list=['slot desc']
             ))
         )

proxy/indexer/costs_db.py

@@ -0,0 +1,21 @@
+from typing import List
+
+from ..indexer.utils import CostInfo
+from ..indexer.base_db import BaseDB
+
+
+class CostsDB(BaseDB):
+    def __init__(self):
+        BaseDB.__init__(self, 'solana_neon_transactions_costs')
+
+    def add_costs(self, tx_costs: List[CostInfo]):
+        rows = []
+        for cost_info in tx_costs:
+            rows.append(tuple(cost_info))
+
+        with self._conn.cursor() as cursor:
+            cursor.executemany(f'''
+                INSERT INTO solana_neon_transactions_costs
+                (sol_sign, operator, heap_size, bpf_instructions, sol_cost, neon_income)
+                VALUES(%s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING''',
+                rows)

proxy/indexer/indexer.py

@@ -5,9 +5,10 @@
 from logged_groups import logged_group, logging_context
 from solana.system_program import SYS_PROGRAM_ID
 
+from ..indexer.accounts_db import NeonAccountInfo
 from ..indexer.indexer_base import IndexerBase
 from ..indexer.indexer_db import IndexerDB
-from ..indexer.utils import SolanaIxSignInfo, MetricsToLogBuff
+from ..indexer.utils import SolanaIxSignInfo, MetricsToLogBuff, CostInfo
 from ..indexer.utils import check_error
 from ..indexer.canceller import Canceller
 
@@ -21,6 +22,7 @@
 class SolanaIxInfo:
     def __init__(self, sign: str, slot: int, tx: {}):
         self.sign = SolanaIxSignInfo(sign=sign, slot=slot, idx=-1)
+        self.cost_info = CostInfo(sign, tx, EVM_LOADER_ID)
         self.tx = tx
         self._is_valid = isinstance(tx, dict)
         self._msg = self.tx['transaction']['message'] if self._is_valid else None
@@ -261,8 +263,8 @@ def iter_txs(self):
         for tx in self._tx_table.values():
             yield tx
 
-    def add_account_to_db(self, neon_account: str, pda_account: str, code_account: str, slot: int):
-        self._db.fill_account_info_by_indexer(neon_account, pda_account, code_account, slot)
+    def add_account_to_db(self, neon_account: NeonAccountInfo):
+        self._db.fill_account_info_by_indexer(neon_account)
 
 
 @logged_group("neon.Indexer")
@@ -484,7 +486,7 @@ def execute(self) -> bool:
 
         self.debug(f"neon_account({neon_account}), pda_account({pda_account}), code_account({code_account}), slot({self.ix.sign.slot})")
 
-        self.state.add_account_to_db(neon_account, pda_account, code_account, self.ix.sign.slot)
+        self.state.add_account_to_db(NeonAccountInfo(neon_account, pda_account, code_account, self.ix.sign.slot, None, self.ix.sign.sign))
         return True
 
 class CreateAccount2IxDecoder(DummyIxDecoder):
@@ -508,7 +510,7 @@ def execute(self) -> bool:
 
         self.debug(f"neon_account({neon_account}), pda_account({pda_account}), code_account({code_account}), slot({self.ix.sign.slot})")
 
-        self.state.add_account_to_db(neon_account, pda_account, code_account, self.ix.sign.slot)
+        self.state.add_account_to_db(NeonAccountInfo(neon_account, pda_account, code_account, self.ix.sign.slot, None, self.ix.sign.sign))
         return True
 
 class ResizeStorageAccountIxDecoder(DummyIxDecoder):
@@ -526,7 +528,7 @@ def execute(self) -> bool:
 
         self.debug(f"pda_account({pda_account}), code_account({code_account}), slot({self.ix.sign.slot})")
 
-        self.state.add_account_to_db(None, pda_account, code_account, self.ix.sign.slot)
+        self.state.add_account_to_db(NeonAccountInfo(None, pda_account, code_account, self.ix.sign.slot, None, self.ix.sign.sign))
         return True
 
 
@@ -549,6 +551,7 @@ def execute(self) -> bool:
 
         neon_res = NeonTxResultInfo(neon_tx.sign, self.ix.tx, self.ix.sign.idx)
         tx = NeonTxObject('', neon_tx=neon_tx, neon_res=neon_res)
+
         return self._decoding_done(tx, 'call success')
 
 
@@ -578,6 +581,8 @@ def execute(self) -> bool:
 
         tx = self._getadd_tx(storage_account, neon_tx=neon_tx, blocked_accounts=blocked_accounts)
         tx.step_count.append(step_count)
+
+        self.ix.sign.set_steps(step_count)
         return self._decode_tx(tx)
 
 
@@ -610,6 +615,8 @@ def execute(self) -> bool:
 
         tx = self._getadd_tx(storage_account, blocked_accounts=blocked_accounts)
         tx.step_count.append(step_count)
+
+        self.ix.sign.set_steps(step_count)
         return self._decode_tx(tx)
 
 
@@ -639,6 +646,8 @@ def execute(self) -> bool:
         if not tx:
             return self._decoding_skip(f'fail to init in storage {storage_account} from holder {holder_account}')
         tx.step_count.append(step_count)
+
+        self.ix.sign.set_steps(step_count)
         return self._decode_tx(tx)
 
 
@@ -667,6 +676,7 @@ def execute(self) -> bool:
             return self._decoding_fail(tx, f'cannot find storage {tx}')
 
         tx.neon_res.canceled(self.ix.tx)
+
         return self._decoding_done(tx, f'cancel success')
 
 
@@ -696,6 +706,8 @@ def execute(self) -> bool:
         if not tx:
             return self._decoding_skip(f'fail to init the storage {storage_account} from the holder {holder_account}')
         tx.step_count.append(step_count)
+
+        self.ix.sign.set_steps(step_count)
         return self._decode_tx(tx)
 
 
@@ -769,6 +781,8 @@ def process_functions(self):
         self.blocked_storages = {}
 
     def process_receipts(self):
+        tx_costs = []
+
         start_time = time.time()
 
         max_slot = 0
@@ -782,14 +796,17 @@ def process_receipts(self):
                 self.state.complete_done_txs()
                 max_slot = max(max_slot, slot)
 
-            ix_info = SolanaIxInfo(sign=sign, slot=slot,  tx=tx)
+            ix_info = SolanaIxInfo(sign=sign, slot=slot, tx=tx)
 
             for _ in ix_info.iter_ixs():
                 req_id = ix_info.sign.get_req_id()
                 with logging_context(sol_tx=req_id):
                         self.state.set_ix(ix_info)
                         (self.ix_decoder_map.get(ix_info.evm_ix) or self.def_decoder).execute()
 
+            tx_costs.append(ix_info.cost_info)
+
+
         self.indexed_slot = last_block_slot
         self.db.set_min_receipt_slot(self.state.find_min_used_slot(self.indexed_slot))
 
@@ -802,6 +819,7 @@ def process_receipts(self):
 
         # after last instruction and slot
         self.state.complete_done_txs()
+        self.db.add_tx_costs(tx_costs)
 
         process_receipts_ms = (time.time() - start_time) * 1000  # convert this into milliseconds
         self.counted_logger.print(