diff --git a/proxy/airdropper/airdropper.py b/proxy/airdropper/airdropper.py index 81f380106..b4ed81d86 100644 --- a/proxy/airdropper/airdropper.py +++ b/proxy/airdropper/airdropper.py @@ -265,7 +265,7 @@ def process_functions(self): def process_receipts(self): max_slot = 0 - for slot, _, trx in self.transaction_receipts.get_trxs(self.latest_processed_slot): + for slot, _, trx in self.transaction_receipts.get_txs(self.latest_processed_slot): max_slot = max(max_slot, slot) if trx['transaction']['message']['instructions'] is not None: self.process_trx_airdropper_mode(trx) diff --git a/proxy/common_neon/address.py b/proxy/common_neon/address.py index f85f15931..14d019681 100644 --- a/proxy/common_neon/address.py +++ b/proxy/common_neon/address.py @@ -5,7 +5,6 @@ from eth_keys import keys as eth_keys from hashlib import sha256 from solana.publickey import PublicKey -from typing import NamedTuple class EthereumAddress: @@ -36,7 +35,7 @@ def __repr__(self): def __bytes__(self): return self.data -def accountWithSeed(base, seed): +def accountWithSeed(base: bytes, seed: bytes) -> PublicKey: from ..environment import EVM_LOADER_ID result = PublicKey(sha256(bytes(base) + bytes(seed) + bytes(PublicKey(EVM_LOADER_ID))).digest()) @@ -59,25 +58,3 @@ def ether2program(ether): seed = [ACCOUNT_SEED_VERSION, bytes.fromhex(ether)] (pda, nonce) = PublicKey.find_program_address(seed, PublicKey(EVM_LOADER_ID)) return str(pda), nonce - - -class AccountInfoLayout(NamedTuple): - ether: eth_keys.PublicKey - balance: int - trx_count: int - code_account: PublicKey - - def is_payed(self) -> bool: - return self.state != 0 - - @staticmethod - def frombytes(data) -> AccountInfoLayout: - from .layouts import ACCOUNT_INFO_LAYOUT - - cont = ACCOUNT_INFO_LAYOUT.parse(data) - return AccountInfoLayout( - ether=cont.ether, - balance=int.from_bytes(cont.balance, "little"), - trx_count=int.from_bytes(cont.trx_count, "little"), - code_account=PublicKey(cont.code_account) - ) diff --git a/proxy/common_neon/constants.py b/proxy/common_neon/constants.py index 9dcbd23e6..edc91143d 100644 --- a/proxy/common_neon/constants.py +++ b/proxy/common_neon/constants.py @@ -9,5 +9,6 @@ COLLATERALL_POOL_MAX=10 -EMPTY_STORAGE_TAG=0 -FINALIZED_STORAGE_TAG=5 +EMPTY_STORAGE_TAG = 0 +FINALIZED_STORAGE_TAG = 5 +ACTIVE_STORAGE_TAG = 30 diff --git a/proxy/common_neon/emulator_interactor.py b/proxy/common_neon/emulator_interactor.py index 70591e365..f0b49f9a9 100644 --- a/proxy/common_neon/emulator_interactor.py +++ b/proxy/common_neon/emulator_interactor.py @@ -207,34 +207,30 @@ def _find_account(line_list: [str], hdr: str) -> str: return account -class AccountUninitializedParser(FindAccount): - def execute(self, err: subprocess.CalledProcessError) -> str: - msg = 'error on trying to call the not-initialized contract: ' - hdr = 'NeonCli Error (212): Uninitialized account. account=' - account = self._find_account(err.stderr.split('\n'), hdr) - return msg + account - - class AccountAlreadyInitializedParser(FindAccount): - def execute(self, err: subprocess.CalledProcessError) -> str: + def execute(self, err: subprocess.CalledProcessError) -> (str, int): msg = 'error on trying to initialize already initialized contract: ' hdr = 'NeonCli Error (213): Account is already initialized. account=' account = self._find_account(err.stderr.split('\n'), hdr) - return msg + account + return msg + account, self._code class DeployToExistingAccountParser(FindAccount): - def execute(self, err: subprocess.CalledProcessError) -> str: + def execute(self, err: subprocess.CalledProcessError) -> (str, int): msg = 'error on trying to deploy contract to user account: ' hdr = 'NeonCli Error (221): Attempt to deploy to existing account at address ' account = self._find_account(err.stderr.split('\n'), hdr) - return msg + account + return msg + account, self._code class TooManyStepsErrorParser(BaseNeonCliErrorParser): pass +class TrxCountOverflowErrorParser(BaseNeonCliErrorParser): + pass + + class NeonCliErrorParser: ERROR_PARSER_DICT = { 102: ProxyConfigErrorParser('cannot read/write data to/from disk'), @@ -254,13 +250,13 @@ class NeonCliErrorParser: 208: StorageErrorParser('code account required'), 215: StorageErrorParser('contract account expected'), - 212: AccountUninitializedParser('AccountUninitialized'), - 213: AccountAlreadyInitializedParser('AccountAlreadyInitialized'), 221: DeployToExistingAccountParser('DeployToExistingAccount'), 245: TooManyStepsErrorParser('execution requires too lot of EVM steps'), + + 249: TrxCountOverflowErrorParser('transaction counter overflow') } def execute(self, caption: str, err: subprocess.CalledProcessError) -> (str, int): diff --git a/proxy/common_neon/layouts.py b/proxy/common_neon/layouts.py index 052643955..f149157f9 100644 --- a/proxy/common_neon/layouts.py +++ b/proxy/common_neon/layouts.py @@ -3,14 +3,14 @@ from construct import Struct STORAGE_ACCOUNT_INFO_LAYOUT = Struct( - # "tag" / Int8ul, + "tag" / Int8ul, "caller" / Bytes(20), "nonce" / Int64ul, "gas_limit" / Bytes(32), "gas_price" / Bytes(32), "slot" / Int64ul, "operator" / Bytes(32), - "accounts_len" / Int64ul, + "account_list_len" / Int64ul, "executor_data_size" / Int64ul, "evm_data_size" / Int64ul, "gas_used_and_paid" / Bytes(32), diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py index 8bebe014b..b5e2bdac4 100644 --- a/proxy/common_neon/neon_instruction.py +++ b/proxy/common_neon/neon_instruction.py @@ -236,17 +236,21 @@ def make_noniterative_call_transaction(self, length_before: int = 0) -> Transact trx.add(self.make_05_call_instruction()) return trx - def make_cancel_transaction(self, cancel_keys=None) -> Transaction: + def make_cancel_transaction(self, storage=None, nonce=None, cancel_keys=None) -> Transaction: if cancel_keys: append_keys = cancel_keys else: append_keys = self.eth_accounts append_keys += obligatory_accounts + if not nonce: + nonce = self.eth_trx.nonce + if not storage: + storage = self.storage return TransactionWithComputeBudget().add(TransactionInstruction( program_id = EVM_LOADER_ID, - data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), + data = bytearray.fromhex("15") + nonce.to_bytes(8, 'little'), keys=[ - AccountMeta(pubkey=self.storage, is_signer=False, is_writable=True), + AccountMeta(pubkey=storage, is_signer=False, is_writable=True), AccountMeta(pubkey=self.operator_account, is_signer=True, is_writable=True), AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), ] + append_keys diff --git a/proxy/common_neon/solana_interactor.py b/proxy/common_neon/solana_interactor.py index 0e3c55dd4..1b4c35358 100644 --- a/proxy/common_neon/solana_interactor.py +++ b/proxy/common_neon/solana_interactor.py @@ -5,6 +5,7 @@ import time import traceback import requests +import json from typing import Optional @@ -24,9 +25,8 @@ from ..environment import FUZZING_BLOCKHASH, CONFIRM_TIMEOUT, FINALIZED from ..environment import RETRY_ON_FAIL -from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT +from ..common_neon.layouts import ACCOUNT_INFO_LAYOUT, STORAGE_ACCOUNT_INFO_LAYOUT from ..common_neon.address import EthereumAddress, ether2program -from ..common_neon.address import AccountInfoLayout from ..common_neon.utils import get_from_dict @@ -37,6 +37,78 @@ class AccountInfo(NamedTuple): data: bytes +class NeonAccountInfo(NamedTuple): + ether: str + nonce: int + trx_count: int + balance: int + code_account: PublicKey + is_rw_blocked: bool + ro_blocked_cnt: int + + @staticmethod + def frombytes(data) -> NeonAccountInfo: + cont = ACCOUNT_INFO_LAYOUT.parse(data) + return NeonAccountInfo( + ether=cont.ether.hex(), + nonce=cont.nonce, + trx_count=int.from_bytes(cont.trx_count, "little"), + balance=int.from_bytes(cont.balance, "little"), + code_account=PublicKey(cont.code_account), + is_rw_blocked=(cont.is_rw_blocked != 0), + ro_blocked_cnt=cont.ro_blocked_cnt + ) + + +class StorageAccountInfo(NamedTuple): + tag: int + caller: str + nonce: int + gas_limit: int + gas_price: int + slot: int + operator: PublicKey + account_list_len: int + executor_data_size: int + evm_data_size: int + gas_used_and_paid: int + number_of_payments: int + sign: bytes + account_list: [str] + + @staticmethod + def frombytes(data) -> StorageAccountInfo: + storage = STORAGE_ACCOUNT_INFO_LAYOUT.parse(data) + + account_list = [] + offset = STORAGE_ACCOUNT_INFO_LAYOUT.sizeof() + for _ in range(storage.account_list_len): + writable = (data[offset] > 0) + offset += 1 + + some_pubkey = PublicKey(data[offset:offset + 32]) + offset += 32 + + account_list.append((writable, str(some_pubkey))) + + return StorageAccountInfo( + tag=storage.tag, + caller=storage.caller.hex(), + nonce=storage.nonce, + gas_limit=int.from_bytes(storage.gas_limit, "little"), + gas_price=int.from_bytes(storage.gas_price, "little"), + slot=storage.slot, + operator=PublicKey(storage.operator), + account_list_len=storage.account_list_len, + executor_data_size=storage.executor_data_size, + evm_data_size=storage.evm_data_size, + gas_used_and_paid=int.from_bytes(storage.gas_used_and_paid, "little"), + number_of_payments=storage.number_of_payments, + sign=storage.sign, + account_list=account_list + ) + + class SendResult(NamedTuple): error: dict result: dict @@ -124,9 +196,15 @@ def _send_rpc_batch_request(self, method: str, params_list: List[Any]) -> List[R def get_cluster_nodes(self) -> [dict]: return self._send_rpc_request("getClusterNodes").get('result', []) - def is_health(self) -> bool: - status = self._send_rpc_request('getHealth').get('result', 'bad') - return status == 'ok' + def get_slots_behind(self) -> Optional[int]: + response = self._send_rpc_request('getHealth') + status = response.get('result') + if status == 'ok': + return 0 + slots_behind = get_from_dict(response, 'error', 'data', 'numSlotsBehind') + if slots_behind: + return int(slots_behind) + return None def get_signatures_for_address(self, before: Optional[str], limit: int, commitment='confirmed') -> []: opts: Dict[str, Union[int, str]] = {} @@ -237,7 +315,7 @@ def get_token_account_balance_list(self, pubkey_list: [Union[str, PublicKey]], c return balance_list - def get_account_info_layout(self, eth_account: EthereumAddress) -> Optional[AccountInfoLayout]: + def get_neon_account_info(self, eth_account: EthereumAddress) -> Optional[NeonAccountInfo]: account_sol, nonce = ether2program(eth_account) info = self.get_account_info(account_sol) if info is None: @@ -245,7 +323,18 @@ def get_account_info_layout(self, eth_account: EthereumAddress) -> Optional[Acco elif len(info.data) < ACCOUNT_INFO_LAYOUT.sizeof(): raise RuntimeError(f"Wrong data length for account data {account_sol}: " + f"{len(info.data)} < {ACCOUNT_INFO_LAYOUT.sizeof()}") - return AccountInfoLayout.frombytes(info.data) + return NeonAccountInfo.frombytes(info.data) + + def get_storage_account_info(self, storage_account: PublicKey) -> Optional[StorageAccountInfo]: + info = self.get_account_info(storage_account, length=0) + if info is None: + return None + elif info.tag != 30: + return None + elif len(info.data) < STORAGE_ACCOUNT_INFO_LAYOUT.sizeof(): + raise RuntimeError(f"Wrong data length for storage data {storage_account}: " + + f"{len(info.data)} < {STORAGE_ACCOUNT_INFO_LAYOUT.sizeof()}") + return StorageAccountInfo.frombytes(info.data) def get_multiple_rent_exempt_balances_for_size(self, size_list: [int], commitment='confirmed') -> [int]: opts = { @@ -402,7 +491,7 @@ def _send_multiple_transactions(self, signer: SolanaAccount, tx_list: [Transacti for response, tx in zip(response_list, tx_list): result = response.get('result') error = response.get('error') - if error and get_from_dict('data', 'err') == 'AlreadyProcessed': + if error and get_from_dict(error, 'data', 'err') == 'AlreadyProcessed': error = None result = tx.signature() result_list.append(SendResult(result=result, error=error)) @@ -420,7 +509,7 @@ def send_multiple_transactions(self, signer: SolanaAccount, tx_list: [], waiter, receipt_list = [] for s in send_result_list: if s.error: - self.debug(f'Got error on preflight check of transaction: {s.error}') + self.debug(f'Got error on preflight check of transaction: {json.dumps(s.error, sort_keys=True)}') receipt_list.append(s.error) else: receipt_list.append(confirmed_list.pop(0)) diff --git a/proxy/common_neon/solana_tx_list_sender.py b/proxy/common_neon/solana_tx_list_sender.py index 1141dbad6..1438d91b8 100644 --- a/proxy/common_neon/solana_tx_list_sender.py +++ b/proxy/common_neon/solana_tx_list_sender.py @@ -5,6 +5,7 @@ from logged_groups import logged_group from typing import Optional from solana.transaction import Transaction +from base58 import b58encode from .costs import update_transaction_cost from .solana_receipt_parser import SolReceiptParser, SolTxError @@ -24,9 +25,9 @@ def __init__(self, sender, tx_list: [Transaction], name: str, self._blockhash = None self._retry_idx = 0 - self._total_success_cnt = 0 self._slots_behind = 0 self._tx_list = tx_list + self.success_sign_list = [] self._node_behind_list = [] self._bad_block_list = [] self._blocked_account_list = [] @@ -66,7 +67,7 @@ def send(self) -> SolTxListSender: receipt_list = solana.send_multiple_transactions(signer, self._tx_list, waiter, skip, commitment) self.update_transaction_cost(receipt_list) - success_cnt = 0 + success_sign_list = [] for receipt, tx in zip(receipt_list, self._tx_list): receipt_parser = SolReceiptParser(receipt) slots_behind = receipt_parser.get_slots_behind() @@ -83,20 +84,20 @@ def send(self) -> SolTxListSender: elif receipt_parser.check_if_error(): self._unknown_error_list.append(receipt) else: - success_cnt += 1 + success_sign_list.append(b58encode(tx.signature()).decode("utf-8")) self._retry_idx = 0 self._on_success_send(tx, receipt) self.debug(f'retry {self._retry_idx}, ' + f'total receipts {len(receipt_list)}, ' + - f'success receipts {self._total_success_cnt}(+{success_cnt}), ' + + f'success receipts {len(self.success_sign_list)}(+{len(success_sign_list)}), ' + f'node behind {len(self._node_behind_list)}, ' f'bad blocks {len(self._bad_block_list)}, ' + f'blocked accounts {len(self._blocked_account_list)}, ' + f'budget exceeded {len(self._budget_exceeded_list)}, ' + f'unknown error: {len(self._unknown_error_list)}') - self._total_success_cnt += success_cnt + self.success_sign_list += success_sign_list self._on_post_send() if len(self._tx_list): diff --git a/proxy/common_neon/transaction_sender.py b/proxy/common_neon/transaction_sender.py index a653ecf4e..5edc3399a 100644 --- a/proxy/common_neon/transaction_sender.py +++ b/proxy/common_neon/transaction_sender.py @@ -22,10 +22,10 @@ from .address import accountWithSeed, EthereumAddress, ether2program from .compute_budget import TransactionWithComputeBudget -from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACCOUNT_SEED_VERSION +from .constants import STORAGE_SIZE, EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG, ACTIVE_STORAGE_TAG, ACCOUNT_SEED_VERSION from .emulator_interactor import call_emulated from .neon_instruction import NeonInstruction as NeonIxBuilder -from .solana_interactor import SolanaInteractor +from .solana_interactor import SolanaInteractor, StorageAccountInfo from .solana_tx_list_sender import SolTxListSender from .solana_receipt_parser import SolTxError, SolReceiptParser, Measurements from .transaction_validator import NeonTxValidator @@ -55,6 +55,28 @@ def build(self): pass +@logged_group("neon.Proxy") +class NeonCancelTxStage(NeonTxStage, abc.ABC): + NAME = 'cancelWithNonce' + + def __init__(self, sender, account: PublicKey): + NeonTxStage.__init__(self, sender) + self._account = account + self._storage = self.s.solana.get_storage_account_account(account) + + def _cancel_tx(self): + return self.s.builder.make_cancel_transaction(storage=self._account, + nonce=self._storage.nonce, + cancel_keys=self._storage.account_list) + + def build(self): + assert self._is_empty() + assert self._storage is not None + + self.debug(f'Cancel transaction in storage account {str(self._account)}') + self.tx = self._cancel_tx() + + class NeonCreateAccountWithSeedStage(NeonTxStage, abc.ABC): def __init__(self, sender): NeonTxStage.__init__(self, sender) @@ -402,29 +424,42 @@ def _create_ether_account(self) -> EthereumAddress: return ether_address def _create_perm_accounts(self, seed_list): + rid = self._resource.rid + opkey = str(self._resource.public_key()) + tx = TransactionWithComputeBudget() + tx_name_list = set() stage_list = [NeonCreatePermAccount(self._s, seed, STORAGE_SIZE) for seed in seed_list] account_list = [s.sol_account for s in stage_list] info_list = self._s.solana.get_account_info_list(account_list) balance = self._s.solana.get_multiple_rent_exempt_balances_for_size([STORAGE_SIZE])[0] - for idx, account, stage in zip(range(2), info_list, stage_list): + for idx, account, stage in zip(range(len(seed_list)), info_list, stage_list): if not account: + self.debug(f"Create new accounts for resource {opkey}:{rid}") stage.balance = balance stage.build() + tx_name_list.add(stage.NAME) tx.add(stage.tx) + continue elif account.lamports < balance: raise RuntimeError(f"insufficient balance of {str(stage.sol_account)}") elif PublicKey(account.owner) != PublicKey(EVM_LOADER_ID): raise RuntimeError(f"wrong owner for: {str(stage.sol_account)}") - elif (idx == 0) and (account.tag not in {EMPTY_STORAGE_TAG, FINALIZED_STORAGE_TAG}): + elif idx != 0: + continue + + if account.tag == ACTIVE_STORAGE_TAG: + self.debug(f"Cancel transaction in {str(stage.sol_account)} for resource {opkey}:{rid}") + cancel_stage = NeonCancelTxStage(self._s, stage.sol_account) + cancel_stage.build() + tx_name_list.add(cancel_stage.NAME) + tx.add(cancel_stage.tx) + elif account.tag not in (FINALIZED_STORAGE_TAG, EMPTY_STORAGE_TAG): raise RuntimeError(f"not empty, not finalized: {str(stage.sol_account)}") - rid = self._resource.rid - opkey = str(self._resource.public_key()) if len(tx.instructions): - self.debug(f"Create new accounts for resource {opkey}:{rid}") - SolTxListSender(self._s, [tx], NeonCreatePermAccount.NAME).send() + SolTxListSender(self._s, [tx], ' + '.join(tx_name_list)).send() else: self.debug(f"Use existing accounts for resource {opkey}:{rid}") return account_list @@ -499,8 +534,9 @@ def _validate_execution(self): self._resource_list.init_resource_info() self._validate_pend_tx() + self._neon_tx_validator.prevalidate_tx(self.signer) self._call_emulated() - self._neon_tx_validator.prevalidate_tx(self.signer, self._emulator_json) + self._neon_tx_validator.prevalidate_emulator(self._emulator_json) def _validate_pend_tx(self): operator = f'{str(self.resource.public_key())}:{self.resource.rid}' @@ -516,8 +552,8 @@ def _execute(self): continue self.debug(f'Use strategy {Strategy.NAME}') - neon_res = strategy.execute() - self._submit_tx_into_db(neon_res) + neon_res, sign_list = strategy.execute() + self._submit_tx_into_db(neon_res, sign_list) return neon_res except Exception as e: if (not Strategy.IS_SIMPLE) or (not SolReceiptParser(e).check_if_budget_exceeded()): @@ -542,10 +578,10 @@ def _pend_tx_into_db(self, slot: int): self._pending_tx.slot = slot self._db.pend_transaction(self._pending_tx) - def _submit_tx_into_db(self, neon_res: NeonTxResultInfo): + def _submit_tx_into_db(self, neon_res: NeonTxResultInfo, sign_list: [str]): neon_tx = NeonTxInfo() neon_tx.init_from_eth_tx(self.eth_tx) - self._db.submit_transaction(neon_tx, neon_res) + self._db.submit_transaction(neon_tx, neon_res, sign_list) def _prepare_execution(self): # Parse information from the emulator output @@ -655,8 +691,8 @@ def __init__(self, sender: NeonTxSender): self.is_valid = self._validate() @abc.abstractmethod - def execute(self) -> NeonTxResultInfo: - return NeonTxResultInfo() + def execute(self) -> (NeonTxResultInfo, [str]): + return NeonTxResultInfo(), [] @abc.abstractmethod def build_tx(self) -> Transaction: @@ -746,7 +782,7 @@ def build_tx(self) -> Transaction: tx.add(self.s.builder.make_noniterative_call_transaction(len(tx.instructions))) return tx - def execute(self) -> NeonTxResultInfo: + def execute(self) -> (NeonTxResultInfo, [str]): tx_list = self.s.build_account_txs(not self._skip_create_account) if len(tx_list) > 0: SolTxListSender(self.s, tx_list, self.s.account_txs_name).send() @@ -755,7 +791,7 @@ def execute(self) -> NeonTxResultInfo: tx_sender = SimpleNeonTxSender(self, self.s, [self.build_tx()], self.NAME).send() if not tx_sender.neon_res.is_valid(): raise tx_sender.raise_budget_exceeded() - return tx_sender.neon_res + return tx_sender.neon_res, tx_sender.success_sign_list @logged_group("neon.Proxy") @@ -831,14 +867,14 @@ def _on_post_send(self): self._raise_error() self._unknown_error_list.clear() - if self._total_success_cnt: + if len(self.success_sign_list): return self._cancel() self._raise_error() # There is no more retries to send transactions if self._retry_idx >= RETRY_ON_FAIL: self._set_postponed_exception(EthereumError(message='No more retries to complete transaction!')) - if (not self._is_canceled) and (self._total_success_cnt > 0): + if (not self._is_canceled) and len(self.success_sign_list): return self._cancel() self._raise_error() @@ -888,7 +924,7 @@ def _build_preparation_txs(self) -> [Transaction]: self._preparation_txs_name = self.s.account_txs_name return self.s.build_account_txs(False) - def execute(self) -> NeonTxResultInfo: + def execute(self) -> (NeonTxResultInfo, [str]): tx_list = self._build_preparation_txs() if len(tx_list): SolTxListSender(self.s, tx_list, self._preparation_txs_name).send() @@ -898,7 +934,9 @@ def execute(self) -> NeonTxResultInfo: cnt = math.ceil(self.s.steps_emulated / (self.steps - cnt)) + 2 # +1 on begin, +1 on end tx_list = [self.build_tx() for _ in range(cnt)] self.debug(f'Total iterations {len(tx_list)} for {self.s.steps_emulated} ({self.steps}) EVM steps') - return IterativeNeonTxSender(self, self.s, tx_list, self.NAME).send().neon_res + tx_sender = IterativeNeonTxSender(self, self.s, tx_list, self.NAME) + tx_sender.send() + return tx_sender.neon_res, tx_sender.success_sign_list @logged_group("neon.Proxy") diff --git a/proxy/common_neon/transaction_validator.py b/proxy/common_neon/transaction_validator.py index 06ebc4d8a..4369acfe1 100644 --- a/proxy/common_neon/transaction_validator.py +++ b/proxy/common_neon/transaction_validator.py @@ -24,7 +24,7 @@ def __init__(self, solana: SolanaInteractor, tx: EthTx): self._tx = tx self._sender = '0x' + tx.sender() - self._account_info = self._solana.get_account_info_layout(EthereumAddress(self._sender)) + self._neon_account_info = self._solana.get_neon_account_info(EthereumAddress(self._sender)) self._deployed_contract = tx.contract() if self._deployed_contract: @@ -36,7 +36,7 @@ def __init__(self, solana: SolanaInteractor, tx: EthTx): self._tx_hash = '0x' + self._tx.hash_signed().hex() - def prevalidate_tx(self, signer: SolanaAccount, emulator_json: dict): + def prevalidate_tx(self, signer: SolanaAccount): self._prevalidate_whitelist(signer) self._prevalidate_tx_nonce() @@ -44,6 +44,8 @@ def prevalidate_tx(self, signer: SolanaAccount, emulator_json: dict): self._prevalidate_tx_chain_id() self._prevalidate_tx_size() self._prevalidate_sender_balance() + + def prevalidate_emulator(self, emulator_json: dict): self._prevalidate_gas_usage(emulator_json) def extract_ethereum_error(self, e: Exception): @@ -77,26 +79,26 @@ def _prevalidate_tx_size(self): raise EthereumError(message='transaction size is too big') def _prevalidate_tx_nonce(self): - if not self._account_info: + if not self._neon_account_info: return tx_nonce = int(self._tx.nonce) - if self.MAX_U64 not in (self._account_info.trx_count, tx_nonce): - if tx_nonce == self._account_info.trx_count: + if self.MAX_U64 not in (self._neon_account_info.trx_count, tx_nonce): + if tx_nonce == self._neon_account_info.trx_count: return - self._raise_nonce_error(self._account_info.trx_count, tx_nonce) + self._raise_nonce_error(self._neon_account_info.trx_count, tx_nonce) def _prevalidate_sender_eoa(self): - if not self._account_info: + if not self._neon_account_info: return - if self._account_info.code_account: + if self._neon_account_info.code_account: raise EthereumError("sender not an eoa") def _prevalidate_sender_balance(self): - if self._account_info: - user_balance = self._account_info.balance + if self._neon_account_info: + user_balance = self._neon_account_info.balance else: user_balance = 0 diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py index 0f9c8b10b..1d8d1c3d2 100644 --- a/proxy/common_neon/utils.py +++ b/proxy/common_neon/utils.py @@ -149,6 +149,8 @@ def is_valid(self) -> bool: class NeonTxInfo: def __init__(self, rlp_sign=None, rlp_data=None): + self.tx_idx = 0 + self._set_defaults() if isinstance(rlp_sign, bytes) and isinstance(rlp_data, bytes): self.decode(rlp_sign, rlp_data) @@ -163,7 +165,6 @@ def __setstate__(self, src): self.__dict__ = src def _set_defaults(self): - self.tx = None self.addr = None self.sign = None self.nonce = None @@ -179,8 +180,6 @@ def _set_defaults(self): self.error = None def init_from_eth_tx(self, tx: EthTx): - self.tx = tx - self.v = hex(tx.v) self.r = hex(tx.r) self.s = hex(tx.s) diff --git a/proxy/db/scheme.sql b/proxy/db/scheme.sql index 31675598a..c28ecd18d 100644 --- a/proxy/db/scheme.sql +++ b/proxy/db/scheme.sql @@ -116,33 +116,20 @@ UNIQUE(sol_sign, idx) ); - CREATE TABLE IF NOT EXISTS transaction_receipts ( - slot BIGINT, - signature VARCHAR(88), - trx BYTEA, - PRIMARY KEY (slot, signature) - ); - - CREATE TABLE IF NOT EXISTS constants ( - key TEXT UNIQUE, - value BYTEA - ) - - CREATE TABLE IF NOT EXISTS airdrop_scheduled ( - key TEXT UNIQUE, - value BYTEA - ) + ALTER TABLE neon_transactions ADD COLUMN IF NOT EXISTS tx_idx INT DEFAULT 0; - CREATE TABLE IF NOT EXISTS transaction_receipts ( + CREATE TABLE IF NOT EXISTS solana_transaction_receipts ( slot BIGINT, + tx_idx INT, signature VARCHAR(88), - trx BYTEA, + tx BYTEA, PRIMARY KEY (slot, signature) ); CREATE TABLE IF NOT EXISTS test_storage ( slot BIGINT, + tx_idx INT, signature VARCHAR(88), - trx BYTEA, + tx BYTEA, PRIMARY KEY (slot, signature) ); diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index b2e3e5f30..eca047697 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -10,10 +10,10 @@ @logged_group("neon.Indexer") class Canceller: - def __init__(self): + def __init__(self, solana: SolanaInteractor): # Initialize user account self.signer = get_solana_accounts()[0] - self.solana = SolanaInteractor(SOLANA_URL) + self.solana = solana self.waiter = None self._operator = self.signer.public_key() self.builder = NeonInstruction(self._operator) @@ -29,10 +29,9 @@ def unlock_accounts(self, blocked_storages): for is_writable, acc in blocked_accounts: keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=is_writable)) - self.builder.init_eth_trx(neon_tx.tx, None) self.builder.init_iterative(storage, None, 0) - tx = self.builder.make_cancel_transaction(keys) + tx = self.builder.make_cancel_transaction(nonce=int(neon_tx.nonce[2:]), cancel_keys=keys) tx_list.append(tx) if not len(tx_list): diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index f0b94a82c..0727db494 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -8,7 +8,7 @@ from ..indexer.indexer_base import IndexerBase from ..indexer.indexer_db import IndexerDB from ..indexer.utils import SolanaIxSignInfo, MetricsToLogBuff -from ..indexer.utils import get_accounts_from_storage, check_error +from ..indexer.utils import check_error from ..indexer.canceller import Canceller from ..common_neon.utils import NeonTxResultInfo, NeonTxInfo, str_fmt_object @@ -132,6 +132,7 @@ def __init__(self, storage_account: str, neon_tx: NeonTxInfo, neon_res: NeonTxRe self.holder_account = '' self.blocked_accounts = [] self.canceled = False + self.done = False def __str__(self): return str_fmt_object(self) @@ -230,6 +231,9 @@ def done_tx(self, tx: NeonTxObject): Continue waiting of ixs in the slot with the same neon tx, because the parsing order can be other than the execution order. """ + if tx.done: + return + tx.done = True self._done_tx_list.append(tx) def complete_done_txs(self): @@ -723,7 +727,7 @@ def __init__(self, solana_url): last_known_slot = self.db.get_min_receipt_slot() IndexerBase.__init__(self, solana, last_known_slot) self.indexed_slot = self.last_slot - self.canceller = Canceller() + self.canceller = Canceller(solana) self.blocked_storages = {} self.block_indexer = BlocksIndexer(db=self.db, solana=solana) self.counted_logger = MetricsToLogBuff() @@ -768,9 +772,9 @@ def process_receipts(self): start_time = time.time() max_slot = 0 - last_block_slot = self.db.get_latest_block().slot + last_block_slot = self.db.get_latest_block_slot() - for slot, sign, tx in self.transaction_receipts.get_trxs(self.indexed_slot, reverse=False): + for slot, sign, tx in self.transaction_receipts.get_txs(self.indexed_slot): if slot > last_block_slot: break @@ -819,23 +823,37 @@ def unlock_accounts(self, tx) -> bool: self.warning(f"Transaction {tx.neon_tx} hasn't blocked accounts.") return False - storage_accounts_list = get_accounts_from_storage(self.solana, tx.storage_account) - if storage_accounts_list is None: - self.warning(f"Transaction {tx.neon_tx} has empty storage.") + storage = self.solana.get_storage_account_info(tx.storage_account) + if not storage: + self.warning(f"Storage {str(tx.storage_account)} for tx {tx.neon_tx.sign} is empty") + return False + + if storage.caller != tx.neon_tx.addr[2:]: + self.warning(f"Storage {str(tx.storage_account)} for tx {tx.neon_tx.sign} has another caller: "+ + f"{str(storage.caller)} != {tx.neon_tx.addr[2:]}") + return False + + if storage.caller != int(tx.neon_tx.nonce[2:]): + self.warning(f"Storage {str(tx.storage_account)} for tx {tx.neon_tx.sign} has another nonce: " + + f"0x{hex(storage.nonce)} != {tx.neon_tx.nonce}") + return False + + if not len(storage.account_list): + self.warning(f"Storage {str(tx.storage_account)} for tx {tx.neon_tx.sign} has empty account list.") return False - if len(storage_accounts_list) != len(tx.blocked_accounts): + if len(storage.account_list) != len(tx.blocked_accounts): self.warning(f"Transaction {tx.neon_tx} has another list of accounts than storage.") return False - for (writable, account), (idx, tx_account) in zip(storage_accounts_list, enumerate(tx.blocked_accounts)): + for (writable, account), (idx, tx_account) in zip(storage.account_list, enumerate(tx.blocked_accounts)): if account != tx_account: self.warning(f"Transaction {tx.neon_tx} has another list of accounts than storage: " + f"{idx}: {account} != {tx_account}") return False - self.debug(f'Neon tx is blocked: storage {tx.storage_account}, {tx.neon_tx}, {storage_accounts_list}') - self.blocked_storages[tx.storage_account] = (tx.neon_tx, storage_accounts_list) + self.debug(f'Neon tx is blocked: storage {tx.storage_account}, {tx.neon_tx}, {storage.account_list}') + self.blocked_storages[tx.storage_account] = (tx.neon_tx, storage.account_list) tx.canceled = True return True diff --git a/proxy/indexer/indexer_base.py b/proxy/indexer/indexer_base.py index 79709416f..e5bd18e44 100644 --- a/proxy/indexer/indexer_base.py +++ b/proxy/indexer/indexer_base.py @@ -5,7 +5,7 @@ from logged_groups import logged_group from typing import Optional -from .trx_receipts_storage import TrxReceiptsStorage +from .trx_receipts_storage import TxReceiptsStorage from .utils import MetricsToLogBuff from ..common_neon.solana_interactor import SolanaInteractor from ..indexer.sql_dict import SQLDict @@ -20,7 +20,7 @@ def __init__(self, solana: SolanaInteractor, last_slot: int): self.solana = solana - self.transaction_receipts = TrxReceiptsStorage('transaction_receipts') + self.transaction_receipts = TxReceiptsStorage('solana_transaction_receipts') self.last_slot = self._init_last_slot('receipt', last_slot) self.current_slot = 0 self.counter_ = 0 @@ -92,7 +92,7 @@ def process_functions(self): def gather_unknown_transactions(self): start_time = time.time() - poll_txs = set() + poll_txs = [] minimal_tx = None maximum_tx = None @@ -117,20 +117,27 @@ def gather_unknown_transactions(self): gathered_signatures += len_results counter += 1 + tx_idx = 0 + prev_slot = 0 + for tx in results: - solana_signature = tx["signature"] + sol_sign = tx["signature"] slot = tx["slot"] + if slot != prev_slot: + tx_idx = 0 + prev_slot = slot + if slot < self.last_slot: continue_flag = False break - if solana_signature in [HISTORY_START, self._maximum_tx]: + if sol_sign in [HISTORY_START, self._maximum_tx]: continue_flag = False break - if not self.transaction_receipts.contains(slot, solana_signature): - poll_txs.add(solana_signature) + if not self.transaction_receipts.contains(slot, sol_sign): + poll_txs.append((sol_sign, slot, tx_idx)) pool = ThreadPool(PARALLEL_REQUESTS) pool.map(self._get_tx_receipts, poll_txs) @@ -154,14 +161,15 @@ def _get_signatures(self, before: Optional[str], limit: int) -> []: self.warning(f'Fail to get signatures: {error}') return result - def _get_tx_receipts(self, solana_signature): - # trx = None + def _get_tx_receipts(self, param): + # tx = None retry = RETRY_ON_FAIL_ON_GETTING_CONFIRMED_TRANSACTION + (sol_sign, slot, tx_idx) = param while retry > 0: try: - trx = self.solana.get_confirmed_transaction(solana_signature)['result'] - self._add_trx(solana_signature, trx) + tx = self.solana.get_confirmed_transaction(sol_sign)['result'] + self._add_tx(sol_sign, tx, slot, tx_idx) retry = 0 except Exception as err: self.debug(f'Exception on get_confirmed_transaction: "{err}"') @@ -174,15 +182,16 @@ def _get_tx_receipts(self, solana_signature): if self.counter_ % 100 == 0: self.debug(f"Acquired {self.counter_} receipts") - def _add_trx(self, solana_signature, trx): - if trx is not None: + def _add_tx(self, sol_sign, tx, slot, tx_idx): + if tx is not None: add = False - for instruction in trx['transaction']['message']['instructions']: - if trx["transaction"]["message"]["accountKeys"][instruction["programIdIndex"]] == EVM_LOADER_ID: + msg = tx['transaction']['message'] + for instruction in msg['instructions']: + if msg["accountKeys"][instruction["programIdIndex"]] == EVM_LOADER_ID: add = True if add: - self.debug((trx['slot'], solana_signature)) - self.transaction_receipts.add_trx(trx['slot'], solana_signature, trx) + self.debug((slot, tx_idx, sol_sign)) + self.transaction_receipts.add_tx(slot, tx_idx, sol_sign, tx) else: - self.debug(f"trx is None {solana_signature}") + self.debug(f"trx is None {sol_sign}") diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index ab9de4c79..6c75237f1 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -25,18 +25,33 @@ def __init__(self, solana: SolanaInteractor): self._txs_db = NeonTxsDB() self._account_db = NeonAccountDB() self._solana = solana + self._block = SolanaBlockInfo(slot=0) + self._tx_idx = 0 + self._starting_block = SolanaBlockInfo(slot=0) self._constants = SQLDict(tablename="constants") - for k in ['min_receipt_slot', 'latest_slot']: + for k in ['min_receipt_slot', 'latest_slot', 'starting_slot']: if k not in self._constants: self._constants[k] = 0 def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo]): try: - block = self.get_block_by_slot(neon_res.slot) + block = self._block + if block.slot != neon_res.slot: + block = self.get_block_by_slot(neon_res.slot) + self._tx_idx = 0 if block.hash is None: self.critical(f'Unable to submit transaction {neon_tx.sign} because slot {neon_res.slot} not found') return + self._block = block + if not self._starting_block.slot: + if self._constants['starting_slot'] == 0: + self._constants['starting_slot'] = block.slot + self._starting_block = block + else: + self.get_starting_block() + neon_tx.tx_idx = self._tx_idx + self._tx_idx += 1 self.debug(f'{neon_tx} {neon_res} {block}') neon_res.fill_block_info(block) self._logs_db.push_logs(neon_res.logs, block) @@ -90,7 +105,23 @@ def get_full_block_by_slot(self, slot) -> SolanaBlockInfo: return block def get_latest_block(self) -> SolanaBlockInfo: - return SolanaBlockInfo(slot=self._constants['latest_slot']) + slot = self._constants['latest_slot'] + if slot == 0: + SolanaBlockInfo(slot=0) + return self.get_block_by_slot(slot) + + def get_latest_block_slot(self) -> int: + return self._constants['latest_slot'] + + def get_starting_block(self) -> SolanaBlockInfo: + if self._starting_block.slot != 0: + return self._starting_block + + slot = self._constants['starting_slot'] + if slot == 0: + SolanaBlockInfo(slot=0) + self._starting_block = self.get_block_by_slot(slot) + return self._starting_block def set_latest_block(self, slot: int): self._constants['latest_slot'] = slot diff --git a/proxy/indexer/transactions_db.py b/proxy/indexer/transactions_db.py index 64199baf1..66d0fd89b 100644 --- a/proxy/indexer/transactions_db.py +++ b/proxy/indexer/transactions_db.py @@ -42,9 +42,9 @@ def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: class NeonTxsDB(BaseDB): def __init__(self): BaseDB.__init__(self, 'neon_transactions') - self._column_lst = ('neon_sign', 'from_addr', 'sol_sign', 'slot', 'block_hash', 'idx', + self._column_lst = ['neon_sign', 'from_addr', 'sol_sign', 'slot', 'block_hash', 'idx', 'tx_idx', 'nonce', 'gas_price', 'gas_limit', 'to_addr', 'contract', 'value', 'calldata', - 'v', 'r', 's', 'status', 'gas_used', 'return_value', 'logs') + 'v', 'r', 's', 'status', 'gas_used', 'return_value', 'logs'] self._sol_neon_txs_db = SolanaNeonTxsDB() def _tx_from_value(self, value) -> Optional[NeonTxFullInfo]: diff --git a/proxy/indexer/trx_receipts_storage.py b/proxy/indexer/trx_receipts_storage.py index e9a6afe73..2c5c9d4c3 100644 --- a/proxy/indexer/trx_receipts_storage.py +++ b/proxy/indexer/trx_receipts_storage.py @@ -2,7 +2,7 @@ from proxy.indexer.base_db import BaseDB -class TrxReceiptsStorage(BaseDB): +class TxReceiptsStorage(BaseDB): def __init__(self, table_name): BaseDB.__init__(self, table_name) @@ -16,25 +16,25 @@ def size(self): rows = cur.fetchone()[0] return rows if rows is not None else 0 - def max_known_trx(self): + def max_known_tx(self): with self._conn.cursor() as cur: - cur.execute(f'SELECT slot, signature FROM {self._table_name} ORDER BY slot DESC, signature DESC LIMIT 1') + cur.execute(f'SELECT slot, signature FROM {self._table_name} ORDER BY slot DESC, tx_idx ASC LIMIT 1') row = cur.fetchone() if row is not None: - return (row[0], row[1]) - return (0, None) #table empty - return default value + return row[0], row[1] + return 0, None # table empty - return default value - def add_trx(self, slot, signature, trx): - bin_trx = encode(trx) + def add_tx(self, slot, tx_idx, signature, tx): + bin_tx = encode(tx) with self._conn.cursor() as cur: cur.execute(f''' - INSERT INTO {self._table_name} (slot, signature, trx) - VALUES ({slot},%s,%s) + INSERT INTO {self._table_name} (slot, tx_idx, signature, tx) + VALUES (%s, %s, %s, %s) ON CONFLICT (slot, signature) DO UPDATE SET - trx = EXCLUDED.trx + tx = EXCLUDED.tx ''', - (signature, bin_trx) + (slot, tx_idx, signature, bin_tx) ) def contains(self, slot, signature): @@ -42,10 +42,10 @@ def contains(self, slot, signature): cur.execute(f'SELECT 1 FROM {self._table_name} WHERE slot = %s AND signature = %s', (slot, signature,)) return cur.fetchone() is not None - def get_trxs(self, start_slot = 0, reverse = False): - order = 'DESC' if reverse else 'ASC' + def get_txs(self, start_slot=0): with self._conn.cursor() as cur: - cur.execute(f'SELECT slot, signature, trx FROM {self._table_name} WHERE slot >= {start_slot} ORDER BY slot {order}') + cur.execute(f'SELECT slot, signature, tx FROM {self._table_name}' + + f' WHERE slot >= {start_slot} ORDER BY slot ASC, tx_idx DESC') rows = cur.fetchall() for row in rows: yield int(row[0]), row[1], decode(row[2]) diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index d42bbc9e8..ee5dc3397 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -39,35 +39,6 @@ def get_req_id(self): return f"{self.idx}{self.sign}"[:7] -@logged_group("neon.Indexer") -def get_accounts_from_storage(solana: SolanaInteractor, storage_account, *, logger): - info = solana.get_account_info(storage_account, length=0) - # logger.debug("\n{}".format(json.dumps(result, indent=4, sort_keys=True))) - - if info is None: - raise Exception(f"Can't get information about {storage_account}") - - if info.tag == 30: - logger.debug("Not empty storage") - - acc_list = [] - storage = STORAGE_ACCOUNT_INFO_LAYOUT.parse(info.data[1:]) - offset = 1 + STORAGE_ACCOUNT_INFO_LAYOUT.sizeof() - for _ in range(storage.accounts_len): - writable = (info.data[offset] > 0) - offset += 1 - - some_pubkey = PublicKey(info.data[offset:offset + 32]) - offset += 32 - - acc_list.append((writable, str(some_pubkey))) - - return acc_list - else: - logger.debug("Empty") - return None - - @logged_group("neon.Indexer") def get_accounts_by_neon_address(solana: SolanaInteractor, neon_address, *, logger): pda_address, _nonce = ether2program(neon_address) diff --git a/proxy/memdb/blocks_db.py b/proxy/memdb/blocks_db.py index 6ab5b7cc9..6d51d7176 100644 --- a/proxy/memdb/blocks_db.py +++ b/proxy/memdb/blocks_db.py @@ -48,7 +48,7 @@ def execute(self) -> bool: return False def _get_latest_db_block(self): - self.latest_db_block_slot = self._b.db.get_latest_block().slot + self.latest_db_block_slot = self._b.db.get_latest_block_slot() latest_solana_block_slot = self._b.solana.get_recent_blockslot(commitment=FINALIZED) if not self.latest_db_block_slot or (latest_solana_block_slot - self.latest_db_block_slot > 300): self.latest_db_block_slot = latest_solana_block_slot @@ -254,6 +254,9 @@ def get_latest_block_slot(self) -> int: self._update_block_dicts() return self._latest_block.slot + def get_staring_block_slot(self) -> int: + return self.db.get_starting_block().slot + def get_db_block_slot(self) -> int: self._update_block_dicts() return self._latest_db_block_slot @@ -278,12 +281,18 @@ def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: @staticmethod def generate_fake_block(block_slot: int, block_time=1) -> SolanaBlockInfo: - # TODO: return predictable information about block hashes and block time + def slot_hash(slot: int): + hex_num = hex(slot)[2:] + num_len = len(hex_num) + hex_num = '00' + hex_num.rjust(((num_len >> 1) + (num_len % 2)) << 1, '0') + return '0x' + hex_num.rjust(64, 'f') + + # TODO: return predictable information about block time return SolanaBlockInfo( slot=block_slot, time=(block_time or 1), - hash='0x' + os.urandom(32).hex(), - parent_hash='0x' + os.urandom(32).hex(), + hash=slot_hash(block_slot), + parent_hash=slot_hash(block_slot - 1), is_fake=True ) diff --git a/proxy/memdb/memdb.py b/proxy/memdb/memdb.py index 3b059a56e..0eaaba9e4 100644 --- a/proxy/memdb/memdb.py +++ b/proxy/memdb/memdb.py @@ -22,7 +22,7 @@ def __init__(self, solana: SolanaInteractor): self._pending_tx_db = MemPendingTxsDB(self._db) def _before_slot(self) -> int: - return self._blocks_db.get_db_block_slot() + return self._blocks_db.get_db_block_slot() - 5 def get_latest_block(self) -> SolanaBlockInfo: return self._blocks_db.get_latest_block() @@ -30,6 +30,9 @@ def get_latest_block(self) -> SolanaBlockInfo: def get_latest_block_slot(self) -> int: return self._blocks_db.get_latest_block_slot() + def get_starting_block_slot(self) -> int: + return self._blocks_db.get_staring_block_slot() + def get_full_block_by_slot(self, block_slot: int) -> SolanaBlockInfo: return self._blocks_db.get_full_block_by_slot(block_slot) @@ -39,10 +42,10 @@ def get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: def pend_transaction(self, tx: NeonPendingTxInfo): self._pending_tx_db.pend_transaction(tx, self._before_slot()) - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo): + def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, sign_list: [str]): block = self._blocks_db.submit_block(neon_res) neon_res.fill_block_info(block) - self._txs_db.submit_transaction(neon_tx, neon_res, self._before_slot()) + self._txs_db.submit_transaction(neon_tx, neon_res, sign_list, self._before_slot()) def get_tx_list_by_sol_sign(self, is_finalized: bool, sol_sign_list: [str]) -> [NeonTxFullInfo]: if (not sol_sign_list) or (not len(sol_sign_list)): @@ -61,4 +64,6 @@ def get_contract_code(self, address: str) -> str: return self._db.get_contract_code(address) def get_sol_sign_list_by_neon_sign(self, neon_sign: str) -> [str]: - return self._db.get_sol_sign_list_by_neon_sign(neon_sign) + before_slot = self._before_slot() + is_pended_tx = self._pending_tx_db.is_exist(neon_sign, before_slot) + return self._txs_db.get_sol_sign_list_by_neon_sign(neon_sign, is_pended_tx, before_slot) diff --git a/proxy/memdb/transactions_db.py b/proxy/memdb/transactions_db.py index 2a26f5db7..cb220314f 100644 --- a/proxy/memdb/transactions_db.py +++ b/proxy/memdb/transactions_db.py @@ -101,8 +101,19 @@ def _has_topics(src_topics, dst_topics): return result_list + self._db.get_logs(from_block, to_block, addresses, topics, block_hash) - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, before_slot: int): - tx = NeonTxFullInfo(neon_tx=neon_tx, neon_res=neon_res) + def get_sol_sign_list_by_neon_sign(self, neon_sign: str, is_pended_tx: bool, before_slot: int) -> [str]: + if not is_pended_tx: + return self._db.get_sol_sign_list_by_neon_sign(neon_sign) + + with self._tx_slot.get_lock(): + self._rm_finalized_txs(before_slot) + data = self._tx_by_neon_sign.get(neon_sign) + if data: + return pickle.loads(data).used_ixs + return None + + def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, sign_list: [str], before_slot: int): + tx = NeonTxFullInfo(neon_tx=neon_tx, neon_res=neon_res, used_ixs=sign_list) data = pickle.dumps(tx) with self._tx_slot.get_lock(): diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 85bee8dfa..83771b33c 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -17,7 +17,7 @@ import sha3 from logged_groups import logged_group, logging_context -from typing import Optional +from typing import Optional, Union from ..common.utils import build_http_response from ..http.codes import httpStatusCodes @@ -46,7 +46,7 @@ modelInstanceLock = threading.Lock() modelInstance = None -NEON_PROXY_PKG_VERSION = '0.7.5-dev' +NEON_PROXY_PKG_VERSION = '0.7.7-dev' NEON_PROXY_REVISION = 'NEON_PROXY_REVISION_TO_BE_REPLACED' @@ -71,19 +71,24 @@ def __init__(self): self.debug(f"Worker id {self.proxy_id}") - def neon_proxy_version(self): + @staticmethod + def neon_proxy_version(): return 'Neon-proxy/v' + NEON_PROXY_PKG_VERSION + '-' + NEON_PROXY_REVISION - def web3_clientVersion(self): + @staticmethod + def web3_clientVersion(): return 'Neon/v' + NEON_EVM_VERSION + '-' + NEON_EVM_REVISION - def eth_chainId(self): + @staticmethod + def eth_chainId(): return hex(int(CHAIN_ID)) - def neon_cli_version(self): + @staticmethod + def neon_cli_version(): return neon_cli().version() - def net_version(self): + @staticmethod + def net_version(): return str(CHAIN_ID) def eth_gasPrice(self): @@ -105,24 +110,38 @@ def eth_estimateGas(self, param): def __repr__(self): return str(self.__dict__) - def process_block_tag(self, tag) -> SolanaBlockInfo: - if tag == "latest": + def _process_block_tag(self, tag) -> SolanaBlockInfo: + if tag in ("latest", "pending"): block = self._db.get_latest_block() - elif tag in ('earliest', 'pending'): - raise EthereumError(message=f"Invalid tag {tag}") + elif tag in ('earliest'): + raise EthereumError(message=f"invalid tag {tag}") elif isinstance(tag, str): try: block = SolanaBlockInfo(slot=int(tag.strip(), 16)) except: - raise EthereumError(message=f'Failed to parse block tag: {tag}') + raise EthereumError(message=f'failed to parse block tag: {tag}') elif isinstance(tag, int): block = SolanaBlockInfo(slot=tag) else: - raise EthereumError(message=f'Failed to parse block tag: {tag}') + raise EthereumError(message=f'failed to parse block tag: {tag}') return block - def _getFullBlockByNumber(self, tag) -> SolanaBlockInfo: - block = self.process_block_tag(tag) + @staticmethod + def _validate_tx_id(tag: str) -> Optional[str]: + try: + if not isinstance(tag, str): + return 'bad transaction-id format' + tag = tag.lower().strip() + assert len(tag) == 66 + assert tag[:2] == '0x' + + int(tag[2:], 16) + return None + except: + return 'transaction-id is not hex' + + def _get_full_block_by_number(self, tag) -> SolanaBlockInfo: + block = self._process_block_tag(tag) if block.slot is None: self.debug(f"Not found block by number {tag}") return block @@ -142,13 +161,17 @@ def eth_getBalance(self, account, tag) -> str: """account - address to check for balance. tag - integer block number, or the string "latest", "earliest" or "pending" """ + if tag not in ("latest", "pending"): + self.debug(f"Block type '{tag}' is not supported yet") + raise EthereumError(message=f"Not supported block identifier: {tag}") + self.debug(f'eth_getBalance: {account}') try: - acc_info = self._solana.get_account_info_layout(EthereumAddress(account)) - if acc_info is None: + neon_account_info = self._solana.get_neon_account_info(EthereumAddress(account)) + if neon_account_info is None: return hex(0) - return hex(acc_info.balance) + return hex(neon_account_info.balance) except Exception as err: self.debug(f"eth_getBalance: Can't get account info: {err}") return hex(0) @@ -168,9 +191,9 @@ def to_list(items): block_hash = None if 'fromBlock' in obj and obj['fromBlock'] != '0': - from_block = self.process_block_tag(obj['fromBlock']).slot - if 'toBlock' in obj and obj['toBlock'] != 'latest': - to_block = self.process_block_tag(obj['toBlock']).slot + from_block = self._process_block_tag(obj['fromBlock']).slot + if 'toBlock' in obj and obj['toBlock'] not in ('latest', 'pending'): + to_block = self._process_block_tag(obj['toBlock']).slot if 'address' in obj: addresses = to_list(obj['address']) if 'topics' in obj: @@ -180,7 +203,7 @@ def to_list(items): return self._db.get_logs(from_block, to_block, addresses, topics, block_hash) - def getBlockBySlot(self, block: SolanaBlockInfo, full, skip_transaction): + def _get_block_by_slot(self, block: SolanaBlockInfo, full, skip_transaction) -> Optional[dict]: if block.is_empty(): block = self._db.get_full_block_by_slot(block.slot) if block.is_empty(): @@ -188,7 +211,6 @@ def getBlockBySlot(self, block: SolanaBlockInfo, full, skip_transaction): sign_list = [] gas_used = 0 - tx_index = 0 if skip_transaction: tx_list = [] else: @@ -198,9 +220,7 @@ def getBlockBySlot(self, block: SolanaBlockInfo, full, skip_transaction): gas_used += int(tx.neon_res.gas_used, 16) if full: - receipt = self._getTransaction(tx) - receipt['transactionIndex'] = hex(tx_index) - tx_index += 1 + receipt = self._get_transaction(tx) sign_list.append(receipt) else: sign_list.append(tx.neon_tx.sign) @@ -221,7 +241,7 @@ def eth_getStorageAt(self, account, position, block_identifier): '''Retrieves storage data by given position Currently supports only 'latest' block ''' - if block_identifier != "latest": + if block_identifier not in ("latest", "pending"): self.debug(f"Block type '{block_identifier}' is not supported yet") raise EthereumError(message=f"Not supported block identifier: {block_identifier}") @@ -232,9 +252,11 @@ def eth_getStorageAt(self, account, position, block_identifier): self.error(f"eth_getStorageAt: Neon-cli failed to execute: {err}") return '0x00' - def _getBlockByHash(self, block_hash: str) -> SolanaBlockInfo: + def _get_block_by_hash(self, block_hash: str) -> SolanaBlockInfo: try: - block_hash = block_hash.lower() + block_hash = block_hash.strip().lower() + assert block_hash[:2] == '0x' + bin_block_hash = bytes.fromhex(block_hash[2:]) assert len(bin_block_hash) == 32 except: @@ -246,27 +268,27 @@ def _getBlockByHash(self, block_hash: str) -> SolanaBlockInfo: return block - def eth_getBlockByHash(self, block_hash: str, full): + def eth_getBlockByHash(self, block_hash: str, full: bool) -> Optional[dict]: """Returns information about a block by hash. block_hash - Hash of a block. full - If true it returns the full transaction objects, if false only the hashes of the transactions. """ - block = self._getBlockByHash(block_hash) + block = self._get_block_by_hash(block_hash) if block.slot is None: return None - ret = self.getBlockBySlot(block, full, False) + ret = self._get_block_by_slot(block, full, False) return ret - def eth_getBlockByNumber(self, tag, full): + def eth_getBlockByNumber(self, tag, full) -> Optional[dict]: """Returns information about a block by block number. tag - integer of a block number, or the string "earliest", "latest" or "pending", as in the default block parameter. full - If true it returns the full transaction objects, if false only the hashes of the transactions. """ - block = self.process_block_tag(tag) + block = self._process_block_tag(tag) if block.slot is None: self.debug(f"Not found block by number {tag}") return None - ret = self.getBlockBySlot(block, full, tag == 'latest') + ret = self._get_block_by_slot(block, full, tag in ('latest', 'pending')) return ret def eth_call(self, obj, tag): @@ -281,6 +303,10 @@ def eth_call(self, obj, tag): data: DATA - (optional) Hash of the method signature and encoded parameters. For details see Ethereum Contract ABI in the Solidity documentation tag - integer block number, or the string "latest", "earliest" or "pending", see the default block parameter """ + if tag not in ("latest", "pending"): + self.debug(f"Block type '{tag}' is not supported yet") + raise EthereumError(message=f"Not supported block identifier: {tag}") + if not obj['data']: raise EthereumError(message="Missing data") try: caller_id = obj.get('from', "0x0000000000000000000000000000000000000000") @@ -295,17 +321,22 @@ def eth_call(self, obj, tag): raise def eth_getTransactionCount(self, account, tag): + if tag not in ("latest", "pending"): + self.debug(f"Block type '{tag}' is not supported yet") + raise EthereumError(message=f"Not supported block identifier: {tag}") + try: - acc_info = self._solana.get_account_info_layout(EthereumAddress(account)) - return hex(acc_info.trx_count) + neon_account_info = self._solana.get_neon_account_info(EthereumAddress(account)) + return hex(neon_account_info.trx_count) except Exception as err: self.debug(f"eth_getTransactionCount: Can't get account info: {err}") return hex(0) - def _getTransactionReceipt(self, tx): + @staticmethod + def _get_transaction_receipt(tx) -> dict: result = { "transactionHash": tx.neon_tx.sign, - "transactionIndex": hex(0), + "transactionIndex": hex(tx.neon_tx.tx_idx), "blockHash": tx.neon_res.block_hash, "blockNumber": hex(tx.neon_res.slot), "from": tx.neon_tx.addr, @@ -320,17 +351,20 @@ def _getTransactionReceipt(self, tx): return result - def eth_getTransactionReceipt(self, trxId): - self.debug('eth_getTransactionReceipt: %s', trxId) + def eth_getTransactionReceipt(self, NeonTxId: str) -> Optional[dict]: + error = self._validate_tx_id(NeonTxId) + if error: + raise EthereumError(message=error) + neon_sign = NeonTxId.strip().lower() - neon_sign = trxId.lower() tx = self._db.get_tx_by_neon_sign(neon_sign) if not tx: self.debug("Not found receipt") return None - return self._getTransactionReceipt(tx) + return self._get_transaction_receipt(tx) - def _getTransaction(self, tx): + @staticmethod + def _get_transaction(tx) -> dict: t = tx.neon_tx r = tx.neon_res @@ -338,7 +372,7 @@ def _getTransaction(self, tx): "blockHash": r.block_hash, "blockNumber": hex(r.slot), "hash": t.sign, - "transactionIndex": hex(0), + "transactionIndex": hex(t.tx_idx), "from": t.addr, "nonce": t.nonce, "gasPrice": t.gas_price, @@ -353,15 +387,17 @@ def _getTransaction(self, tx): return result - def eth_getTransactionByHash(self, trxId): - self.debug('eth_getTransactionByHash: %s', trxId) + def eth_getTransactionByHash(self, NeontxId: str) -> Optional[dict]: + error = self._validate_tx_id(NeontxId) + if error: + raise EthereumError(message=error) - neon_sign = trxId.lower() + neon_sign = NeontxId.strip().lower() tx = self._db.get_tx_by_neon_sign(neon_sign) if tx is None: self.debug("Not found receipt") return None - return self._getTransaction(tx) + return self._get_transaction(tx) def eth_getCode(self, account, _tag): account = account.lower() @@ -386,10 +422,6 @@ def eth_sendRawTransaction(self, rawTrx): except PendingTxError as err: self.debug(f'{err}') return eth_signature - except SolTxError as err: - err_msg = json.dumps(err.result, indent=3) - self.error(f"Got SendTransactionError: {err_msg}") - raise except EthereumError as err: # self.debug(f"eth_sendRawTransaction EthereumError: {err}") raise @@ -397,7 +429,7 @@ def eth_sendRawTransaction(self, rawTrx): # self.error(f"eth_sendRawTransaction type(err): {type(err}}, Exception: {err}") raise - def _getTransactionByIndex(self, block: SolanaBlockInfo, tx_idx: int) -> Optional[dict]: + def _get_transaction_by_index(self, block: SolanaBlockInfo, tx_idx: int) -> Optional[dict]: try: if isinstance(tx_idx, str): tx_idx = int(tx_idx, 16) @@ -415,24 +447,24 @@ def _getTransactionByIndex(self, block: SolanaBlockInfo, tx_idx: int) -> Optiona if tx_idx >= len(tx_list): return None - return self._getTransaction(tx_list[tx_idx]) + return self._get_transaction(tx_list[tx_idx]) def eth_getTransactionByBlockNumberAndIndex(self, tag: str, tx_idx: int) -> Optional[dict]: - block = self.process_block_tag(tag) + block = self._process_block_tag(tag) if block.is_empty(): self.debug(f"Not found block by number {tag}") return None - return self._getTransactionByIndex(block, tx_idx) + return self._get_transaction_by_index(block, tx_idx) def eth_getTransactionByBlockHashAndIndex(self, block_hash: str, tx_idx: int) -> Optional[dict]: - block = self._getBlockByHash(block_hash) + block = self._get_block_by_hash(block_hash) if block.is_empty(): return None - return self._getTransactionByIndex(block, tx_idx) + return self._get_transaction_by_index(block, tx_idx) def eth_getBlockTransactionCountByHash(self, block_hash: str) -> str: - block = self._getBlockByHash(block_hash) + block = self._get_block_by_hash(block_hash) if block.slot is None: return hex(0) if block.is_empty(): @@ -445,7 +477,7 @@ def eth_getBlockTransactionCountByHash(self, block_hash: str) -> str: return hex(len(tx_list)) def eth_getBlockTransactionCountByNumber(self, tag: str) -> str: - block = self._getFullBlockByNumber(tag) + block = self._get_full_block_by_number(tag) if block.is_empty(): return hex(0) @@ -543,8 +575,23 @@ def eth_hashrate() -> str: def eth_getWork() -> [str]: return ['', '', '', ''] - def eth_syncing(self) -> bool: - return self._solana.is_health() + def eth_syncing(self) -> Union[bool, dict]: + try: + slots_behind = self._solana.get_slots_behind() + latest_slot = self._db.get_latest_block_slot() + first_slot = self._db.get_starting_block_slot() + + self.debug(f'slots_behind: {slots_behind}, latest_slot: {latest_slot}, first_slot: {first_slot}') + if (slots_behind is None) or (latest_slot is None) or (first_slot is None): + return False + + return { + 'startingblock': first_slot, + 'currentblock': latest_slot, + 'highestblock': latest_slot + slots_behind + } + except: + return False def net_peerCount(self) -> str: cluster_node_list = self._solana.get_cluster_nodes() @@ -554,10 +601,11 @@ def net_peerCount(self) -> str: def net_listening() -> bool: return False - def neon_getSolanaTransactionByNeonTransaction(self, neonTxId: str) -> [str]: - if not isinstance(neonTxId, str): - return [] - return self._db.get_sol_sign_list_by_neon_sign(neonTxId) + def neon_getSolanaTransactionByNeonTransaction(self, NeonTxId: str) -> Union[str, list]: + error = self._validate_tx_id(NeonTxId) + if error: + raise EthereumError(message=error) + return self._db.get_sol_sign_list_by_neon_sign(NeonTxId.strip().lower()) class JsonEncoder(json.JSONEncoder): @@ -603,7 +651,11 @@ def process_request(self, request): 'jsonrpc': '2.0', 'id': request.get('id', None), } + def is_private_api(method: str) -> bool: + if method.startswith('_'): + return True + if ENABLE_PRIVATE_API: return False diff --git a/proxy/testing/test_trx_receipts_storage.py b/proxy/testing/test_trx_receipts_storage.py index bd141516d..b45e10e64 100644 --- a/proxy/testing/test_trx_receipts_storage.py +++ b/proxy/testing/test_trx_receipts_storage.py @@ -1,13 +1,13 @@ from unittest import TestCase -from proxy.indexer.trx_receipts_storage import TrxReceiptsStorage +from proxy.indexer.trx_receipts_storage import TxReceiptsStorage from random import randint from base58 import b58encode -class TestTrxReceiptsStorage(TestCase): +class TestTxReceiptsStorage(TestCase): @classmethod def setUpClass(cls) -> None: - cls.trx_receipts_storage = TrxReceiptsStorage('test_storage') + cls.tx_receipts_storage = TxReceiptsStorage('test_storage') def create_signature(self): signature = b'' @@ -23,48 +23,43 @@ def test_data_consistency(self): """ Test that data put into container is stored there """ - self.trx_receipts_storage.clear() - self.assertEqual(self.trx_receipts_storage.size(), 0) - self.assertEqual(self.trx_receipts_storage.max_known_trx(), (0, None)) + self.tx_receipts_storage.clear() + self.assertEqual(self.tx_receipts_storage.size(), 0) + self.assertEqual(self.tx_receipts_storage.max_known_tx(), (0, None)) max_slot = 10 num_items = 100 expected_items = [] - for _ in range(0, num_items): + for idx in range(0, num_items): slot, signature = self.create_slot_sig(max_slot) - trx = { 'slot': slot, 'signature': signature } - self.trx_receipts_storage.add_trx(slot, signature, trx) - expected_items.append((slot, signature, trx)) + tx = {'slot': slot, 'signature': signature} + self.tx_receipts_storage.add_tx(slot, idx, signature, tx) + expected_items.append((slot, idx, signature, tx)) - self.assertEqual(self.trx_receipts_storage.max_known_trx()[0], max_slot) - self.assertEqual(self.trx_receipts_storage.size(), num_items) + self.assertEqual(self.tx_receipts_storage.max_known_tx()[0], max_slot) + self.assertEqual(self.tx_receipts_storage.size(), num_items) for item in expected_items: - self.assertTrue(self.trx_receipts_storage.contains(item[0], item[1])) + self.assertTrue(self.tx_receipts_storage.contains(item[0], item[2])) def test_query(self): """ - Test get_trxs method workds as expected + Test get_txs method works as expected """ - self.trx_receipts_storage.clear() - self.assertEqual(self.trx_receipts_storage.size(), 0) + self.tx_receipts_storage.clear() + self.assertEqual(self.tx_receipts_storage.size(), 0) max_slot = 50 num_items = 100 expected_items = [] - for _ in range(0, num_items): + for idx in range(0, num_items): slot, signature = self.create_slot_sig(max_slot) - trx = { 'slot': slot, 'signature': signature } - self.trx_receipts_storage.add_trx(slot, signature, trx) - expected_items.append((slot, signature, trx)) + trx = {'slot': slot, 'signature': signature} + self.tx_receipts_storage.add_tx(slot, idx, signature, trx) + expected_items.append((slot, idx, signature, trx)) start_slot = randint(0, 50) # query in ascending order - retrieved_trxs = [item for item in self.trx_receipts_storage.get_trxs(start_slot, False)] - self.assertGreaterEqual(retrieved_trxs[0][0], start_slot) - self.assertLessEqual(retrieved_trxs[-1][0], max_slot) - - # query in descending order - retrieved_trxs = [item for item in self.trx_receipts_storage.get_trxs(start_slot, True)] - self.assertLessEqual(retrieved_trxs[0][0], max_slot) - self.assertGreaterEqual(retrieved_trxs[-1][0], start_slot) + retrieved_txs = [item for item in self.tx_receipts_storage.get_txs(start_slot)] + self.assertGreaterEqual(retrieved_txs[0][0], start_slot) + self.assertLessEqual(retrieved_txs[-1][0], max_slot)