From 59acba247eaea270a2b7b4d7409046c0dde65500 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 7 Feb 2022 10:55:50 +0300 Subject: [PATCH 1/5] #492 use neon_instruction and solana_interactor in indexer --- proxy/common_neon/neon_instruction.py | 14 +++-- proxy/docker-compose-test.yml | 1 + proxy/indexer/canceller.py | 69 ++++++++++++++++++++++++ proxy/indexer/indexer.py | 6 ++- proxy/indexer/utils.py | 78 +++------------------------ 5 files changed, 89 insertions(+), 79 deletions(-) create mode 100644 proxy/indexer/canceller.py diff --git a/proxy/common_neon/neon_instruction.py b/proxy/common_neon/neon_instruction.py index de5a6a902..4bdf73cba 100644 --- a/proxy/common_neon/neon_instruction.py +++ b/proxy/common_neon/neon_instruction.py @@ -267,7 +267,14 @@ def make_noniterative_call_transaction(self, length_before: int = 0) -> Transact trx.add(self.make_05_call_instruction()) return trx - def make_cancel_transaction(self) -> Transaction: + def make_cancel_transaction(self, cancel_keys = None) -> Transaction: + append_keys = [] + if cancel_keys: + append_keys = cancel_keys + else: + append_keys = self.eth_accounts + append_keys.append(AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False)) + append_keys += obligatory_accounts return Transaction().add(TransactionInstruction( program_id = EVM_LOADER_ID, data = bytearray.fromhex("15") + self.eth_trx.nonce.to_bytes(8, 'little'), @@ -279,10 +286,7 @@ def make_cancel_transaction(self) -> Transaction: AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False), - ] + self.eth_accounts + [ - - AccountMeta(pubkey=SYSVAR_INSTRUCTION_PUBKEY, is_signer=False, is_writable=False), - ] + obligatory_accounts + ] + append_keys )) def make_partial_call_or_continue_instruction(self, steps=0) -> TransactionInstruction: diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 968219111..90cc15c8c 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -148,6 +148,7 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass CONFIG: ci + WRITE_TRANSACTION_COST_IN_DB: NO hostname: indexer depends_on: postgres: diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py new file mode 100644 index 000000000..04142d636 --- /dev/null +++ b/proxy/indexer/canceller.py @@ -0,0 +1,69 @@ + +import traceback +from solana.publickey import PublicKey +from solana.rpc.api import Client +from solana.transaction import AccountMeta +from spl.token.instructions import get_associated_token_address +from proxy.common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY +from proxy.common_neon.neon_instruction import NeonInstruction +from proxy.common_neon.solana_interactor import SolanaInteractor +from proxy.environment import ETH_TOKEN_MINT_ID, SOLANA_URL, get_solana_accounts +from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID, get_solana_accounts +from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY +from solana.system_program import SYS_PROGRAM_ID +from spl.token.constants import TOKEN_PROGRAM_ID +from logged_groups import logged_group + + +@logged_group("neon.Indexer") +class Canceller: + readonly_accs = [ + PublicKey(EVM_LOADER_ID), + PublicKey(ETH_TOKEN_MINT_ID), + PublicKey(TOKEN_PROGRAM_ID), + PublicKey(SYSVAR_CLOCK_PUBKEY), + PublicKey(SYSVAR_INSTRUCTION_PUBKEY), + PublicKey(KECCAK_PROGRAM), + PublicKey(SYSVAR_RENT_PUBKEY), + PublicKey(INCINERATOR_PUBKEY), + PublicKey(SYS_PROGRAM_ID), + ] + + def __init__(self): + # Initialize user account + self.signer = get_solana_accounts()[0] + self._operator = self.signer.public_key() + self._client = Client(SOLANA_URL) + self.operator_token = get_associated_token_address(PublicKey(self._operator), ETH_TOKEN_MINT_ID) + + self.solana = SolanaInteractor(self._client) + self.builder = NeonInstruction(self._operator) + + + def unlock_accounts(self, blocked_storages): + for storage, tx_accounts in blocked_storages.items(): + (neon_tx, blocked_accounts) = tx_accounts + if blocked_accounts is None: + self.error(f"Empty blocked accounts for the Neon tx {neon_tx}.") + else: + keys = [] + for acc in blocked_accounts: + is_writable = False if PublicKey(acc) in self.readonly_accs else True + keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=is_writable)) + + self.builder.init_eth_trx(neon_tx.tx, None, self.operator_token) + self.builder.init_iterative(storage, None, 0) + + trx = self.builder.make_cancel_transaction(keys) + + self.debug(f"Send Cancel: {trx}") + try: + cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], None, None)[0] + self.debug(f"cancel result: {cancel_result}") + except Exception as err: + err_tb = "".join(traceback.format_tb(err.__traceback__)) + self.error('Exception on submitting transaction. ' + + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') + else: + self.debug(f"Canceled: {blocked_accounts}") diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index d013f6dc9..6ba868631 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -8,13 +8,15 @@ try: from indexer_base import IndexerBase from indexer_db import IndexerDB - from utils import SolanaIxSignInfo, NeonTxResultInfo, NeonTxInfo, Canceller, str_fmt_object + from utils import SolanaIxSignInfo, NeonTxResultInfo, NeonTxInfo, str_fmt_object from utils import get_accounts_from_storage, check_error + from canceller import Canceller except ImportError: from .indexer_base import IndexerBase from .indexer_db import IndexerDB - from .utils import SolanaIxSignInfo, NeonTxResultInfo, NeonTxInfo, Canceller, str_fmt_object + from .utils import SolanaIxSignInfo, NeonTxResultInfo, NeonTxInfo, str_fmt_object from .utils import get_accounts_from_storage, check_error + from .canceller import Canceller from ..environment import EVM_LOADER_ID, FINALIZED, CANCEL_TIMEOUT, SOLANA_URL diff --git a/proxy/indexer/utils.py b/proxy/indexer/utils.py index cffaf6d58..0cfb8a778 100644 --- a/proxy/indexer/utils.py +++ b/proxy/indexer/utils.py @@ -5,31 +5,21 @@ import json import multiprocessing import psycopg2 -import subprocess -import traceback from eth_utils import big_endian_to_int -from solana.account import Account from solana.publickey import PublicKey from solana.rpc.api import Client from solana.rpc.commitment import Confirmed -from solana.rpc.types import TxOpts -from solana.system_program import SYS_PROGRAM_ID -from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY -from solana.transaction import AccountMeta, Transaction, TransactionInstruction -from spl.token.constants import TOKEN_PROGRAM_ID -from spl.token.instructions import get_associated_token_address from logged_groups import logged_group from ..common_neon.address import ether2program -from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM from ..common_neon.layouts import STORAGE_ACCOUNT_INFO_LAYOUT, CODE_ACCOUNT_INFO_LAYOUT, ACCOUNT_INFO_LAYOUT from ..common_neon.eth_proto import Trx as EthTx from ..common_neon.utils import get_from_dict -from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID, get_solana_accounts +from ..environment import EVM_LOADER_ID -from proxy.indexer.pg_common import POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST -from proxy.indexer.pg_common import encode, decode +from .pg_common import POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_HOST +from .pg_common import encode, decode basedb_lock_glob = multiprocessing.Lock() @@ -164,6 +154,7 @@ def __str__(self): return str_fmt_object(self) def _set_defaults(self): + self.tx = None self.addr = None self.sign = None self.nonce = None @@ -179,6 +170,8 @@ def _set_defaults(self): self.error = None def init_from_eth_tx(self, tx: EthTx): + self.tx = tx + self.v = hex(tx.v) self.r = hex(tx.r) self.s = hex(tx.s) @@ -467,62 +460,3 @@ def del_not_finalized(self, from_slot: int, to_slot: int): cursor.execute(f'DELETE FROM {self._table_name} WHERE slot >= %s AND slot <= %s AND finalized = false', (from_slot, to_slot)) - -@logged_group("neon.Indexer") -class Canceller: - def __init__(self): - # Initialize user account - self.signer = get_solana_accounts()[0] - - self.client = Client(SOLANA_URL) - - self.operator = self.signer.public_key() - self.operator_token = get_associated_token_address(PublicKey(self.operator), ETH_TOKEN_MINT_ID) - - def unlock_accounts(self, blocked_storages): - readonly_accs = [ - PublicKey(EVM_LOADER_ID), - PublicKey(ETH_TOKEN_MINT_ID), - PublicKey(TOKEN_PROGRAM_ID), - PublicKey(SYSVAR_CLOCK_PUBKEY), - PublicKey(SYSVAR_INSTRUCTION_PUBKEY), - PublicKey(KECCAK_PROGRAM), - PublicKey(SYSVAR_RENT_PUBKEY), - PublicKey(INCINERATOR_PUBKEY), - PublicKey(SYS_PROGRAM_ID), - ] - for storage, tx_accounts in blocked_storages.items(): - (neon_tx, blocked_accounts) = tx_accounts - if blocked_accounts is None: - self.error(f"Emtpy blocked accounts for the Neon tx {neon_tx}.") - - if blocked_accounts is not None: - keys = [ - AccountMeta(pubkey=storage, is_signer=False, is_writable=True), - AccountMeta(pubkey=self.operator, is_signer=True, is_writable=True), - AccountMeta(pubkey=self.operator_token, is_signer=False, is_writable=True), - AccountMeta(pubkey=blocked_accounts[4], is_signer=False, is_writable=True), - AccountMeta(pubkey=INCINERATOR_PUBKEY, is_signer=False, is_writable=True), - AccountMeta(pubkey=SYS_PROGRAM_ID, is_signer=False, is_writable=False) - ] - for acc in blocked_accounts: - is_writable = False if PublicKey(acc) in readonly_accs else True - keys.append(AccountMeta(pubkey=acc, is_signer=False, is_writable=is_writable)) - - trx = Transaction() - nonce = int(neon_tx.nonce, 16) - trx.add(TransactionInstruction( - program_id=EVM_LOADER_ID, - data=bytearray.fromhex("15") + nonce.to_bytes(8, 'little'), - keys=keys - )) - - self.debug("Send Cancel") - try: - self.client.send_transaction(trx, self.signer, opts=TxOpts(preflight_commitment=Confirmed)) - except Exception as err: - err_tb = "".join(traceback.format_tb(err.__traceback__)) - self.error('Exception on submitting transaction. ' + - f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - else: - self.debug(f"Canceled: {blocked_accounts}") From ea2c632deb89b5b1bd7720138cfd0e88d62af4a2 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 7 Feb 2022 11:08:32 +0300 Subject: [PATCH 2/5] fix --- proxy/docker-compose-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 90cc15c8c..27a2f7840 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -148,7 +148,7 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass CONFIG: ci - WRITE_TRANSACTION_COST_IN_DB: NO + WRITE_TRANSACTION_COST_IN_DB: Nope hostname: indexer depends_on: postgres: From 1e4b161307e4784c4ed751601cc20c4aa477b4c9 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Mon, 7 Feb 2022 19:23:49 +0300 Subject: [PATCH 3/5] #492 catch send cancel error --- proxy/indexer/canceller.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 04142d636..6da5ea8e4 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -10,6 +10,7 @@ from proxy.environment import ETH_TOKEN_MINT_ID, SOLANA_URL, get_solana_accounts from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID, get_solana_accounts from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM +from ..common_neon.utils import get_from_dict from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.system_program import SYS_PROGRAM_ID from spl.token.constants import TOKEN_PROGRAM_ID @@ -61,6 +62,9 @@ def unlock_accounts(self, blocked_storages): try: cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], None, None)[0] self.debug(f"cancel result: {cancel_result}") + result_error = get_from_dict(cancel_result, 'meta', 'err') + if result_error: + self.error(f'Error sending cancel transaction: {result_error}') except Exception as err: err_tb = "".join(traceback.format_tb(err.__traceback__)) self.error('Exception on submitting transaction. ' + From 185efe74e401d1f6d053decbd6223210319c8b8e Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 8 Feb 2022 17:03:38 +0300 Subject: [PATCH 4/5] fix merge errors #492 --- proxy/common_neon/utils.py | 3 +++ proxy/indexer/canceller.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/proxy/common_neon/utils.py b/proxy/common_neon/utils.py index 335c909bd..bd1c0dd3d 100644 --- a/proxy/common_neon/utils.py +++ b/proxy/common_neon/utils.py @@ -159,6 +159,7 @@ def __setstate__(self, src): self.__dict__ = src def _set_defaults(self): + self.tx = None self.addr = None self.sign = None self.nonce = None @@ -174,6 +175,8 @@ def _set_defaults(self): self.error = None def init_from_eth_tx(self, tx: EthTx): + self.tx = tx + self.v = hex(tx.v) self.r = hex(tx.r) self.s = hex(tx.s) diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 6da5ea8e4..02c38246e 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -1,20 +1,20 @@ import traceback + +from logged_groups import logged_group from solana.publickey import PublicKey from solana.rpc.api import Client +from solana.system_program import SYS_PROGRAM_ID +from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY from solana.transaction import AccountMeta +from spl.token.constants import TOKEN_PROGRAM_ID from spl.token.instructions import get_associated_token_address -from proxy.common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY + +from proxy.common_neon.constants import INCINERATOR_PUBKEY, KECCAK_PROGRAM, SYSVAR_INSTRUCTION_PUBKEY from proxy.common_neon.neon_instruction import NeonInstruction from proxy.common_neon.solana_interactor import SolanaInteractor -from proxy.environment import ETH_TOKEN_MINT_ID, SOLANA_URL, get_solana_accounts -from ..environment import SOLANA_URL, EVM_LOADER_ID, ETH_TOKEN_MINT_ID, get_solana_accounts -from ..common_neon.constants import SYSVAR_INSTRUCTION_PUBKEY, INCINERATOR_PUBKEY, KECCAK_PROGRAM -from ..common_neon.utils import get_from_dict -from solana.sysvar import SYSVAR_CLOCK_PUBKEY, SYSVAR_RENT_PUBKEY -from solana.system_program import SYS_PROGRAM_ID -from spl.token.constants import TOKEN_PROGRAM_ID -from logged_groups import logged_group +from proxy.common_neon.utils import get_from_dict +from proxy.environment import ETH_TOKEN_MINT_ID, EVM_LOADER_ID, SOLANA_URL, get_solana_accounts @logged_group("neon.Indexer") From 8bc5768030524b606e991b7071e0270e3a853e04 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Tue, 8 Feb 2022 18:51:37 +0300 Subject: [PATCH 5/5] Return write to costs table on send cancel #492 --- proxy/docker-compose-test.yml | 1 - proxy/indexer/canceller.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/proxy/docker-compose-test.yml b/proxy/docker-compose-test.yml index 27a2f7840..968219111 100644 --- a/proxy/docker-compose-test.yml +++ b/proxy/docker-compose-test.yml @@ -148,7 +148,6 @@ services: POSTGRES_USER: neon-proxy POSTGRES_PASSWORD: neon-proxy-pass CONFIG: ci - WRITE_TRANSACTION_COST_IN_DB: Nope hostname: indexer depends_on: postgres: diff --git a/proxy/indexer/canceller.py b/proxy/indexer/canceller.py index 02c38246e..6f9dfc15e 100644 --- a/proxy/indexer/canceller.py +++ b/proxy/indexer/canceller.py @@ -60,7 +60,7 @@ def unlock_accounts(self, blocked_storages): self.debug(f"Send Cancel: {trx}") try: - cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], None, None)[0] + cancel_result = self.solana.send_multiple_transactions(self.signer, [trx], neon_tx.tx, "CancelWithNonce")[0] self.debug(f"cancel result: {cancel_result}") result_error = get_from_dict(cancel_result, 'meta', 'err') if result_error: