From 80ccdd5557aa735bb1592801d3c89b490cb11d17 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Wed, 19 Jan 2022 23:46:00 +0300 Subject: [PATCH 1/6] #455 update finalized transactions --- proxy/indexer/transactions_db.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/proxy/indexer/transactions_db.py b/proxy/indexer/transactions_db.py index 96294c597..479db09c6 100644 --- a/proxy/indexer/transactions_db.py +++ b/proxy/indexer/transactions_db.py @@ -83,7 +83,7 @@ def _create_table_sql(self) -> str: calldata TEXT, logs BYTEA, - UNIQUE(neon_sign, finalized), + UNIQUE(neon_sign), UNIQUE(sol_sign, idx) ); CREATE INDEX IF NOT EXISTS {self._table_name}_finalized ON {self._table_name}(slot, finalized); @@ -136,11 +136,14 @@ def set_tx(self, tx: NeonTxDBInfo): cursor = self._conn.cursor() cursor.execute(f''' - INSERT INTO {self._table_name} - ({', '.join(self._column_lst)}) - VALUES - ({', '.join(['%s' for _ in range(len(self._column_lst))])}) - ON CONFLICT DO NOTHING + INSERT INTO {self._table_name} + ({', '.join(self._column_lst)}) + VALUES + ({', '.join(['%s' for _ in range(len(self._column_lst))])}) + ON CONFLICT (neon_sign) + DO UPDATE SET + {', '.join([col+'=EXCLUDED.'+col for col in self._column_lst])} + WHERE {self._table_name}.finalized=False AND EXCLUDED.finalized=True; ''', row) From 3192a8aa70d3d1ded938207cd66f8044d8831f18 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 20 Jan 2022 15:48:57 +0300 Subject: [PATCH 2/6] #455 use finalized state in local and ci --- proxy/run-proxy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/run-proxy.sh b/proxy/run-proxy.sh index 2483136db..1bc0fe6e7 100755 --- a/proxy/run-proxy.sh +++ b/proxy/run-proxy.sh @@ -14,7 +14,7 @@ if [ "$CONFIG" == "ci" ]; then [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 [[ -z "$RETRY_ON_BLOCKED" ]] && export RETRY_ON_BLOCKED=32 [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="confirmed" + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" elif [ "$CONFIG" == "local" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="http://localhost:8899" [[ -z "$EXTRA_GAS" ]] && export EXTRA_GAS=0 @@ -24,7 +24,7 @@ elif [ "$CONFIG" == "local" ]; then [[ -z "$CANCEL_TIMEOUT" ]] && export CANCEL_TIMEOUT=10 [[ -z "$RETRY_ON_BLOCKED" ]] && export RETRY_ON_BLOCKED=32 [[ -z "$RETRY_ON_FAIL" ]] && export RETRY_ON_FAIL=10 - [[ -z "$FINALIZED" ]] && export FINALIZED="confirmed" + [[ -z "$FINALIZED" ]] && export FINALIZED="finalized" elif [ "$CONFIG" == "devnet" ]; then [[ -z "$SOLANA_URL" ]] && export SOLANA_URL="https://api.devnet.solana.com" [[ -z "$EVM_LOADER" ]] && export EVM_LOADER=eeLSJgWzzxrqKv1UxtRVVH8FX3qCQWUs9QuAjJpETGU From 9e3e1e479cd892b76aa5c8562438c1bead7486de Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 20 Jan 2022 22:17:26 +0300 Subject: [PATCH 3/6] #455 set commitment of requested blocks --- proxy/indexer/blocks_db.py | 1 + proxy/indexer/indexer.py | 2 +- proxy/indexer/indexer_db.py | 27 ++++++++++++++------------- proxy/plugin/solana_rest_api.py | 8 ++++---- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index e12f1d678..8eace4656 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -99,6 +99,7 @@ def set_block(self, block: SolanaBlockDBInfo): parent_hash=EXCLUDED.parent_hash, blocktime=EXCLUDED.blocktime, signatures=EXCLUDED.signatures + WHERE {self._table_name}.finalized=False AND EXCLUDED.finalized=True; ''', (block.slot, block.finalized, block.height, block.hash, block.parent_hash, block.time, self.encode_list(block.signs))) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index 32c50a874..fb3c62a6f 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -241,7 +241,7 @@ def complete_done_txs(self): for tx in self._done_tx_list: self.unmark_ix_used(tx) if tx.neon_tx.is_valid() and tx.neon_res.is_valid(): - self._db.submit_transaction(tx.neon_tx, tx.neon_res, tx.used_ixs) + self._db.submit_transaction(tx.neon_tx, tx.neon_res, tx.used_ixs, commitment=FINALIZED) self.del_tx(tx) self._done_tx_list.clear() diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index e83ec35e2..3a5efba14 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -27,9 +27,9 @@ def __init__(self, client): if k not in self._constants: self._constants[k] = 0 - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo]): + def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo], commitment): try: - block = self.get_block_by_slot(neon_res.slot) + block = self.get_block_by_slot(neon_res.slot, commitment) if block.hash is None: self.critical(f'Unable to submit transaction {neon_tx.sign} because slot {neon_res.slot} not found') return @@ -41,14 +41,15 @@ def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, us rec['blockNumber'] = hex(block.height) self._logs_db.push_logs(neon_res.logs, block) tx = NeonTxDBInfo(neon_tx=neon_tx, neon_res=neon_res, block=block, used_ixs=used_ixs) + self.debug(f'submit_transaction NeonTxDBInfo {tx}') self._txs_db.set_tx(tx) except Exception as err: err_tb = "".join(traceback.format_tb(err.__traceback__)) self.warning('Exception on submitting transaction. ' + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - def _fill_block_from_net(self, block: SolanaBlockDBInfo): - opts = {"commitment": "confirmed", "transactionDetails": "signatures", "rewards": False} + def _fill_block_from_net(self, block: SolanaBlockDBInfo, commitment): + opts = {"commitment": commitment, "transactionDetails": "signatures", "rewards": False} net_block = self._client._provider.make_request("getBlock", block.slot, opts) if (not net_block) or ('result' not in net_block): return block @@ -59,21 +60,21 @@ def _fill_block_from_net(self, block: SolanaBlockDBInfo): block.signs = net_block['signatures'] block.parent_hash = '0x' + base58.b58decode(net_block['previousBlockhash']).hex() block.time = net_block['blockTime'] - block.finalized = ("confirmed" == FINALIZED) + block.finalized = (commitment == FINALIZED) self.debug(f'{block}') self._blocks_db.set_block(block) return block - def get_block_by_slot(self, slot) -> SolanaBlockDBInfo: + def get_block_by_slot(self, slot, commitment) -> SolanaBlockDBInfo: block = self._blocks_db.get_block_by_slot(slot) if not block.hash: - self._fill_block_from_net(block) + self._fill_block_from_net(block, commitment) return block - def get_full_block_by_slot(self, slot) -> SolanaBlockDBInfo: + def get_full_block_by_slot(self, slot, commitment) -> SolanaBlockDBInfo: block = self._blocks_db.get_full_block_by_slot(slot) if not block.parent_hash: - self._fill_block_from_net(block) + self._fill_block_from_net(block, commitment) return block def get_last_block_slot(self): @@ -107,16 +108,16 @@ def get_block_by_hash(self, block_hash): def get_block_by_height(self, block_height): return self._blocks_db.get_block_by_height(block_height) - def get_tx_by_sol_sign(self, sol_sign): + def get_tx_by_sol_sign(self, sol_sign, commitment): tx = self._txs_db.get_tx_by_sol_sign(sol_sign) if tx: - tx.block = self.get_block_by_slot(tx.neon_res.slot) + tx.block = self.get_block_by_slot(tx.neon_res.slot, commitment) return tx - def get_tx_by_neon_sign(self, neon_sign) -> NeonTxDBInfo: + def get_tx_by_neon_sign(self, neon_sign, commitment) -> NeonTxDBInfo: tx = self._txs_db.get_tx_by_neon_sign(neon_sign) if tx: - tx.block = self.get_block_by_slot(tx.neon_res.slot) + tx.block = self.get_block_by_slot(tx.neon_res.slot, commitment) return tx def del_not_finalized(self, from_slot: int, to_slot: int): diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 5b0b0f1cb..3b8b18d16 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -168,7 +168,7 @@ def eth_getLogs(self, obj): return self.db.get_logs(fromBlock, toBlock, address, topics, blockHash) def getBlockBySlot(self, slot, full): - block = self.db.get_full_block_by_slot(slot) + block = self.db.get_full_block_by_slot(slot, commitment='confirmed') if block.slot is None: return None @@ -176,7 +176,7 @@ def getBlockBySlot(self, slot, full): gasUsed = 0 trx_index = 0 for signature in block.signs: - tx = self.db.get_tx_by_sol_sign(signature) + tx = self.db.get_tx_by_sol_sign(signature, commitment='confirmed') if not tx: continue @@ -308,7 +308,7 @@ def eth_getTransactionReceipt(self, trxId): self.debug('eth_getTransactionReceipt: %s', trxId) neon_sign = trxId.lower() - tx = self.db.get_tx_by_neon_sign(neon_sign) + tx = self.db.get_tx_by_neon_sign(neon_sign, commitment='confirmed') if not tx: self.debug("Not found receipt") return None @@ -388,7 +388,7 @@ def eth_sendRawTransaction(self, rawTrx): self.debug('Transaction signature: %s %s', signature, eth_signature) neon_tx = NeonTxInfo() neon_tx.init_from_eth_tx(trx) - self.db.submit_transaction(neon_tx, neon_res, []) + self.db.submit_transaction(neon_tx, neon_res, [], commitment='confirmed') return eth_signature except SolanaTrxError as err: From f823d69def341e825695a194ffb2479de82bacfa Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Thu, 20 Jan 2022 22:31:19 +0300 Subject: [PATCH 4/6] #455 fix return on empty table --- proxy/indexer/blocks_db.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index 8eace4656..d68feb6ee 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -84,7 +84,10 @@ def get_block_by_height(self, block_num) -> SolanaBlockDBInfo: self._fetchone(self._column_lst, [('height', block_num)], ['finalized desc'])) def get_latest_block_height(self) -> int: - return self._fetchone(['height'], [], ['height desc'])[0] + result = self._fetchone(['height'], [], ['height desc']) + if result: + return result[0] + return 0 def set_block(self, block: SolanaBlockDBInfo): cursor = self._conn.cursor() From 532d37c19f169a90387faf8d82874dc06d22c8c6 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Fri, 21 Jan 2022 09:49:11 +0300 Subject: [PATCH 5/6] #455 fix commitment --- proxy/indexer/indexer.py | 2 +- proxy/indexer/indexer_db.py | 26 +++++++++++++------------- proxy/plugin/solana_rest_api.py | 8 ++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/proxy/indexer/indexer.py b/proxy/indexer/indexer.py index fb3c62a6f..32c50a874 100644 --- a/proxy/indexer/indexer.py +++ b/proxy/indexer/indexer.py @@ -241,7 +241,7 @@ def complete_done_txs(self): for tx in self._done_tx_list: self.unmark_ix_used(tx) if tx.neon_tx.is_valid() and tx.neon_res.is_valid(): - self._db.submit_transaction(tx.neon_tx, tx.neon_res, tx.used_ixs, commitment=FINALIZED) + self._db.submit_transaction(tx.neon_tx, tx.neon_res, tx.used_ixs) self.del_tx(tx) self._done_tx_list.clear() diff --git a/proxy/indexer/indexer_db.py b/proxy/indexer/indexer_db.py index 3a5efba14..cdfd8b1ea 100644 --- a/proxy/indexer/indexer_db.py +++ b/proxy/indexer/indexer_db.py @@ -27,9 +27,9 @@ def __init__(self, client): if k not in self._constants: self._constants[k] = 0 - def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo], commitment): + def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, used_ixs: [SolanaIxSignInfo]): try: - block = self.get_block_by_slot(neon_res.slot, commitment) + block = self.get_block_by_slot(neon_res.slot) if block.hash is None: self.critical(f'Unable to submit transaction {neon_tx.sign} because slot {neon_res.slot} not found') return @@ -48,8 +48,8 @@ def submit_transaction(self, neon_tx: NeonTxInfo, neon_res: NeonTxResultInfo, us self.warning('Exception on submitting transaction. ' + f'Type(err): {type(err)}, Error: {err}, Traceback: {err_tb}') - def _fill_block_from_net(self, block: SolanaBlockDBInfo, commitment): - opts = {"commitment": commitment, "transactionDetails": "signatures", "rewards": False} + def _fill_block_from_net(self, block: SolanaBlockDBInfo): + opts = {"commitment": "confirmed", "transactionDetails": "signatures", "rewards": False} net_block = self._client._provider.make_request("getBlock", block.slot, opts) if (not net_block) or ('result' not in net_block): return block @@ -60,21 +60,21 @@ def _fill_block_from_net(self, block: SolanaBlockDBInfo, commitment): block.signs = net_block['signatures'] block.parent_hash = '0x' + base58.b58decode(net_block['previousBlockhash']).hex() block.time = net_block['blockTime'] - block.finalized = (commitment == FINALIZED) + block.finalized = block.finalized if block.finalized else ("confirmed" == FINALIZED) self.debug(f'{block}') self._blocks_db.set_block(block) return block - def get_block_by_slot(self, slot, commitment) -> SolanaBlockDBInfo: + def get_block_by_slot(self, slot) -> SolanaBlockDBInfo: block = self._blocks_db.get_block_by_slot(slot) if not block.hash: - self._fill_block_from_net(block, commitment) + self._fill_block_from_net(block) return block - def get_full_block_by_slot(self, slot, commitment) -> SolanaBlockDBInfo: + def get_full_block_by_slot(self, slot) -> SolanaBlockDBInfo: block = self._blocks_db.get_full_block_by_slot(slot) if not block.parent_hash: - self._fill_block_from_net(block, commitment) + self._fill_block_from_net(block) return block def get_last_block_slot(self): @@ -108,16 +108,16 @@ def get_block_by_hash(self, block_hash): def get_block_by_height(self, block_height): return self._blocks_db.get_block_by_height(block_height) - def get_tx_by_sol_sign(self, sol_sign, commitment): + def get_tx_by_sol_sign(self, sol_sign): tx = self._txs_db.get_tx_by_sol_sign(sol_sign) if tx: - tx.block = self.get_block_by_slot(tx.neon_res.slot, commitment) + tx.block = self.get_block_by_slot(tx.neon_res.slot) return tx - def get_tx_by_neon_sign(self, neon_sign, commitment) -> NeonTxDBInfo: + def get_tx_by_neon_sign(self, neon_sign) -> NeonTxDBInfo: tx = self._txs_db.get_tx_by_neon_sign(neon_sign) if tx: - tx.block = self.get_block_by_slot(tx.neon_res.slot, commitment) + tx.block = self.get_block_by_slot(tx.neon_res.slot) return tx def del_not_finalized(self, from_slot: int, to_slot: int): diff --git a/proxy/plugin/solana_rest_api.py b/proxy/plugin/solana_rest_api.py index 3b8b18d16..5b0b0f1cb 100644 --- a/proxy/plugin/solana_rest_api.py +++ b/proxy/plugin/solana_rest_api.py @@ -168,7 +168,7 @@ def eth_getLogs(self, obj): return self.db.get_logs(fromBlock, toBlock, address, topics, blockHash) def getBlockBySlot(self, slot, full): - block = self.db.get_full_block_by_slot(slot, commitment='confirmed') + block = self.db.get_full_block_by_slot(slot) if block.slot is None: return None @@ -176,7 +176,7 @@ def getBlockBySlot(self, slot, full): gasUsed = 0 trx_index = 0 for signature in block.signs: - tx = self.db.get_tx_by_sol_sign(signature, commitment='confirmed') + tx = self.db.get_tx_by_sol_sign(signature) if not tx: continue @@ -308,7 +308,7 @@ def eth_getTransactionReceipt(self, trxId): self.debug('eth_getTransactionReceipt: %s', trxId) neon_sign = trxId.lower() - tx = self.db.get_tx_by_neon_sign(neon_sign, commitment='confirmed') + tx = self.db.get_tx_by_neon_sign(neon_sign) if not tx: self.debug("Not found receipt") return None @@ -388,7 +388,7 @@ def eth_sendRawTransaction(self, rawTrx): self.debug('Transaction signature: %s %s', signature, eth_signature) neon_tx = NeonTxInfo() neon_tx.init_from_eth_tx(trx) - self.db.submit_transaction(neon_tx, neon_res, [], commitment='confirmed') + self.db.submit_transaction(neon_tx, neon_res, []) return eth_signature except SolanaTrxError as err: From 96f4fcd5321bf756e3a12edf1ec0981433f92874 Mon Sep 17 00:00:00 2001 From: Dmitriy Borisenko Date: Fri, 21 Jan 2022 11:32:37 +0300 Subject: [PATCH 6/6] #455 insert block return logic --- proxy/indexer/blocks_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/proxy/indexer/blocks_db.py b/proxy/indexer/blocks_db.py index d68feb6ee..9d554b656 100644 --- a/proxy/indexer/blocks_db.py +++ b/proxy/indexer/blocks_db.py @@ -102,7 +102,6 @@ def set_block(self, block: SolanaBlockDBInfo): parent_hash=EXCLUDED.parent_hash, blocktime=EXCLUDED.blocktime, signatures=EXCLUDED.signatures - WHERE {self._table_name}.finalized=False AND EXCLUDED.finalized=True; ''', (block.slot, block.finalized, block.height, block.hash, block.parent_hash, block.time, self.encode_list(block.signs)))