From f67b91f9d36b81531b5bf4141ec837f8a4d12147 Mon Sep 17 00:00:00 2001 From: Artemka374 Date: Mon, 27 Nov 2023 13:40:48 +0200 Subject: [PATCH 01/12] add 1 to from_block after poll --- core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 4cabb8e15da3..ffa26dfa7ef2 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -861,9 +861,8 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; *from_block = logs .last() - .map(|log| MiniblockNumber(log.block_number.unwrap().as_u32())) - .unwrap_or(*from_block); - // FIXME: why is `from_block` not updated? + .map(|log| MiniblockNumber(log.block_number.unwrap().as_u32() + 1)) + .unwrap_or(*from_block + 1); FilterChanges::Logs(logs) } }; From 05aa2164041f00caf6a512043dd2b62e33d655e2 Mon Sep 17 00:00:00 2001 From: Artemka374 Date: Mon, 27 Nov 2023 19:16:16 +0200 Subject: [PATCH 02/12] increase from_block only if logs are not empty --- core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index ffa26dfa7ef2..fab853177726 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -862,7 +862,7 @@ impl EthNamespace { *from_block = logs .last() .map(|log| MiniblockNumber(log.block_number.unwrap().as_u32() + 1)) - .unwrap_or(*from_block + 1); + .unwrap_or(*from_block); FilterChanges::Logs(logs) } }; From 21804c8953dd85d8a8b292906fdde9555e775968 Mon Sep 17 00:00:00 2001 From: Artemka374 Date: Tue, 28 Nov 2023 12:12:15 +0200 Subject: [PATCH 03/12] set from_block to minimum from to_block and current latest block polled + 1 --- .../zksync_core/src/api_server/web3/namespaces/eth.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index fab853177726..0ef2dfb5919c 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -826,6 +826,10 @@ impl EthNamespace { .state .resolve_filter_block_number(filter.to_block) .await?; + let latest_block = self + .state + .resolve_filter_block_number(Some(BlockNumber::Latest)) + .await?; let mut storage = self .state @@ -859,10 +863,7 @@ impl EthNamespace { .get_logs(get_logs_filter, i32::MAX as usize) .await .map_err(|err| internal_error(METHOD_NAME, err))?; - *from_block = logs - .last() - .map(|log| MiniblockNumber(log.block_number.unwrap().as_u32() + 1)) - .unwrap_or(*from_block); + *from_block = to_block.min(latest_block) + 1; FilterChanges::Logs(logs) } }; From 5df7727aaad7d3a2381730a11740011db055f8d1 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Thu, 30 Nov 2023 22:54:22 +0200 Subject: [PATCH 04/12] update cursor to be inclusive in all types of filters --- core/lib/dal/src/blocks_web3_dal.rs | 4 +-- core/lib/dal/src/transactions_web3_dal.rs | 4 +-- .../src/api_server/web3/namespaces/eth.rs | 32 ++++++++++++++----- .../zksync_core/src/api_server/web3/pubsub.rs | 2 +- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/core/lib/dal/src/blocks_web3_dal.rs b/core/lib/dal/src/blocks_web3_dal.rs index e42a645966ff..215dc728e496 100644 --- a/core/lib/dal/src/blocks_web3_dal.rs +++ b/core/lib/dal/src/blocks_web3_dal.rs @@ -162,14 +162,14 @@ impl BlocksWeb3Dal<'_, '_> { } /// Returns hashes of blocks with numbers greater than `from_block` and the number of the last block. - pub async fn get_block_hashes_after( + pub async fn get_block_hashes_since( &mut self, from_block: MiniblockNumber, limit: usize, ) -> sqlx::Result<(Vec, Option)> { let rows = sqlx::query!( "SELECT number, hash FROM miniblocks \ - WHERE number > $1 \ + WHERE number >= $1 \ ORDER BY number ASC \ LIMIT $2", from_block.0 as i64, diff --git a/core/lib/dal/src/transactions_web3_dal.rs b/core/lib/dal/src/transactions_web3_dal.rs index 5e2342d05b75..7995faec3951 100644 --- a/core/lib/dal/src/transactions_web3_dal.rs +++ b/core/lib/dal/src/transactions_web3_dal.rs @@ -255,7 +255,7 @@ impl TransactionsWeb3Dal<'_, '_> { } /// Returns hashes of txs which were received after `from_timestamp` and the time of receiving the last tx. - pub async fn get_pending_txs_hashes_after( + pub async fn get_pending_txs_hashes_since( &mut self, from_timestamp: NaiveDateTime, limit: Option, @@ -264,7 +264,7 @@ impl TransactionsWeb3Dal<'_, '_> { "SELECT transactions.hash, transactions.received_at \ FROM transactions \ LEFT JOIN miniblocks ON miniblocks.number = miniblock_number \ - WHERE received_at > $1 \ + WHERE received_at >= $1 \ ORDER BY received_at ASC \ LIMIT $2", from_timestamp, diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 0ef2dfb5919c..b30606aaa73b 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -1,3 +1,4 @@ +use zksync_types::api::BlockIdVariant::BlockNumber; use zksync_types::{ api::{ BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, @@ -773,10 +774,15 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; let (block_hashes, last_block_number) = conn .blocks_web3_dal() - .get_block_hashes_after(*from_block, self.state.api_config.req_entities_limit) + .get_block_hashes_since(*from_block, self.state.api_config.req_entities_limit) .await .map_err(|err| internal_error(METHOD_NAME, err))?; - *from_block = last_block_number.unwrap_or(*from_block); + + *from_block = match last_block_number { + Some(last_block_number) => last_block_number + 1, + None => *from_block, + }; + FilterChanges::Hashes(block_hashes) } @@ -789,13 +795,18 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; let (tx_hashes, last_timestamp) = conn .transactions_web3_dal() - .get_pending_txs_hashes_after( + .get_pending_txs_hashes_since( *from_timestamp, Some(self.state.api_config.req_entities_limit), ) .await .map_err(|err| internal_error(METHOD_NAME, err))?; - *from_timestamp = last_timestamp.unwrap_or(*from_timestamp); + + *from_timestamp = match last_timestamp { + Some(last_timestamp) => last_timestamp + 1, + None => *from_timestamp, + }; + FilterChanges::Hashes(tx_hashes) } @@ -826,10 +837,15 @@ impl EthNamespace { .state .resolve_filter_block_number(filter.to_block) .await?; - let latest_block = self - .state - .resolve_filter_block_number(Some(BlockNumber::Latest)) - .await?; + + let latest_block = + if filter.to_block.is_none() || filter.to_block == Some(BlockNumber::Latest) { + self.state + .resolve_filter_block_number(Some(BlockNumber::Latest)) + .await? + } else { + to_block + }; let mut storage = self .state diff --git a/core/lib/zksync_core/src/api_server/web3/pubsub.rs b/core/lib/zksync_core/src/api_server/web3/pubsub.rs index 946c0744ba4d..670d61202fef 100644 --- a/core/lib/zksync_core/src/api_server/web3/pubsub.rs +++ b/core/lib/zksync_core/src/api_server/web3/pubsub.rs @@ -157,7 +157,7 @@ impl PubSubNotifier> { .await .context("access_storage_tagged")? .transactions_web3_dal() - .get_pending_txs_hashes_after(last_time, None) + .get_pending_txs_hashes_since(last_time, None) .await .context("get_pending_txs_hashes_after()") } From f06b79d721a20d9b9fc63e69e057da114aa4caaf Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Thu, 30 Nov 2023 22:55:21 +0200 Subject: [PATCH 05/12] remove multiple imports --- core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index b30606aaa73b..77d90a133af7 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -1,4 +1,3 @@ -use zksync_types::api::BlockIdVariant::BlockNumber; use zksync_types::{ api::{ BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, From 91cdf89dfc021dc787e159ccf2f3ed8d80a569a6 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:41:46 +0200 Subject: [PATCH 06/12] refactor get_logs to accept definite range of blocks --- core/lib/dal/sqlx-data.json | 108 +++++++++--------- core/lib/dal/src/events_web3_dal.rs | 15 +-- core/lib/types/src/api/mod.rs | 2 +- .../src/api_server/web3/namespaces/eth.rs | 33 +++--- .../src/api_server/web3/namespaces/zks.rs | 2 +- .../zksync_core/src/api_server/web3/state.rs | 3 +- 6 files changed, 84 insertions(+), 79 deletions(-) diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 3776b4f84b34..2e782be7934e 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -1928,6 +1928,33 @@ }, "query": "SELECT * from prover_jobs where id=$1" }, + "2044947d6d29f29cda508b2160c39f74a8bfd524afa2ffc20a98ae039bc86ed7": { + "describe": { + "columns": [ + { + "name": "number", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "hash", + "ordinal": 1, + "type_info": "Bytea" + } + ], + "nullable": [ + false, + false + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + } + }, + "query": "SELECT number, hash FROM miniblocks WHERE number >= $1 ORDER BY number ASC LIMIT $2" + }, "20b22fd457417e9a72f5941887448f9a11b97b449db4759da0b9d368ce93996b": { "describe": { "columns": [ @@ -8759,33 +8786,6 @@ }, "query": "\n UPDATE scheduler_witness_jobs_fri\n SET status='queued'\n WHERE l1_batch_number = $1\n AND status != 'successful'\n AND status != 'in_progress'\n " }, - "a4a14eb42b9acca3f93c67e5760ba700c333b5e9a38c132a3060a94c988e7f13": { - "describe": { - "columns": [ - { - "name": "hash", - "ordinal": 0, - "type_info": "Bytea" - }, - { - "name": "received_at", - "ordinal": 1, - "type_info": "Timestamp" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Timestamp", - "Int8" - ] - } - }, - "query": "SELECT transactions.hash, transactions.received_at FROM transactions LEFT JOIN miniblocks ON miniblocks.number = miniblock_number WHERE received_at > $1 ORDER BY received_at ASC LIMIT $2" - }, "a4f240188c1447f5b6dcef33dfcc9d00b105f62a6b4c3949a825bea979954160": { "describe": { "columns": [], @@ -9451,33 +9451,6 @@ }, "query": "\n SELECT id, circuit_input_blob_url FROM prover_jobs\n WHERE status='successful'\n AND circuit_input_blob_url is NOT NULL\n AND updated_at < NOW() - INTERVAL '30 days'\n LIMIT $1;\n " }, - "b479b7d3334f8d4566c294a44e2adb282fbc66a87be5c248c65211c2a8a07db0": { - "describe": { - "columns": [ - { - "name": "number", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "hash", - "ordinal": 1, - "type_info": "Bytea" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - } - }, - "query": "SELECT number, hash FROM miniblocks WHERE number > $1 ORDER BY number ASC LIMIT $2" - }, "b4a3c902646725188f7c79ebac992cdce5896fc6fcc9f485c0cba9d90c4c982c": { "describe": { "columns": [ @@ -12019,6 +11992,33 @@ }, "query": "SELECT number, timestamp, hash, l1_tx_count, l2_tx_count, base_fee_per_gas, l1_gas_price, l2_fair_gas_price, bootloader_code_hash, default_aa_code_hash, protocol_version, virtual_blocks\n FROM miniblocks ORDER BY number DESC LIMIT 1" }, + "fa29c9770df1a33b408397af262eb4cb0b51336c770b61977c3849b8f7a77bda": { + "describe": { + "columns": [ + { + "name": "hash", + "ordinal": 0, + "type_info": "Bytea" + }, + { + "name": "received_at", + "ordinal": 1, + "type_info": "Timestamp" + } + ], + "nullable": [ + false, + false + ], + "parameters": { + "Left": [ + "Timestamp", + "Int8" + ] + } + }, + "query": "SELECT transactions.hash, transactions.received_at FROM transactions LEFT JOIN miniblocks ON miniblocks.number = miniblock_number WHERE received_at >= $1 ORDER BY received_at ASC LIMIT $2" + }, "fa2b4316aaef09e96d93b70f96b129ed123951732e01d63f30b4b292d441ea39": { "describe": { "columns": [ diff --git a/core/lib/dal/src/events_web3_dal.rs b/core/lib/dal/src/events_web3_dal.rs index 82a65c18444b..704a2b62862d 100644 --- a/core/lib/dal/src/events_web3_dal.rs +++ b/core/lib/dal/src/events_web3_dal.rs @@ -1,8 +1,10 @@ use sqlx::Row; +use zksync_types::api::BlockNumber; +use zksync_types::zkevm_test_harness::zk_evm::zkevm_opcode_defs::decoding::AllowedPcOrImm; use zksync_types::{ api::{GetLogsFilter, Log}, - Address, MiniblockNumber, H256, + Address, MiniblockNumber, H256, U64, }; use crate::{ @@ -119,10 +121,10 @@ impl EventsWeb3Dal<'_, '_> { let mut where_sql = format!("(miniblock_number >= {})", filter.from_block.0 as i64); - if let Some(to_block) = filter.to_block { - let block_sql = web3_block_number_to_sql(to_block); - where_sql += &format!(" AND (miniblock_number <= {})", block_sql); - } + let block_sql = + web3_block_number_to_sql(BlockNumber::Number(U64::from(filter.to_block.as_u64()))); + where_sql += &format!(" AND (miniblock_number <= {})", block_sql); + if !filter.addresses.is_empty() { where_sql += &format!(" AND (address = ANY(${}))", arg_index); arg_index += 1; @@ -172,7 +174,6 @@ impl EventsWeb3Dal<'_, '_> { #[cfg(test)] mod tests { - use zksync_types::api::BlockNumber; use zksync_types::{Address, H256}; use super::*; @@ -185,7 +186,7 @@ mod tests { let events_web3_dal = EventsWeb3Dal { storage }; let filter = GetLogsFilter { from_block: MiniblockNumber(100), - to_block: Some(BlockNumber::Number(200.into())), + to_block: MiniblockNumber(200), addresses: vec![Address::from_low_u64_be(123)], topics: vec![(0, vec![H256::from_low_u64_be(456)])], }; diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 24ac74ab335e..1ad54ce6d1ab 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -548,7 +548,7 @@ pub struct TransactionDetails { #[derive(Debug, Clone)] pub struct GetLogsFilter { pub from_block: MiniblockNumber, - pub to_block: Option, + pub to_block: MiniblockNumber, pub addresses: Vec
, pub topics: Vec<(u32, Vec)>, } diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 77d90a133af7..5303baa57a0a 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -1,3 +1,5 @@ +use std::ops::Add; +use std::time::Duration; use zksync_types::{ api::{ BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, @@ -802,7 +804,7 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; *from_timestamp = match last_timestamp { - Some(last_timestamp) => last_timestamp + 1, + Some(last_timestamp) => last_timestamp.add(Duration::from_secs(1)), None => *from_timestamp, }; @@ -826,25 +828,26 @@ impl EthNamespace { } else { vec![] }; - let get_logs_filter = GetLogsFilter { - from_block: *from_block, - to_block: filter.to_block, - addresses, - topics, - }; - let to_block = self + + let mut to_block = self .state .resolve_filter_block_number(filter.to_block) .await?; - let latest_block = - if filter.to_block.is_none() || filter.to_block == Some(BlockNumber::Latest) { + if let Some(BlockNumber::Number(_)) = filter.to_block { + to_block = to_block.min( self.state .resolve_filter_block_number(Some(BlockNumber::Latest)) - .await? - } else { - to_block - }; + .await?, + ); + } + + let get_logs_filter = GetLogsFilter { + from_block: *from_block, + to_block, + addresses, + topics, + }; let mut storage = self .state @@ -878,7 +881,7 @@ impl EthNamespace { .get_logs(get_logs_filter, i32::MAX as usize) .await .map_err(|err| internal_error(METHOD_NAME, err))?; - *from_block = to_block.min(latest_block) + 1; + *from_block = to_block + 1; FilterChanges::Logs(logs) } }; diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs index 7f38c6afc525..9e3a90dde043 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs @@ -283,7 +283,7 @@ impl ZksNamespace { .get_logs( GetLogsFilter { from_block: first_miniblock_of_l1_batch, - to_block: Some(block_number.0.into()), + to_block: block_number, addresses: vec![L1_MESSENGER_ADDRESS], topics: vec![(2, vec![address_to_h256(&sender)]), (3, vec![msg])], }, diff --git a/core/lib/zksync_core/src/api_server/web3/state.rs b/core/lib/zksync_core/src/api_server/web3/state.rs index 0463d4823209..ea52b2ae61cc 100644 --- a/core/lib/zksync_core/src/api_server/web3/state.rs +++ b/core/lib/zksync_core/src/api_server/web3/state.rs @@ -505,9 +505,10 @@ impl RpcState { .enumerate() .filter_map(|(idx, topics)| topics.map(|topics| (idx as u32 + 1, topics.0))) .collect(); + let get_logs_filter = GetLogsFilter { from_block, - to_block: filter.to_block, + to_block, addresses, topics, }; From f5b5a2c8f33c844a8e57bcb0a7d54259a1db25eb Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:59:02 +0200 Subject: [PATCH 07/12] fix tests --- .../src/api_server/web3/tests/mod.rs | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/core/lib/zksync_core/src/api_server/web3/tests/mod.rs b/core/lib/zksync_core/src/api_server/web3/tests/mod.rs index 12bb6481213d..27178915fbee 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/mod.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/mod.rs @@ -386,10 +386,10 @@ impl HttpTest for LogFilterChanges { assert_logs_match(&topics_logs, &[events[1], events[3]]); let new_all_logs = client.get_filter_changes(all_logs_filter_id).await?; - let FilterChanges::Logs(new_all_logs) = new_all_logs else { + let FilterChanges::Hashes(new_all_logs) = new_all_logs else { panic!("Unexpected getFilterChanges output: {:?}", new_all_logs); }; - assert_eq!(new_all_logs, all_logs); // FIXME(#546): update test after behavior is fixed + assert!(new_all_logs.is_empty()); Ok(()) } } @@ -458,11 +458,10 @@ impl HttpTest for LogFilterChangesWithBlockBoundaries { }; assert_logs_match(&lower_bound_logs, &new_events); - // FIXME(#546): update test after behavior is fixed let new_upper_bound_logs = client.get_filter_changes(upper_bound_filter_id).await?; - assert_eq!(new_upper_bound_logs, FilterChanges::Logs(upper_bound_logs)); + assert_eq!(new_upper_bound_logs, FilterChanges::Hashes(vec![])); let new_bounded_logs = client.get_filter_changes(bounded_filter_id).await?; - assert_eq!(new_bounded_logs, FilterChanges::Logs(bounded_logs)); + assert_eq!(new_bounded_logs, FilterChanges::Hashes(vec![])); // Add miniblock #3. It should not be picked up by the bounded and upper bound filters, // and should be picked up by the lower bound filter. @@ -472,20 +471,16 @@ impl HttpTest for LogFilterChangesWithBlockBoundaries { let new_events: Vec<_> = new_events.iter().collect(); let bounded_logs = client.get_filter_changes(bounded_filter_id).await?; - let FilterChanges::Logs(bounded_logs) = bounded_logs else { + let FilterChanges::Hashes(bounded_logs) = bounded_logs else { panic!("Unexpected getFilterChanges output: {:?}", bounded_logs); }; - assert!(bounded_logs - .iter() - .all(|log| log.block_number.unwrap() < 3.into())); + assert!(bounded_logs.is_empty()); let upper_bound_logs = client.get_filter_changes(upper_bound_filter_id).await?; - let FilterChanges::Logs(upper_bound_logs) = upper_bound_logs else { + let FilterChanges::Hashes(upper_bound_logs) = upper_bound_logs else { panic!("Unexpected getFilterChanges output: {:?}", upper_bound_logs); }; - assert!(upper_bound_logs - .iter() - .all(|log| log.block_number.unwrap() < 3.into())); + assert!(upper_bound_logs.is_empty()); let lower_bound_logs = client.get_filter_changes(lower_bound_filter_id).await?; let FilterChanges::Logs(lower_bound_logs) = lower_bound_logs else { From 0f293e82ea8c4ee577e866edbe786af1b8f21004 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Fri, 1 Dec 2023 19:04:47 +0200 Subject: [PATCH 08/12] update basic filter changes --- core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 5303baa57a0a..45fa9d601dc8 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -566,7 +566,7 @@ impl EthNamespace { .installed_filters .lock() .await - .add(TypedFilter::Blocks(last_block_number)); + .add(TypedFilter::Blocks(last_block_number + 1)); method_latency.observe(); Ok(idx) } From 2aa73df1211e401969a55a921870c30307ba792f Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:52:57 +0200 Subject: [PATCH 09/12] apply suggestions --- core/lib/dal/src/events_web3_dal.rs | 11 +++-------- core/lib/dal/src/transactions_web3_dal.rs | 4 ++-- .../zksync_core/src/api_server/web3/namespaces/eth.rs | 8 +++----- core/lib/zksync_core/src/api_server/web3/pubsub.rs | 2 +- core/lib/zksync_core/src/api_server/web3/tests/mod.rs | 7 +++---- 5 files changed, 12 insertions(+), 20 deletions(-) diff --git a/core/lib/dal/src/events_web3_dal.rs b/core/lib/dal/src/events_web3_dal.rs index 704a2b62862d..7f0cd6b83ade 100644 --- a/core/lib/dal/src/events_web3_dal.rs +++ b/core/lib/dal/src/events_web3_dal.rs @@ -1,16 +1,13 @@ use sqlx::Row; -use zksync_types::api::BlockNumber; use zksync_types::zkevm_test_harness::zk_evm::zkevm_opcode_defs::decoding::AllowedPcOrImm; use zksync_types::{ api::{GetLogsFilter, Log}, - Address, MiniblockNumber, H256, U64, + Address, MiniblockNumber, H256, }; use crate::{ - instrument::InstrumentExt, - models::{storage_block::web3_block_number_to_sql, storage_event::StorageWeb3Log}, - SqlxError, StorageProcessor, + instrument::InstrumentExt, models::storage_event::StorageWeb3Log, SqlxError, StorageProcessor, }; #[derive(Debug)] @@ -121,9 +118,7 @@ impl EventsWeb3Dal<'_, '_> { let mut where_sql = format!("(miniblock_number >= {})", filter.from_block.0 as i64); - let block_sql = - web3_block_number_to_sql(BlockNumber::Number(U64::from(filter.to_block.as_u64()))); - where_sql += &format!(" AND (miniblock_number <= {})", block_sql); + where_sql += &format!(" AND (miniblock_number <= {})", filter.to_block.0 as i64); if !filter.addresses.is_empty() { where_sql += &format!(" AND (address = ANY(${}))", arg_index); diff --git a/core/lib/dal/src/transactions_web3_dal.rs b/core/lib/dal/src/transactions_web3_dal.rs index 7995faec3951..5e2342d05b75 100644 --- a/core/lib/dal/src/transactions_web3_dal.rs +++ b/core/lib/dal/src/transactions_web3_dal.rs @@ -255,7 +255,7 @@ impl TransactionsWeb3Dal<'_, '_> { } /// Returns hashes of txs which were received after `from_timestamp` and the time of receiving the last tx. - pub async fn get_pending_txs_hashes_since( + pub async fn get_pending_txs_hashes_after( &mut self, from_timestamp: NaiveDateTime, limit: Option, @@ -264,7 +264,7 @@ impl TransactionsWeb3Dal<'_, '_> { "SELECT transactions.hash, transactions.received_at \ FROM transactions \ LEFT JOIN miniblocks ON miniblocks.number = miniblock_number \ - WHERE received_at >= $1 \ + WHERE received_at > $1 \ ORDER BY received_at ASC \ LIMIT $2", from_timestamp, diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 45fa9d601dc8..f3d4c001b493 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -1,5 +1,3 @@ -use std::ops::Add; -use std::time::Duration; use zksync_types::{ api::{ BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, @@ -796,7 +794,7 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; let (tx_hashes, last_timestamp) = conn .transactions_web3_dal() - .get_pending_txs_hashes_since( + .get_pending_txs_hashes_after( *from_timestamp, Some(self.state.api_config.req_entities_limit), ) @@ -804,7 +802,7 @@ impl EthNamespace { .map_err(|err| internal_error(METHOD_NAME, err))?; *from_timestamp = match last_timestamp { - Some(last_timestamp) => last_timestamp.add(Duration::from_secs(1)), + Some(last_timestamp) => last_timestamp, None => *from_timestamp, }; @@ -834,7 +832,7 @@ impl EthNamespace { .resolve_filter_block_number(filter.to_block) .await?; - if let Some(BlockNumber::Number(_)) = filter.to_block { + if matches!(filter.to_block, BlockNumber::Number(_)) { to_block = to_block.min( self.state .resolve_filter_block_number(Some(BlockNumber::Latest)) diff --git a/core/lib/zksync_core/src/api_server/web3/pubsub.rs b/core/lib/zksync_core/src/api_server/web3/pubsub.rs index 670d61202fef..946c0744ba4d 100644 --- a/core/lib/zksync_core/src/api_server/web3/pubsub.rs +++ b/core/lib/zksync_core/src/api_server/web3/pubsub.rs @@ -157,7 +157,7 @@ impl PubSubNotifier> { .await .context("access_storage_tagged")? .transactions_web3_dal() - .get_pending_txs_hashes_since(last_time, None) + .get_pending_txs_hashes_after(last_time, None) .await .context("get_pending_txs_hashes_after()") } diff --git a/core/lib/zksync_core/src/api_server/web3/tests/mod.rs b/core/lib/zksync_core/src/api_server/web3/tests/mod.rs index 27178915fbee..1bb14df52fa2 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/mod.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/mod.rs @@ -459,9 +459,9 @@ impl HttpTest for LogFilterChangesWithBlockBoundaries { assert_logs_match(&lower_bound_logs, &new_events); let new_upper_bound_logs = client.get_filter_changes(upper_bound_filter_id).await?; - assert_eq!(new_upper_bound_logs, FilterChanges::Hashes(vec![])); + assert_matches!(new_upper_bound_logs, FilterChanges::Hashes(hashes) if hashes.is_empty()); let new_bounded_logs = client.get_filter_changes(bounded_filter_id).await?; - assert_eq!(new_bounded_logs, FilterChanges::Hashes(vec![])); + assert_matches!(new_bounded_logs, FilterChanges::Hashes(hashes) if hashes.is_empty()); // Add miniblock #3. It should not be picked up by the bounded and upper bound filters, // and should be picked up by the lower bound filter. @@ -486,8 +486,7 @@ impl HttpTest for LogFilterChangesWithBlockBoundaries { let FilterChanges::Logs(lower_bound_logs) = lower_bound_logs else { panic!("Unexpected getFilterChanges output: {:?}", lower_bound_logs); }; - let start_idx = lower_bound_logs.len() - 4; - assert_logs_match(&lower_bound_logs[start_idx..], &new_events); + assert_logs_match(&lower_bound_logs, &new_events); Ok(()) } } From 1d952e4deb32330b09bd9761c64f8e6575b0a1e6 Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:11:32 +0200 Subject: [PATCH 10/12] apply suggestions --- core/lib/dal/sqlx-data.json | 54 +++++++++---------- .../src/api_server/web3/namespaces/eth.rs | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index ed0e7b24fe91..28735e943735 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -8771,6 +8771,33 @@ }, "query": "\n UPDATE scheduler_witness_jobs_fri\n SET status='queued'\n WHERE l1_batch_number = $1\n AND status != 'successful'\n AND status != 'in_progress'\n " }, + "a4a14eb42b9acca3f93c67e5760ba700c333b5e9a38c132a3060a94c988e7f13": { + "describe": { + "columns": [ + { + "name": "hash", + "ordinal": 0, + "type_info": "Bytea" + }, + { + "name": "received_at", + "ordinal": 1, + "type_info": "Timestamp" + } + ], + "nullable": [ + false, + false + ], + "parameters": { + "Left": [ + "Timestamp", + "Int8" + ] + } + }, + "query": "SELECT transactions.hash, transactions.received_at FROM transactions LEFT JOIN miniblocks ON miniblocks.number = miniblock_number WHERE received_at > $1 ORDER BY received_at ASC LIMIT $2" + }, "a4f240188c1447f5b6dcef33dfcc9d00b105f62a6b4c3949a825bea979954160": { "describe": { "columns": [], @@ -11992,33 +12019,6 @@ }, "query": "SELECT number, timestamp, hash, l1_tx_count, l2_tx_count, base_fee_per_gas, l1_gas_price, l2_fair_gas_price, bootloader_code_hash, default_aa_code_hash, protocol_version, virtual_blocks\n FROM miniblocks ORDER BY number DESC LIMIT 1" }, - "fa29c9770df1a33b408397af262eb4cb0b51336c770b61977c3849b8f7a77bda": { - "describe": { - "columns": [ - { - "name": "hash", - "ordinal": 0, - "type_info": "Bytea" - }, - { - "name": "received_at", - "ordinal": 1, - "type_info": "Timestamp" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Timestamp", - "Int8" - ] - } - }, - "query": "SELECT transactions.hash, transactions.received_at FROM transactions LEFT JOIN miniblocks ON miniblocks.number = miniblock_number WHERE received_at >= $1 ORDER BY received_at ASC LIMIT $2" - }, "fa2b4316aaef09e96d93b70f96b129ed123951732e01d63f30b4b292d441ea39": { "describe": { "columns": [ diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index f3d4c001b493..85ba436fd864 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -832,7 +832,7 @@ impl EthNamespace { .resolve_filter_block_number(filter.to_block) .await?; - if matches!(filter.to_block, BlockNumber::Number(_)) { + if matches!(filter.to_block, Some(BlockNumber::Number(_))) { to_block = to_block.min( self.state .resolve_filter_block_number(Some(BlockNumber::Latest)) From 3ffec67a863eb4004e7b1d3ff748074d579e5c9b Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:12:05 +0200 Subject: [PATCH 11/12] remove redundant import --- core/lib/dal/src/events_web3_dal.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/lib/dal/src/events_web3_dal.rs b/core/lib/dal/src/events_web3_dal.rs index 7f0cd6b83ade..7cdf2dba6467 100644 --- a/core/lib/dal/src/events_web3_dal.rs +++ b/core/lib/dal/src/events_web3_dal.rs @@ -1,6 +1,5 @@ use sqlx::Row; -use zksync_types::zkevm_test_harness::zk_evm::zkevm_opcode_defs::decoding::AllowedPcOrImm; use zksync_types::{ api::{GetLogsFilter, Log}, Address, MiniblockNumber, H256, From 8df660ab981cc0c5ceae8d9207abe6240f11df3b Mon Sep 17 00:00:00 2001 From: Lech <88630083+Artemka374@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:18:58 +0200 Subject: [PATCH 12/12] apply suggestions --- core/lib/dal/src/blocks_web3_dal.rs | 2 +- .../zksync_core/src/api_server/web3/namespaces/eth.rs | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/lib/dal/src/blocks_web3_dal.rs b/core/lib/dal/src/blocks_web3_dal.rs index 215dc728e496..0c2a8b4e1885 100644 --- a/core/lib/dal/src/blocks_web3_dal.rs +++ b/core/lib/dal/src/blocks_web3_dal.rs @@ -161,7 +161,7 @@ impl BlocksWeb3Dal<'_, '_> { })) } - /// Returns hashes of blocks with numbers greater than `from_block` and the number of the last block. + /// Returns hashes of blocks with numbers starting from `from_block` and the number of the last block. pub async fn get_block_hashes_since( &mut self, from_block: MiniblockNumber, diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 85ba436fd864..0aa9255c3dbf 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -785,7 +785,7 @@ impl EthNamespace { FilterChanges::Hashes(block_hashes) } - TypedFilter::PendingTransactions(from_timestamp) => { + TypedFilter::PendingTransactions(from_timestamp_excluded) => { let mut conn = self .state .connection_pool @@ -795,16 +795,13 @@ impl EthNamespace { let (tx_hashes, last_timestamp) = conn .transactions_web3_dal() .get_pending_txs_hashes_after( - *from_timestamp, + *from_timestamp_excluded, Some(self.state.api_config.req_entities_limit), ) .await .map_err(|err| internal_error(METHOD_NAME, err))?; - *from_timestamp = match last_timestamp { - Some(last_timestamp) => last_timestamp, - None => *from_timestamp, - }; + *from_timestamp_excluded = last_timestamp.unwrap_or(*from_timestamp_excluded); FilterChanges::Hashes(tx_hashes) }