From 2fe7bb61fcbbae045cd2829fd77031fc73a6bc46 Mon Sep 17 00:00:00 2001 From: green Date: Fri, 11 Oct 2024 19:31:29 +0200 Subject: [PATCH 01/19] Prepare the GraphQL service for the switching to `async` methods --- crates/fuel-core/src/coins_query.rs | 2 - crates/fuel-core/src/graphql_api/database.rs | 114 +++++++----------- crates/fuel-core/src/query.rs | 8 -- crates/fuel-core/src/query/balance.rs | 22 +--- .../src/query/balance/asset_query.rs | 8 +- crates/fuel-core/src/query/blob.rs | 16 +-- crates/fuel-core/src/query/block.rs | 49 +------- crates/fuel-core/src/query/chain.rs | 13 -- crates/fuel-core/src/query/coin.rs | 40 +----- crates/fuel-core/src/query/contract.rs | 61 ++-------- crates/fuel-core/src/query/message.rs | 102 ++++++++-------- crates/fuel-core/src/query/message/test.rs | 11 +- crates/fuel-core/src/query/tx.rs | 45 +------ crates/fuel-core/src/query/upgrades.rs | 26 ++-- crates/fuel-core/src/schema/balance.rs | 1 - crates/fuel-core/src/schema/blob.rs | 1 - crates/fuel-core/src/schema/block.rs | 6 - crates/fuel-core/src/schema/chain.rs | 4 - crates/fuel-core/src/schema/coins.rs | 5 +- crates/fuel-core/src/schema/contract.rs | 3 +- crates/fuel-core/src/schema/da_compressed.rs | 1 - crates/fuel-core/src/schema/gas_price.rs | 4 - crates/fuel-core/src/schema/message.rs | 6 +- crates/fuel-core/src/schema/relayed_tx.rs | 7 +- crates/fuel-core/src/schema/tx.rs | 4 - crates/fuel-core/src/schema/tx/types.rs | 5 - crates/fuel-core/src/schema/upgrades.rs | 1 - 27 files changed, 146 insertions(+), 419 deletions(-) delete mode 100644 crates/fuel-core/src/query/chain.rs diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index 532b67a1bb5..07ca65bd5d6 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -1002,7 +1002,6 @@ mod tests { } pub fn owned_coins(&self, owner: &Address) -> Vec { - use crate::query::CoinQueryData; let query = self.service_database(); let query = query.test_view(); query @@ -1013,7 +1012,6 @@ mod tests { } pub fn owned_messages(&self, owner: &Address) -> Vec { - use crate::query::MessageQueryData; let query = self.service_database(); let query = query.test_view(); query diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 418e14c1318..dfaabd42733 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -1,12 +1,6 @@ use crate::fuel_core_graphql_api::{ database::arc_wrapper::ArcWrapper, ports::{ - DatabaseBlocks, - DatabaseChain, - DatabaseContracts, - DatabaseMessageProof, - DatabaseMessages, - DatabaseRelayedTransactions, OffChainDatabase, OnChainDatabase, }, @@ -72,8 +66,6 @@ use std::{ sync::Arc, }; -use super::ports::DatabaseDaCompressedBlocks; - mod arc_wrapper; /// The on-chain view of the database used by the [`ReadView`] to fetch on-chain data. @@ -132,13 +124,13 @@ impl ReadDatabase { #[derive(Clone)] pub struct ReadView { - genesis_height: BlockHeight, - on_chain: OnChainView, - off_chain: OffChainView, + pub(crate) genesis_height: BlockHeight, + pub(crate) on_chain: OnChainView, + pub(crate) off_chain: OffChainView, } -impl DatabaseBlocks for ReadView { - fn transaction(&self, tx_id: &TxId) -> StorageResult { +impl ReadView { + pub fn transaction(&self, tx_id: &TxId) -> StorageResult { let result = self.on_chain.transaction(tx_id); if result.is_not_found() { if let Some(tx) = self.old_transaction(tx_id)? { @@ -151,7 +143,7 @@ impl DatabaseBlocks for ReadView { } } - fn block(&self, height: &BlockHeight) -> StorageResult { + pub fn block(&self, height: &BlockHeight) -> StorageResult { if *height >= self.genesis_height { self.on_chain.block(height) } else { @@ -159,7 +151,7 @@ impl DatabaseBlocks for ReadView { } } - fn blocks( + pub fn blocks( &self, height: Option, direction: IterDirection, @@ -202,11 +194,11 @@ impl DatabaseBlocks for ReadView { } } - fn latest_height(&self) -> StorageResult { + pub fn latest_height(&self) -> StorageResult { self.on_chain.latest_height() } - fn consensus(&self, id: &BlockHeight) -> StorageResult { + pub fn consensus(&self, id: &BlockHeight) -> StorageResult { if *id >= self.genesis_height { self.on_chain.consensus(id) } else { @@ -215,45 +207,34 @@ impl DatabaseBlocks for ReadView { } } -impl DatabaseDaCompressedBlocks for ReadView { - fn da_compressed_block(&self, id: &BlockHeight) -> StorageResult> { - self.off_chain.da_compressed_block(id) - } - - fn latest_height(&self) -> StorageResult { - self.on_chain.latest_height() - } -} - -impl StorageInspect for ReadView -where - M: Mappable, - dyn OnChainDatabase: StorageInspect, -{ +impl StorageInspect for ReadView { type Error = StorageError; - fn get(&self, key: &M::Key) -> StorageResult>> { - self.on_chain.get(key) + fn get( + &self, + key: &::Key, + ) -> StorageResult::OwnedValue>>> { + StorageInspect::::get(self.on_chain.as_ref(), key) } - fn contains_key(&self, key: &M::Key) -> StorageResult { - self.on_chain.contains_key(key) + fn contains_key(&self, key: &::Key) -> StorageResult { + StorageInspect::::contains_key(self.on_chain.as_ref(), key) } } impl StorageSize for ReadView { fn size_of_value(&self, key: &BlobId) -> Result, Self::Error> { - self.on_chain.size_of_value(key) + StorageSize::::size_of_value(self.on_chain.as_ref(), key) } } impl StorageRead for ReadView { fn read(&self, key: &BlobId, buf: &mut [u8]) -> Result, Self::Error> { - self.on_chain.read(key, buf) + StorageRead::::read(self.on_chain.as_ref(), key, buf) } fn read_alloc(&self, key: &BlobId) -> Result>, Self::Error> { - self.on_chain.read_alloc(key) + StorageRead::::read_alloc(self.on_chain.as_ref(), key) } } @@ -263,8 +244,8 @@ impl PredicateStorageRequirements for ReadView { } } -impl DatabaseMessages for ReadView { - fn all_messages( +impl ReadView { + pub fn all_messages( &self, start_message_id: Option, direction: IterDirection, @@ -272,23 +253,18 @@ impl DatabaseMessages for ReadView { self.on_chain.all_messages(start_message_id, direction) } - fn message_exists(&self, nonce: &Nonce) -> StorageResult { + pub fn message_exists(&self, nonce: &Nonce) -> StorageResult { self.on_chain.message_exists(nonce) } -} -impl DatabaseRelayedTransactions for ReadView { - fn transaction_status( + pub fn relayed_transaction_status( &self, id: Bytes32, ) -> StorageResult> { - let maybe_status = self.off_chain.relayed_tx_status(id)?; - Ok(maybe_status) + self.off_chain.relayed_tx_status(id) } -} -impl DatabaseContracts for ReadView { - fn contract_balances( + pub fn contract_balances( &self, contract: ContractId, start_asset: Option, @@ -297,16 +273,12 @@ impl DatabaseContracts for ReadView { self.on_chain .contract_balances(contract, start_asset, direction) } -} -impl DatabaseChain for ReadView { - fn da_height(&self) -> StorageResult { + pub fn da_height(&self) -> StorageResult { self.on_chain.da_height() } -} -impl DatabaseMessageProof for ReadView { - fn block_history_proof( + pub fn block_history_proof( &self, message_block_height: &BlockHeight, commit_block_height: &BlockHeight, @@ -316,22 +288,20 @@ impl DatabaseMessageProof for ReadView { } } -impl OnChainDatabase for ReadView {} - -impl OffChainDatabase for ReadView { - fn block_height(&self, block_id: &BlockId) -> StorageResult { +impl ReadView { + pub fn block_height(&self, block_id: &BlockId) -> StorageResult { self.off_chain.block_height(block_id) } - fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult> { + pub fn da_compressed_block(&self, height: &BlockHeight) -> StorageResult> { self.off_chain.da_compressed_block(height) } - fn tx_status(&self, tx_id: &TxId) -> StorageResult { + pub fn tx_status(&self, tx_id: &TxId) -> StorageResult { self.off_chain.tx_status(tx_id) } - fn owned_coins_ids( + pub fn owned_coins_ids( &self, owner: &Address, start_coin: Option, @@ -340,7 +310,7 @@ impl OffChainDatabase for ReadView { self.off_chain.owned_coins_ids(owner, start_coin, direction) } - fn owned_message_ids( + pub fn owned_message_ids( &self, owner: &Address, start_message_id: Option, @@ -350,7 +320,7 @@ impl OffChainDatabase for ReadView { .owned_message_ids(owner, start_message_id, direction) } - fn owned_transactions_ids( + pub fn owned_transactions_ids( &self, owner: Address, start: Option, @@ -360,15 +330,15 @@ impl OffChainDatabase for ReadView { .owned_transactions_ids(owner, start, direction) } - fn contract_salt(&self, contract_id: &ContractId) -> StorageResult { + pub fn contract_salt(&self, contract_id: &ContractId) -> StorageResult { self.off_chain.contract_salt(contract_id) } - fn old_block(&self, height: &BlockHeight) -> StorageResult { + pub fn old_block(&self, height: &BlockHeight) -> StorageResult { self.off_chain.old_block(height) } - fn old_blocks( + pub fn old_blocks( &self, height: Option, direction: IterDirection, @@ -376,25 +346,25 @@ impl OffChainDatabase for ReadView { self.off_chain.old_blocks(height, direction) } - fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult { + pub fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult { self.off_chain.old_block_consensus(height) } - fn old_transaction( + pub fn old_transaction( &self, id: &TxId, ) -> StorageResult> { self.off_chain.old_transaction(id) } - fn relayed_tx_status( + pub fn relayed_tx_status( &self, id: Bytes32, ) -> StorageResult> { self.off_chain.relayed_tx_status(id) } - fn message_is_spent(&self, nonce: &Nonce) -> StorageResult { + pub fn message_is_spent(&self, nonce: &Nonce) -> StorageResult { self.off_chain.message_is_spent(nonce) } } diff --git a/crates/fuel-core/src/query.rs b/crates/fuel-core/src/query.rs index fc2dc79ea9b..0dc744bcec0 100644 --- a/crates/fuel-core/src/query.rs +++ b/crates/fuel-core/src/query.rs @@ -1,7 +1,6 @@ mod balance; mod blob; mod block; -mod chain; mod coin; mod contract; mod message; @@ -13,12 +12,5 @@ pub mod da_compressed; // TODO: Remove reexporting of everything pub use balance::*; -pub use blob::*; -pub use block::*; -pub use chain::*; -pub use coin::*; -pub use contract::*; pub use message::*; pub(crate) use subscriptions::*; -pub use tx::*; -pub use upgrades::*; diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index ecbc47620bd..34e7d5c8d95 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -27,24 +27,8 @@ use std::{ pub mod asset_query; -pub trait BalanceQueryData: Send + Sync { - fn balance( - &self, - owner: Address, - asset_id: AssetId, - base_asset_id: AssetId, - ) -> StorageResult; - - fn balances( - &self, - owner: Address, - direction: IterDirection, - base_asset_id: AssetId, - ) -> BoxedIter>; -} - -impl BalanceQueryData for ReadView { - fn balance( +impl ReadView { + pub fn balance( &self, owner: Address, asset_id: AssetId, @@ -75,7 +59,7 @@ impl BalanceQueryData for ReadView { }) } - fn balances( + pub fn balances( &self, owner: Address, direction: IterDirection, diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index 57225b07930..e9ecf206aed 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -1,10 +1,4 @@ -use crate::{ - graphql_api::database::ReadView, - query::{ - CoinQueryData, - MessageQueryData, - }, -}; +use crate::graphql_api::database::ReadView; use fuel_core_storage::{ iter::IterDirection, Error as StorageError, diff --git a/crates/fuel-core/src/query/blob.rs b/crates/fuel-core/src/query/blob.rs index a3987c4ff10..6c7f9e7de8d 100644 --- a/crates/fuel-core/src/query/blob.rs +++ b/crates/fuel-core/src/query/blob.rs @@ -1,7 +1,4 @@ -use crate::graphql_api::ports::{ - OffChainDatabase, - OnChainDatabase, -}; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ not_found, tables::BlobData, @@ -10,17 +7,12 @@ use fuel_core_storage::{ }; use fuel_core_types::fuel_tx::BlobId; -pub trait BlobQueryData: Send + Sync { - fn blob_exists(&self, id: BlobId) -> StorageResult; - fn blob_bytecode(&self, id: BlobId) -> StorageResult>; -} - -impl BlobQueryData for D { - fn blob_exists(&self, id: BlobId) -> StorageResult { +impl ReadView { + pub fn blob_exists(&self, id: BlobId) -> StorageResult { self.storage::().contains_key(&id) } - fn blob_bytecode(&self, id: BlobId) -> StorageResult> { + pub fn blob_bytecode(&self, id: BlobId) -> StorageResult> { let blob = self .storage::() .get(&id)? diff --git a/crates/fuel-core/src/query/block.rs b/crates/fuel-core/src/query/block.rs index 9c365a3ef26..6ef7cf38afa 100644 --- a/crates/fuel-core/src/query/block.rs +++ b/crates/fuel-core/src/query/block.rs @@ -1,4 +1,4 @@ -use crate::fuel_core_graphql_api::ports::DatabaseBlocks; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ iter::{ BoxedIter, @@ -7,61 +7,24 @@ use fuel_core_storage::{ Result as StorageResult, }; use fuel_core_types::{ - blockchain::{ - block::CompressedBlock, - consensus::Consensus, - }, + blockchain::block::CompressedBlock, fuel_types::BlockHeight, }; -pub trait SimpleBlockData: Send + Sync { - fn block(&self, id: &BlockHeight) -> StorageResult; -} - -impl SimpleBlockData for D -where - D: DatabaseBlocks + ?Sized + Send + Sync, -{ - fn block(&self, id: &BlockHeight) -> StorageResult { - self.block(id) - } -} - -pub trait BlockQueryData: Send + Sync + SimpleBlockData { - fn latest_block_height(&self) -> StorageResult; - - fn latest_block(&self) -> StorageResult; - - fn compressed_blocks( - &self, - height: Option, - direction: IterDirection, - ) -> BoxedIter>; - - fn consensus(&self, id: &BlockHeight) -> StorageResult; -} - -impl BlockQueryData for D -where - D: DatabaseBlocks + ?Sized + Send + Sync, -{ - fn latest_block_height(&self) -> StorageResult { +impl ReadView { + pub fn latest_block_height(&self) -> StorageResult { self.latest_height() } - fn latest_block(&self) -> StorageResult { + pub fn latest_block(&self) -> StorageResult { self.block(&self.latest_block_height()?) } - fn compressed_blocks( + pub fn compressed_blocks( &self, height: Option, direction: IterDirection, ) -> BoxedIter> { self.blocks(height, direction) } - - fn consensus(&self, id: &BlockHeight) -> StorageResult { - self.consensus(id) - } } diff --git a/crates/fuel-core/src/query/chain.rs b/crates/fuel-core/src/query/chain.rs deleted file mode 100644 index aebf442cf30..00000000000 --- a/crates/fuel-core/src/query/chain.rs +++ /dev/null @@ -1,13 +0,0 @@ -use crate::fuel_core_graphql_api::ports::OnChainDatabase; -use fuel_core_storage::Result as StorageResult; -use fuel_core_types::blockchain::primitives::DaBlockHeight; - -pub trait ChainQueryData: Send + Sync { - fn da_height(&self) -> StorageResult; -} - -impl ChainQueryData for D { - fn da_height(&self) -> StorageResult { - self.da_height() - } -} diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 171a88168bd..fc944a623f5 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,7 +1,4 @@ -use crate::fuel_core_graphql_api::ports::{ - OffChainDatabase, - OnChainDatabase, -}; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ iter::{ BoxedIter, @@ -19,27 +16,11 @@ use fuel_core_types::{ fuel_types::Address, }; -pub trait CoinQueryData: Send + Sync { - fn coin(&self, utxo_id: UtxoId) -> StorageResult; - - fn owned_coins_ids( - &self, - owner: &Address, - start_coin: Option, - direction: IterDirection, - ) -> BoxedIter>; - - fn owned_coins( - &self, - owner: &Address, - start_coin: Option, - direction: IterDirection, - ) -> BoxedIter>; -} - -impl CoinQueryData for D { - fn coin(&self, utxo_id: UtxoId) -> StorageResult { +impl ReadView { + pub fn coin(&self, utxo_id: UtxoId) -> StorageResult { let coin = self + .on_chain + .as_ref() .storage::() .get(&utxo_id)? .ok_or(not_found!(Coins))? @@ -48,16 +29,7 @@ impl CoinQueryData for D { Ok(coin.uncompress(utxo_id)) } - fn owned_coins_ids( - &self, - owner: &Address, - start_coin: Option, - direction: IterDirection, - ) -> BoxedIter> { - self.owned_coins_ids(owner, start_coin, direction) - } - - fn owned_coins( + pub fn owned_coins( &self, owner: &Address, start_coin: Option, diff --git a/crates/fuel-core/src/query/contract.rs b/crates/fuel-core/src/query/contract.rs index 3311fec5dd9..fa75e05a874 100644 --- a/crates/fuel-core/src/query/contract.rs +++ b/crates/fuel-core/src/query/contract.rs @@ -1,12 +1,5 @@ -use crate::fuel_core_graphql_api::ports::{ - OffChainDatabase, - OnChainDatabase, -}; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ - iter::{ - BoxedIter, - IterDirection, - }, not_found, tables::{ ContractsAssets, @@ -20,38 +13,21 @@ use fuel_core_types::{ AssetId, ContractId, }, - fuel_vm::Salt, services::graphql_api::ContractBalance, }; -pub trait ContractQueryData: Send + Sync { - fn contract_exists(&self, id: ContractId) -> StorageResult; - - fn contract_bytecode(&self, id: ContractId) -> StorageResult>; - - fn contract_salt(&self, id: ContractId) -> StorageResult; - - fn contract_balance( - &self, - contract_id: ContractId, - asset_id: AssetId, - ) -> StorageResult; - - fn contract_balances( - &self, - contract_id: ContractId, - start_asset: Option, - direction: IterDirection, - ) -> BoxedIter>; -} - -impl ContractQueryData for D { - fn contract_exists(&self, id: ContractId) -> StorageResult { - self.storage::().contains_key(&id) +impl ReadView { + pub fn contract_exists(&self, id: ContractId) -> StorageResult { + self.on_chain + .as_ref() + .storage::() + .contains_key(&id) } - fn contract_bytecode(&self, id: ContractId) -> StorageResult> { + pub fn contract_bytecode(&self, id: ContractId) -> StorageResult> { let contract = self + .on_chain + .as_ref() .storage::() .get(&id)? .ok_or(not_found!(ContractsRawCode))? @@ -60,16 +36,14 @@ impl ContractQueryData for D { Ok(contract.into()) } - fn contract_salt(&self, id: ContractId) -> StorageResult { - self.contract_salt(&id) - } - - fn contract_balance( + pub fn contract_balance( &self, contract_id: ContractId, asset_id: AssetId, ) -> StorageResult { let amount = self + .on_chain + .as_ref() .storage::() .get(&(&contract_id, &asset_id).into())? .ok_or(not_found!(ContractsAssets))? @@ -81,13 +55,4 @@ impl ContractQueryData for D { asset_id, }) } - - fn contract_balances( - &self, - contract_id: ContractId, - start_asset: Option, - direction: IterDirection, - ) -> BoxedIter> { - self.contract_balances(contract_id, start_asset, direction) - } } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index ece4a93fec7..aedadd47949 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -1,19 +1,6 @@ -use crate::{ - fuel_core_graphql_api::{ - ports::{ - DatabaseBlocks, - DatabaseMessageProof, - DatabaseMessages, - OffChainDatabase, - OnChainDatabase, - }, - IntoApiResult, - }, - query::{ - SimpleBlockData, - SimpleTransactionData, - TransactionQueryData, - }, +use crate::fuel_core_graphql_api::{ + database::ReadView, + IntoApiResult, }; use fuel_core_storage::{ iter::{ @@ -39,6 +26,7 @@ use fuel_core_types::{ fuel_tx::{ input::message::compute_message_id, Receipt, + Transaction, TxId, }, fuel_types::{ @@ -80,24 +68,17 @@ pub trait MessageQueryData: Send + Sync { ) -> BoxedIter>; } -impl MessageQueryData for D { - fn message(&self, id: &Nonce) -> StorageResult { - self.storage::() +impl ReadView { + pub fn message(&self, id: &Nonce) -> StorageResult { + self.on_chain + .as_ref() + .storage::() .get(id)? .ok_or(not_found!(Messages)) .map(Cow::into_owned) } - fn owned_message_ids( - &self, - owner: &Address, - start_message_id: Option, - direction: IterDirection, - ) -> BoxedIter> { - self.owned_message_ids(owner, start_message_id, direction) - } - - fn owned_messages( + pub fn owned_messages( &self, owner: &Address, start_message_id: Option, @@ -107,37 +88,61 @@ impl MessageQueryData for D { .map(|result| result.and_then(|id| self.message(&id))) .into_boxed() } - - fn all_messages( - &self, - start_message_id: Option, - direction: IterDirection, - ) -> BoxedIter> { - self.all_messages(start_message_id, direction) - } } /// Trait that specifies all the data required by the output message query. -pub trait MessageProofData: - Send + Sync + SimpleBlockData + SimpleTransactionData + DatabaseMessageProof -{ +pub trait MessageProofData { + /// Get the block. + fn block(&self, id: &BlockHeight) -> StorageResult; + + /// Get the transaction. + fn transaction(&self, transaction_id: &TxId) -> StorageResult; + + /// Return all receipts in the given transaction. + fn receipts(&self, transaction_id: &TxId) -> StorageResult>; + /// Get the status of a transaction. fn transaction_status( &self, transaction_id: &TxId, ) -> StorageResult; + + /// Gets the [`MerkleProof`] for the message block at `message_block_height` height + /// relatively to the commit block where message block <= commit block. + fn block_history_proof( + &self, + message_block_height: &BlockHeight, + commit_block_height: &BlockHeight, + ) -> StorageResult; } -impl MessageProofData for D -where - D: OnChainDatabase + DatabaseBlocks + OffChainDatabase + ?Sized, -{ +impl MessageProofData for ReadView { + fn block(&self, id: &BlockHeight) -> StorageResult { + self.block(id) + } + + fn transaction(&self, transaction_id: &TxId) -> StorageResult { + self.transaction(transaction_id) + } + + fn receipts(&self, transaction_id: &TxId) -> StorageResult> { + self.receipts(transaction_id) + } + fn transaction_status( &self, transaction_id: &TxId, ) -> StorageResult { self.status(transaction_id) } + + fn block_history_proof( + &self, + message_block_height: &BlockHeight, + commit_block_height: &BlockHeight, + ) -> StorageResult { + self.block_history_proof(message_block_height, commit_block_height) + } } /// Generate an output proof. @@ -282,13 +287,10 @@ fn message_receipts_proof( } } -pub fn message_status( - database: &T, +pub fn message_status( + database: &ReadView, message_nonce: Nonce, -) -> StorageResult -where - T: OffChainDatabase + DatabaseMessages + ?Sized, -{ +) -> StorageResult { if database.message_is_spent(&message_nonce)? { Ok(MessageStatus::spent()) } else if database.message_exists(&message_nonce)? { diff --git a/crates/fuel-core/src/query/message/test.rs b/crates/fuel-core/src/query/message/test.rs index 3078f3a7de9..d49f173370b 100644 --- a/crates/fuel-core/src/query/message/test.rs +++ b/crates/fuel-core/src/query/message/test.rs @@ -57,24 +57,15 @@ fn receipt(i: Option) -> Receipt { mockall::mock! { pub ProofDataStorage {} - impl SimpleBlockData for ProofDataStorage { + impl MessageProofData for ProofDataStorage { fn block(&self, height: &BlockHeight) -> StorageResult; - } - - impl DatabaseMessageProof for ProofDataStorage { fn block_history_proof( &self, message_block_height: &BlockHeight, commit_block_height: &BlockHeight, ) -> StorageResult; - } - - impl SimpleTransactionData for ProofDataStorage { fn transaction(&self, transaction_id: &TxId) -> StorageResult; fn receipts(&self, transaction_id: &TxId) -> StorageResult>; - } - - impl MessageProofData for ProofDataStorage { fn transaction_status(&self, transaction_id: &TxId) -> StorageResult; } } diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 1d2f1531363..894d05d9c6f 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -1,8 +1,4 @@ -use crate::fuel_core_graphql_api::ports::{ - DatabaseBlocks, - OffChainDatabase, - OnChainDatabase, -}; +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ iter::{ @@ -25,23 +21,8 @@ use fuel_core_types::{ services::txpool::TransactionStatus, }; -pub trait SimpleTransactionData: Send + Sync { - /// Return all receipts in the given transaction. - fn receipts(&self, transaction_id: &TxId) -> StorageResult>; - - /// Get the transaction. - fn transaction(&self, transaction_id: &TxId) -> StorageResult; -} - -impl SimpleTransactionData for D -where - D: OnChainDatabase + DatabaseBlocks + OffChainDatabase + ?Sized, -{ - fn transaction(&self, tx_id: &TxId) -> StorageResult { - self.transaction(tx_id) - } - - fn receipts(&self, tx_id: &TxId) -> StorageResult> { +impl ReadView { + pub fn receipts(&self, tx_id: &TxId) -> StorageResult> { let status = self.status(tx_id)?; let receipts = match status { @@ -51,28 +32,12 @@ where }; receipts.ok_or(not_found!(Transactions)) } -} - -pub trait TransactionQueryData: Send + Sync + SimpleTransactionData { - fn status(&self, tx_id: &TxId) -> StorageResult; - - fn owned_transactions( - &self, - owner: Address, - start: Option, - direction: IterDirection, - ) -> BoxedIter>; -} -impl TransactionQueryData for D -where - D: OnChainDatabase + DatabaseBlocks + OffChainDatabase + ?Sized, -{ - fn status(&self, tx_id: &TxId) -> StorageResult { + pub fn status(&self, tx_id: &TxId) -> StorageResult { self.tx_status(tx_id) } - fn owned_transactions( + pub fn owned_transactions( &self, owner: Address, start: Option, diff --git a/crates/fuel-core/src/query/upgrades.rs b/crates/fuel-core/src/query/upgrades.rs index 4a53edbe12a..850b7e6973f 100644 --- a/crates/fuel-core/src/query/upgrades.rs +++ b/crates/fuel-core/src/query/upgrades.rs @@ -1,3 +1,4 @@ +use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ not_found, tables::{ @@ -13,27 +14,14 @@ use fuel_core_types::{ fuel_vm::UploadedBytecode, }; -use crate::graphql_api::ports::OnChainDatabase; - -pub trait UpgradeQueryData: Send + Sync { - fn state_transition_bytecode_root( - &self, - version: StateTransitionBytecodeVersion, - ) -> StorageResult; - - fn state_transition_bytecode(&self, root: Bytes32) - -> StorageResult; -} - -impl UpgradeQueryData for D -where - D: OnChainDatabase + ?Sized, -{ - fn state_transition_bytecode_root( +impl ReadView { + pub fn state_transition_bytecode_root( &self, version: StateTransitionBytecodeVersion, ) -> StorageResult { let merkle_root = self + .on_chain + .as_ref() .storage::() .get(&version)? .ok_or(not_found!(StateTransitionBytecodeVersions))? @@ -42,11 +30,13 @@ where Ok(merkle_root) } - fn state_transition_bytecode( + pub fn state_transition_bytecode( &self, root: Bytes32, ) -> StorageResult { let bytecode = self + .on_chain + .as_ref() .storage::() .get(&root)? .ok_or(not_found!(UploadedBytecodes))? diff --git a/crates/fuel-core/src/schema/balance.rs b/crates/fuel-core/src/schema/balance.rs index 6f83a831449..7c43e3a9509 100644 --- a/crates/fuel-core/src/schema/balance.rs +++ b/crates/fuel-core/src/schema/balance.rs @@ -3,7 +3,6 @@ use crate::{ api_service::ConsensusProvider, QUERY_COSTS, }, - query::BalanceQueryData, schema::{ scalars::{ Address, diff --git a/crates/fuel-core/src/schema/blob.rs b/crates/fuel-core/src/schema/blob.rs index 938c2a6a1f3..93974346f1f 100644 --- a/crates/fuel-core/src/schema/blob.rs +++ b/crates/fuel-core/src/schema/blob.rs @@ -1,7 +1,6 @@ use crate::{ fuel_core_graphql_api::QUERY_COSTS, graphql_api::IntoApiResult, - query::BlobQueryData, schema::{ scalars::{ BlobId, diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index 81a9bf3c0af..cebd04989b6 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -7,16 +7,10 @@ use crate::{ fuel_core_graphql_api::{ api_service::ConsensusModule, database::ReadView, - ports::OffChainDatabase, Config as GraphQLConfig, IntoApiResult, QUERY_COSTS, }, - query::{ - BlockQueryData, - SimpleBlockData, - SimpleTransactionData, - }, schema::{ scalars::{ BlockId, diff --git a/crates/fuel-core/src/schema/chain.rs b/crates/fuel-core/src/schema/chain.rs index 16ec77b1e46..643902cc0c5 100644 --- a/crates/fuel-core/src/schema/chain.rs +++ b/crates/fuel-core/src/schema/chain.rs @@ -4,10 +4,6 @@ use crate::{ QUERY_COSTS, }, graphql_api::Config, - query::{ - BlockQueryData, - ChainQueryData, - }, schema::{ block::Block, scalars::{ diff --git a/crates/fuel-core/src/schema/coins.rs b/crates/fuel-core/src/schema/coins.rs index 0d0abc97da2..3610c152e6b 100644 --- a/crates/fuel-core/src/schema/coins.rs +++ b/crates/fuel-core/src/schema/coins.rs @@ -8,10 +8,7 @@ use crate::{ QUERY_COSTS, }, graphql_api::api_service::ConsensusProvider, - query::{ - asset_query::AssetSpendTarget, - CoinQueryData, - }, + query::asset_query::AssetSpendTarget, schema::{ scalars::{ Address, diff --git a/crates/fuel-core/src/schema/contract.rs b/crates/fuel-core/src/schema/contract.rs index 2abc0e53a06..a8fe6dea693 100644 --- a/crates/fuel-core/src/schema/contract.rs +++ b/crates/fuel-core/src/schema/contract.rs @@ -3,7 +3,6 @@ use crate::{ IntoApiResult, QUERY_COSTS, }, - query::ContractQueryData, schema::{ scalars::{ AssetId, @@ -60,7 +59,7 @@ impl Contract { async fn salt(&self, ctx: &Context<'_>) -> async_graphql::Result { let query = ctx.read_view()?; query - .contract_salt(self.0) + .contract_salt(&self.0) .map(Into::into) .map_err(Into::into) } diff --git a/crates/fuel-core/src/schema/da_compressed.rs b/crates/fuel-core/src/schema/da_compressed.rs index 3af336f8ba9..9badfb1b8c1 100644 --- a/crates/fuel-core/src/schema/da_compressed.rs +++ b/crates/fuel-core/src/schema/da_compressed.rs @@ -7,7 +7,6 @@ use crate::{ IntoApiResult, QUERY_COSTS, }, - query::da_compressed::DaCompressedBlockData, schema::scalars::U32, }; use async_graphql::{ diff --git a/crates/fuel-core/src/schema/gas_price.rs b/crates/fuel-core/src/schema/gas_price.rs index f1ded5fc106..d2a80d67f9d 100644 --- a/crates/fuel-core/src/schema/gas_price.rs +++ b/crates/fuel-core/src/schema/gas_price.rs @@ -7,10 +7,6 @@ use crate::{ api_service::GasPriceProvider, QUERY_COSTS, }, - query::{ - BlockQueryData, - SimpleTransactionData, - }, schema::ReadViewProvider, }; use async_graphql::{ diff --git a/crates/fuel-core/src/schema/message.rs b/crates/fuel-core/src/schema/message.rs index 8ba18784bf8..05416623934 100644 --- a/crates/fuel-core/src/schema/message.rs +++ b/crates/fuel-core/src/schema/message.rs @@ -11,12 +11,8 @@ use super::{ ReadViewProvider, }; use crate::{ - fuel_core_graphql_api::{ - ports::OffChainDatabase, - QUERY_COSTS, - }, + fuel_core_graphql_api::QUERY_COSTS, graphql_api::IntoApiResult, - query::MessageQueryData, schema::scalars::{ BlockId, U32, diff --git a/crates/fuel-core/src/schema/relayed_tx.rs b/crates/fuel-core/src/schema/relayed_tx.rs index 1bc915df1bf..850c67cda4b 100644 --- a/crates/fuel-core/src/schema/relayed_tx.rs +++ b/crates/fuel-core/src/schema/relayed_tx.rs @@ -1,8 +1,5 @@ use crate::{ - fuel_core_graphql_api::{ - ports::DatabaseRelayedTransactions, - QUERY_COSTS, - }, + fuel_core_graphql_api::QUERY_COSTS, schema::{ scalars::{ RelayedTransactionId, @@ -33,7 +30,7 @@ impl RelayedTransactionQuery { #[graphql(desc = "The id of the relayed tx")] id: RelayedTransactionId, ) -> async_graphql::Result> { let query = ctx.read_view()?; - let status = query.transaction_status(id.0)?.map(|status| status.into()); + let status = query.relayed_tx_status(id.0)?.map(|status| status.into()); Ok(status) } } diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index d0a1474340a..89b20101db0 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -6,7 +6,6 @@ use crate::{ ConsensusProvider, TxPool, }, - ports::OffChainDatabase, IntoApiResult, QUERY_COSTS, }, @@ -16,9 +15,6 @@ use crate::{ }, query::{ transaction_status_change, - BlockQueryData, - SimpleTransactionData, - TransactionQueryData, TxnStatusChangeState, }, schema::{ diff --git a/crates/fuel-core/src/schema/tx/types.rs b/crates/fuel-core/src/schema/tx/types.rs index 820271430e2..097d6f06fdd 100644 --- a/crates/fuel-core/src/schema/tx/types.rs +++ b/crates/fuel-core/src/schema/tx/types.rs @@ -13,11 +13,6 @@ use crate::{ IntoApiResult, QUERY_COSTS, }, - query::{ - SimpleBlockData, - SimpleTransactionData, - TransactionQueryData, - }, schema::{ block::Block, scalars::{ diff --git a/crates/fuel-core/src/schema/upgrades.rs b/crates/fuel-core/src/schema/upgrades.rs index d9e62906a1e..4df3efddf72 100644 --- a/crates/fuel-core/src/schema/upgrades.rs +++ b/crates/fuel-core/src/schema/upgrades.rs @@ -4,7 +4,6 @@ use crate::{ IntoApiResult, QUERY_COSTS, }, - query::UpgradeQueryData, schema::{ chain::ConsensusParameters, scalars::HexString, From ab5e940ffed387f165bd4b7c94204c7169bc71c6 Mon Sep 17 00:00:00 2001 From: green Date: Fri, 11 Oct 2024 19:40:54 +0200 Subject: [PATCH 02/19] Updated CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e5da278c28..35b21a83e99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Changed + +- [2334](https://github.com/FuelLabs/fuel-core/pull/2334): Prepare the GraphQL service for the switching to `async` methods. + ## [Version 0.39.0] ### Added From 178268489b0378fd0d2b491e44cb245623a21cbb Mon Sep 17 00:00:00 2001 From: green Date: Sat, 12 Oct 2024 17:25:06 +0200 Subject: [PATCH 03/19] Updated all pagination queries to work with the `Stream` instead of `Iterator` --- crates/fuel-core/src/coins_query.rs | 323 ++++++++++-------- crates/fuel-core/src/graphql_api/database.rs | 43 ++- crates/fuel-core/src/query/balance.rs | 112 +++--- .../src/query/balance/asset_query.rs | 86 +++-- crates/fuel-core/src/query/block.rs | 10 +- crates/fuel-core/src/query/coin.rs | 20 +- crates/fuel-core/src/query/message.rs | 21 +- crates/fuel-core/src/query/tx.rs | 15 +- crates/fuel-core/src/schema.rs | 10 +- crates/fuel-core/src/schema/balance.rs | 18 +- crates/fuel-core/src/schema/block.rs | 18 +- crates/fuel-core/src/schema/coins.rs | 19 +- crates/fuel-core/src/schema/contract.rs | 3 +- crates/fuel-core/src/schema/message.rs | 12 +- crates/fuel-core/src/schema/tx.rs | 58 ++-- .../src/state/rocks_db_key_iterator.rs | 4 +- crates/services/src/lib.rs | 17 +- crates/storage/src/iter.rs | 14 +- crates/storage/src/kv_store.rs | 7 +- 19 files changed, 464 insertions(+), 346 deletions(-) diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index 07ca65bd5d6..e07630333f8 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -19,7 +19,7 @@ use fuel_core_types::{ Word, }, }; -use itertools::Itertools; +use futures::TryStreamExt; use rand::prelude::*; use std::{ cmp::Reverse, @@ -119,8 +119,13 @@ impl SpendQuery { /// Returns the biggest inputs of the `owner` to satisfy the required `target` of the asset. The /// number of inputs for each asset can't exceed `max_inputs`, otherwise throw an error that query /// can't be satisfied. -pub fn largest_first(query: &AssetQuery) -> Result, CoinsQueryError> { - let mut inputs: Vec<_> = query.coins().try_collect()?; +pub async fn largest_first( + query: AssetQuery<'_>, +) -> Result, CoinsQueryError> { + let target = query.asset.target; + let max = query.asset.max; + let asset_id = query.asset.id; + let mut inputs: Vec = query.coins().try_collect().await?; inputs.sort_by_key(|coin| Reverse(coin.amount())); let mut collected_amount = 0u64; @@ -128,12 +133,12 @@ pub fn largest_first(query: &AssetQuery) -> Result, CoinsQueryErro for coin in inputs { // Break if we don't need any more coins - if collected_amount >= query.asset.target { + if collected_amount >= target { break } // Error if we can't fit more coins - if coins.len() >= query.asset.max { + if coins.len() >= max as usize { return Err(CoinsQueryError::MaxCoinsReached) } @@ -142,9 +147,9 @@ pub fn largest_first(query: &AssetQuery) -> Result, CoinsQueryErro coins.push(coin); } - if collected_amount < query.asset.target { + if collected_amount < target { return Err(CoinsQueryError::InsufficientCoins { - asset_id: query.asset.id, + asset_id, collected_amount, }) } @@ -153,23 +158,31 @@ pub fn largest_first(query: &AssetQuery) -> Result, CoinsQueryErro } // An implementation of the method described on: https://iohk.io/en/blog/posts/2018/07/03/self-organisation-in-coin-selection/ -pub fn random_improve( +// TODO: Reimplement this algorithm to be simpler and faster: +// Instead of selecting random coins first, we can sort them. +// After that, we can split the coins into the part that covers the +// target and the part that does not(by choosing the most expensive coins). +// When the target is satisfied, we can select random coins from the remaining +// coins not used in the target. +pub async fn random_improve( db: &ReadView, spend_query: &SpendQuery, ) -> Result>, CoinsQueryError> { let mut coins_per_asset = vec![]; for query in spend_query.asset_queries(db) { - let mut inputs: Vec<_> = query.coins().try_collect()?; + let target = query.asset.target; + let max = query.asset.max; + + let mut inputs: Vec<_> = query.clone().coins().try_collect().await?; inputs.shuffle(&mut thread_rng()); - inputs.truncate(query.asset.max); + inputs.truncate(max as usize); let mut collected_amount = 0; let mut coins = vec![]; // Set parameters according to spec - let target = query.asset.target; - let upper_target = query.asset.target.saturating_mul(2); + let upper_target = target.saturating_mul(2); for coin in inputs { // Try to improve the result by adding dust to the result. @@ -197,8 +210,8 @@ pub fn random_improve( } // Fallback to largest_first if we can't fit more coins - if collected_amount < query.asset.target { - swap(&mut coins, &mut largest_first(&query)?); + if collected_amount < target { + swap(&mut coins, &mut largest_first(query).await?); } coins_per_asset.push(coins); @@ -266,6 +279,10 @@ mod tests { fuel_asm::Word, fuel_tx::*, }; + use futures::{ + StreamExt, + TryStreamExt, + }; use itertools::Itertools; use rand::{ rngs::StdRng, @@ -324,34 +341,36 @@ mod tests { mod largest_first { use super::*; - fn query( + async fn query( spend_query: &[AssetSpendTarget], owner: &Address, base_asset_id: &AssetId, db: &ServiceDatabase, ) -> Result>, CoinsQueryError> { - let result: Vec<_> = spend_query - .iter() - .map(|asset| { - largest_first(&AssetQuery::new( - owner, - asset, - base_asset_id, - None, - &db.test_view(), - )) - .map(|coins| { - coins - .iter() - .map(|coin| (*coin.asset_id(base_asset_id), coin.amount())) - .collect() - }) - }) - .try_collect()?; - Ok(result) + let mut results = vec![]; + + for asset in spend_query { + let coins = largest_first(AssetQuery::new( + owner, + asset, + base_asset_id, + None, + &db.test_view(), + )) + .await + .map(|coins| { + coins + .iter() + .map(|coin| (*coin.asset_id(base_asset_id), coin.amount())) + .collect() + })?; + results.push(coins); + } + + Ok(results) } - fn single_asset_assert( + async fn single_asset_assert( owner: Address, asset_ids: &[AssetId], base_asset_id: &AssetId, @@ -362,11 +381,12 @@ mod tests { // Query some targets, including higher than the owner's balance for target in 0..20 { let coins = query( - &[AssetSpendTarget::new(asset_id, target, usize::MAX)], + &[AssetSpendTarget::new(asset_id, target, u16::MAX)], &owner, base_asset_id, &db.service_database(), - ); + ) + .await; // Transform result for convenience let coins = coins.map(|coins| { @@ -425,32 +445,33 @@ mod tests { &owner, base_asset_id, &db.service_database(), - ); + ) + .await; assert_matches!(coins, Err(CoinsQueryError::MaxCoinsReached)); } - #[test] - fn single_asset_coins() { + #[tokio::test] + async fn single_asset_coins() { // Setup for coins let (owner, asset_ids, base_asset_id, db) = setup_coins(); - single_asset_assert(owner, &asset_ids, &base_asset_id, db); + single_asset_assert(owner, &asset_ids, &base_asset_id, db).await; } - #[test] - fn single_asset_messages() { + #[tokio::test] + async fn single_asset_messages() { // Setup for messages let (owner, base_asset_id, db) = setup_messages(); - single_asset_assert(owner, &[base_asset_id], &base_asset_id, db); + single_asset_assert(owner, &[base_asset_id], &base_asset_id, db).await; } - #[test] - fn single_asset_coins_and_messages() { + #[tokio::test] + async fn single_asset_coins_and_messages() { // Setup for coins and messages let (owner, asset_ids, base_asset_id, db) = setup_coins_and_messages(); - single_asset_assert(owner, &asset_ids, &base_asset_id, db); + single_asset_assert(owner, &asset_ids, &base_asset_id, db).await; } - fn multiple_assets_helper( + async fn multiple_assets_helper( owner: Address, asset_ids: &[AssetId], base_asset_id: &AssetId, @@ -458,13 +479,14 @@ mod tests { ) { let coins = query( &[ - AssetSpendTarget::new(asset_ids[0], 3, usize::MAX), - AssetSpendTarget::new(asset_ids[1], 6, usize::MAX), + AssetSpendTarget::new(asset_ids[0], 3, u16::MAX), + AssetSpendTarget::new(asset_ids[1], 6, u16::MAX), ], &owner, base_asset_id, &db.service_database(), - ); + ) + .await; let expected = vec![ vec![(asset_ids[0], 5)], vec![(asset_ids[1], 5), (asset_ids[1], 4)], @@ -472,25 +494,25 @@ mod tests { assert_matches!(coins, Ok(coins) if coins == expected); } - #[test] - fn multiple_assets_coins() { + #[tokio::test] + async fn multiple_assets_coins() { // Setup coins let (owner, asset_ids, base_asset_id, db) = setup_coins(); - multiple_assets_helper(owner, &asset_ids, &base_asset_id, db); + multiple_assets_helper(owner, &asset_ids, &base_asset_id, db).await; } - #[test] - fn multiple_assets_coins_and_messages() { + #[tokio::test] + async fn multiple_assets_coins_and_messages() { // Setup coins and messages let (owner, asset_ids, base_asset_id, db) = setup_coins_and_messages(); - multiple_assets_helper(owner, &asset_ids, &base_asset_id, db); + multiple_assets_helper(owner, &asset_ids, &base_asset_id, db).await; } } mod random_improve { use super::*; - fn query( + async fn query( query_per_asset: Vec, owner: Address, asset_ids: &[AssetId], @@ -500,7 +522,8 @@ mod tests { let coins = random_improve( &db.test_view(), &SpendQuery::new(owner, &query_per_asset, None, base_asset_id)?, - ); + ) + .await; // Transform result for convenience coins.map(|coins| { @@ -521,7 +544,7 @@ mod tests { }) } - fn single_asset_assert( + async fn single_asset_assert( owner: Address, asset_ids: &[AssetId], base_asset_id: AssetId, @@ -532,12 +555,13 @@ mod tests { // Query some amounts, including higher than the owner's balance for amount in 0..20 { let coins = query( - vec![AssetSpendTarget::new(asset_id, amount, usize::MAX)], + vec![AssetSpendTarget::new(asset_id, amount, u16::MAX)], owner, asset_ids, base_asset_id, &db.service_database(), - ); + ) + .await; // Transform result for convenience let coins = coins.map(|coins| { @@ -589,32 +613,33 @@ mod tests { asset_ids, base_asset_id, &db.service_database(), - ); + ) + .await; assert_matches!(coins, Err(CoinsQueryError::MaxCoinsReached)); } - #[test] - fn single_asset_coins() { + #[tokio::test] + async fn single_asset_coins() { // Setup for coins let (owner, asset_ids, base_asset_id, db) = setup_coins(); - single_asset_assert(owner, &asset_ids, base_asset_id, db); + single_asset_assert(owner, &asset_ids, base_asset_id, db).await; } - #[test] - fn single_asset_messages() { + #[tokio::test] + async fn single_asset_messages() { // Setup for messages let (owner, base_asset_id, db) = setup_messages(); - single_asset_assert(owner, &[base_asset_id], base_asset_id, db); + single_asset_assert(owner, &[base_asset_id], base_asset_id, db).await; } - #[test] - fn single_asset_coins_and_messages() { + #[tokio::test] + async fn single_asset_coins_and_messages() { // Setup for coins and messages let (owner, asset_ids, base_asset_id, db) = setup_coins_and_messages(); - single_asset_assert(owner, &asset_ids, base_asset_id, db); + single_asset_assert(owner, &asset_ids, base_asset_id, db).await; } - fn multiple_assets_assert( + async fn multiple_assets_assert( owner: Address, asset_ids: &[AssetId], base_asset_id: AssetId, @@ -638,7 +663,8 @@ mod tests { asset_ids, base_asset_id, &db.service_database(), - ); + ) + .await; assert_matches!(coins, Ok(ref coins) if coins.len() <= 6); let coins = coins.unwrap(); assert!( @@ -659,18 +685,18 @@ mod tests { ); } - #[test] - fn multiple_assets_coins() { + #[tokio::test] + async fn multiple_assets_coins() { // Setup coins let (owner, asset_ids, base_asset_id, db) = setup_coins(); - multiple_assets_assert(owner, &asset_ids, base_asset_id, db); + multiple_assets_assert(owner, &asset_ids, base_asset_id, db).await; } - #[test] - fn multiple_assets_coins_and_messages() { + #[tokio::test] + async fn multiple_assets_coins_and_messages() { // Setup coins and messages let (owner, asset_ids, base_asset_id, db) = setup_coins_and_messages(); - multiple_assets_assert(owner, &asset_ids, base_asset_id, db); + multiple_assets_assert(owner, &asset_ids, base_asset_id, db).await; } } @@ -678,7 +704,41 @@ mod tests { use super::*; use fuel_core_types::entities::coins::CoinId; - fn exclusion_assert( + async fn query( + db: &ServiceDatabase, + owner: Address, + base_asset_id: AssetId, + asset_ids: &[AssetId], + query_per_asset: Vec, + excluded_ids: Vec, + ) -> Result, CoinsQueryError> { + let spend_query = SpendQuery::new( + owner, + &query_per_asset, + Some(excluded_ids), + base_asset_id, + )?; + let coins = random_improve(&db.test_view(), &spend_query).await; + + // Transform result for convenience + coins.map(|coins| { + coins + .into_iter() + .flat_map(|coin| { + coin.into_iter() + .map(|coin| (*coin.asset_id(&base_asset_id), coin.amount())) + .sorted_by_key(|(asset_id, amount)| { + ( + asset_ids.iter().position(|c| c == asset_id).unwrap(), + Reverse(*amount), + ) + }) + }) + .collect() + }) + } + + async fn exclusion_assert( owner: Address, asset_ids: &[AssetId], base_asset_id: AssetId, @@ -687,47 +747,17 @@ mod tests { ) { let asset_id = asset_ids[0]; - let query = |query_per_asset: Vec, - excluded_ids: Vec| - -> Result, CoinsQueryError> { - let spend_query = SpendQuery::new( - owner, - &query_per_asset, - Some(excluded_ids), - base_asset_id, - )?; - let coins = - random_improve(&db.service_database().test_view(), &spend_query); - - // Transform result for convenience - coins.map(|coins| { - coins - .into_iter() - .flat_map(|coin| { - coin.into_iter() - .map(|coin| { - (*coin.asset_id(&base_asset_id), coin.amount()) - }) - .sorted_by_key(|(asset_id, amount)| { - ( - asset_ids - .iter() - .position(|c| c == asset_id) - .unwrap(), - Reverse(*amount), - ) - }) - }) - .collect() - }) - }; - // Query some amounts, including higher than the owner's balance for amount in 0..20 { let coins = query( - vec![AssetSpendTarget::new(asset_id, amount, usize::MAX)], + &db.service_database(), + owner, + base_asset_id, + asset_ids, + vec![AssetSpendTarget::new(asset_id, amount, u16::MAX)], excluded_ids.clone(), - ); + ) + .await; // Transform result for convenience let coins = coins.map(|coins| { @@ -769,52 +799,56 @@ mod tests { } } - #[test] - fn exclusion_coins() { + #[tokio::test] + async fn exclusion_coins() { // Setup coins let (owner, asset_ids, base_asset_id, db) = setup_coins(); // Exclude largest coin IDs let excluded_ids = db .owned_coins(&owner) + .await .into_iter() .filter(|coin| coin.amount == 5) .map(|coin| CoinId::Utxo(coin.utxo_id)) .collect_vec(); - exclusion_assert(owner, &asset_ids, base_asset_id, db, excluded_ids); + exclusion_assert(owner, &asset_ids, base_asset_id, db, excluded_ids).await; } - #[test] - fn exclusion_messages() { + #[tokio::test] + async fn exclusion_messages() { // Setup messages let (owner, base_asset_id, db) = setup_messages(); // Exclude largest messages IDs let excluded_ids = db .owned_messages(&owner) + .await .into_iter() .filter(|message| message.amount() == 5) .map(|message| CoinId::Message(*message.id())) .collect_vec(); - exclusion_assert(owner, &[base_asset_id], base_asset_id, db, excluded_ids); + exclusion_assert(owner, &[base_asset_id], base_asset_id, db, excluded_ids) + .await; } - #[test] - fn exclusion_coins_and_messages() { + #[tokio::test] + async fn exclusion_coins_and_messages() { // Setup coins and messages let (owner, asset_ids, base_asset_id, db) = setup_coins_and_messages(); // Exclude largest messages IDs, because coins only 1 and 2 let excluded_ids = db .owned_messages(&owner) + .await .into_iter() .filter(|message| message.amount() == 5) .map(|message| CoinId::Message(*message.id())) .collect_vec(); - exclusion_assert(owner, &asset_ids, base_asset_id, db, excluded_ids); + exclusion_assert(owner, &asset_ids, base_asset_id, db, excluded_ids).await; } } @@ -822,7 +856,7 @@ mod tests { struct TestCase { db_amount: Vec, target_amount: u64, - max_coins: usize, + max_coins: u16, } pub enum CoinType { @@ -830,7 +864,7 @@ mod tests { Message, } - fn test_case_run( + async fn test_case_run( case: TestCase, coin_type: CoinType, base_asset_id: AssetId, @@ -866,23 +900,26 @@ mod tests { None, base_asset_id, )?, - )?; + ) + .await?; assert_eq!(coins.len(), 1); Ok(coins[0].len()) } - #[test] - fn insufficient_coins_returns_error() { + #[tokio::test] + async fn insufficient_coins_returns_error() { let test_case = TestCase { db_amount: vec![0], target_amount: u64::MAX, - max_coins: usize::MAX, + max_coins: u16::MAX, }; let mut rng = StdRng::seed_from_u64(0xF00DF00D); let base_asset_id = rng.gen(); - let coin_result = test_case_run(test_case.clone(), CoinType::Coin, base_asset_id); - let message_result = test_case_run(test_case, CoinType::Message, base_asset_id); + let coin_result = + test_case_run(test_case.clone(), CoinType::Coin, base_asset_id).await; + let message_result = + test_case_run(test_case, CoinType::Message, base_asset_id).await; assert_eq!(coin_result, message_result); assert_matches!( coin_result, @@ -897,7 +934,7 @@ mod tests { TestCase { db_amount: vec![u64::MAX, u64::MAX], target_amount: u64::MAX, - max_coins: usize::MAX, + max_coins: u16::MAX, } => Ok(1) ; "Enough coins in the DB to reach target(u64::MAX) by 1 coin" @@ -920,11 +957,13 @@ mod tests { => Err(CoinsQueryError::MaxCoinsReached) ; "Enough coins in the DB to reach target(u64::MAX) but limit is zero" )] - fn corner_cases(case: TestCase) -> Result { + #[tokio::test] + async fn corner_cases(case: TestCase) -> Result { let mut rng = StdRng::seed_from_u64(0xF00DF00D); let base_asset_id = rng.gen(); - let coin_result = test_case_run(case.clone(), CoinType::Coin, base_asset_id); - let message_result = test_case_run(case, CoinType::Message, base_asset_id); + let coin_result = + test_case_run(case.clone(), CoinType::Coin, base_asset_id).await; + let message_result = test_case_run(case, CoinType::Message, base_asset_id).await; assert_eq!(coin_result, message_result); coin_result } @@ -1001,23 +1040,25 @@ mod tests { message } - pub fn owned_coins(&self, owner: &Address) -> Vec { + pub async fn owned_coins(&self, owner: &Address) -> Vec { let query = self.service_database(); let query = query.test_view(); query .owned_coins_ids(owner, None, IterDirection::Forward) .map(|res| res.map(|id| query.coin(id).unwrap())) .try_collect() + .await .unwrap() } - pub fn owned_messages(&self, owner: &Address) -> Vec { + pub async fn owned_messages(&self, owner: &Address) -> Vec { let query = self.service_database(); let query = query.test_view(); query .owned_message_ids(owner, None, IterDirection::Forward) .map(|res| res.map(|id| query.message(&id).unwrap())) .try_collect() + .await .unwrap() } } diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index dfaabd42733..3f254032b0c 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -61,6 +61,7 @@ use fuel_core_types::{ txpool::TransactionStatus, }, }; +use futures::Stream; use std::{ borrow::Cow, sync::Arc, @@ -249,8 +250,8 @@ impl ReadView { &self, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.on_chain.all_messages(start_message_id, direction) + ) -> impl Stream> + '_ { + futures::stream::iter(self.on_chain.all_messages(start_message_id, direction)) } pub fn message_exists(&self, nonce: &Nonce) -> StorageResult { @@ -269,9 +270,12 @@ impl ReadView { contract: ContractId, start_asset: Option, direction: IterDirection, - ) -> BoxedIter> { - self.on_chain - .contract_balances(contract, start_asset, direction) + ) -> impl Stream> + '_ { + futures::stream::iter(self.on_chain.contract_balances( + contract, + start_asset, + direction, + )) } pub fn da_height(&self) -> StorageResult { @@ -306,18 +310,23 @@ impl ReadView { owner: &Address, start_coin: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.off_chain.owned_coins_ids(owner, start_coin, direction) + ) -> impl Stream> + '_ { + let iter = self.off_chain.owned_coins_ids(owner, start_coin, direction); + + futures::stream::iter(iter) } - pub fn owned_message_ids( - &self, - owner: &Address, + pub fn owned_message_ids<'a>( + &'a self, + owner: &'a Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.off_chain - .owned_message_ids(owner, start_message_id, direction) + ) -> impl Stream> + 'a { + futures::stream::iter(self.off_chain.owned_message_ids( + owner, + start_message_id, + direction, + )) } pub fn owned_transactions_ids( @@ -325,9 +334,11 @@ impl ReadView { owner: Address, start: Option, direction: IterDirection, - ) -> BoxedIter> { - self.off_chain - .owned_transactions_ids(owner, start, direction) + ) -> impl Stream> + '_ { + futures::stream::iter( + self.off_chain + .owned_transactions_ids(owner, start, direction), + ) } pub fn contract_salt(&self, contract_id: &ContractId) -> StorageResult { diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 34e7d5c8d95..1a715b74522 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -5,11 +5,7 @@ use asset_query::{ AssetsQuery, }; use fuel_core_storage::{ - iter::{ - BoxedIter, - IntoBoxedIter, - IterDirection, - }, + iter::IterDirection, Result as StorageResult, }; use fuel_core_types::{ @@ -19,7 +15,12 @@ use fuel_core_types::{ }, services::graphql_api::AddressBalance, }; -use itertools::Itertools; +use futures::{ + FutureExt, + Stream, + StreamExt, + TryStreamExt, +}; use std::{ cmp::Ordering, collections::HashMap, @@ -28,7 +29,7 @@ use std::{ pub mod asset_query; impl ReadView { - pub fn balance( + pub async fn balance( &self, owner: Address, asset_id: AssetId, @@ -36,21 +37,20 @@ impl ReadView { ) -> StorageResult { let amount = AssetQuery::new( &owner, - &AssetSpendTarget::new(asset_id, u64::MAX, usize::MAX), + &AssetSpendTarget::new(asset_id, u64::MAX, u16::MAX), &base_asset_id, None, self, ) .coins() .map(|res| res.map(|coins| coins.amount())) - .try_fold(0u64, |mut balance, res| -> StorageResult<_> { - let amount = res?; - - // Increase the balance - balance = balance.saturating_add(amount); - - Ok(balance) - })?; + .try_fold(0u64, |balance, amount| { + async move { + // Increase the balance + Ok(balance.saturating_add(amount)) + } + }) + .await?; Ok(AddressBalance { owner, @@ -59,54 +59,52 @@ impl ReadView { }) } - pub fn balances( - &self, - owner: Address, + pub fn balances<'a>( + &'a self, + owner: &'a Address, direction: IterDirection, - base_asset_id: AssetId, - ) -> BoxedIter> { - let mut amounts_per_asset = HashMap::new(); - let mut errors = vec![]; + base_asset_id: &'a AssetId, + ) -> impl Stream> + 'a { + let query = AssetsQuery::new(owner, None, None, self, base_asset_id); + let stream = query.coins(); - for coin in AssetsQuery::new(&owner, None, None, self, &base_asset_id).coins() { - match coin { - Ok(coin) => { + stream + .try_fold( + HashMap::new(), + move |mut amounts_per_asset, coin| async move { let amount: &mut u64 = amounts_per_asset - .entry(*coin.asset_id(&base_asset_id)) + .entry(*coin.asset_id(base_asset_id)) .or_default(); *amount = amount.saturating_add(coin.amount()); - } - Err(err) => { - errors.push(err); - } - } - } - - let mut balances = amounts_per_asset - .into_iter() - .map(|(asset_id, amount)| AddressBalance { - owner, - amount, - asset_id, - }) - .collect_vec(); + Ok(amounts_per_asset) + }, + ) + .into_stream() + .try_filter_map(move |amounts_per_asset| async move { + let mut balances = amounts_per_asset + .into_iter() + .map(|(asset_id, amount)| AddressBalance { + owner: *owner, + amount, + asset_id, + }) + .collect::>(); - balances.sort_by(|l, r| { - if l.asset_id < r.asset_id { - Ordering::Less - } else { - Ordering::Greater - } - }); + balances.sort_by(|l, r| { + if l.asset_id < r.asset_id { + Ordering::Less + } else { + Ordering::Greater + } + }); - if direction == IterDirection::Reverse { - balances.reverse(); - } + if direction == IterDirection::Reverse { + balances.reverse(); + } - balances - .into_iter() - .map(Ok) - .chain(errors.into_iter().map(Err)) - .into_boxed() + Ok(Some(futures::stream::iter(balances))) + }) + .map_ok(|stream| stream.map(Ok)) + .try_flatten() } } diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index e9ecf206aed..bbed21c5568 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -1,4 +1,5 @@ use crate::graphql_api::database::ReadView; +use fuel_core_services::stream::IntoBoxStream; use fuel_core_storage::{ iter::IterDirection, Error as StorageError, @@ -14,19 +15,20 @@ use fuel_core_types::{ AssetId, }, }; -use itertools::Itertools; +use futures::Stream; use std::collections::HashSet; +use tokio_stream::StreamExt; /// At least required `target` of the query per asset's `id` with `max` coins. #[derive(Clone)] pub struct AssetSpendTarget { pub id: AssetId, pub target: u64, - pub max: usize, + pub max: u16, } impl AssetSpendTarget { - pub fn new(id: AssetId, target: u64, max: usize) -> Self { + pub fn new(id: AssetId, target: u64, max: u16) -> Self { Self { id, target, max } } } @@ -48,6 +50,7 @@ impl Exclude { } } +#[derive(Clone)] pub struct AssetsQuery<'a> { pub owner: &'a Address, pub assets: Option>, @@ -73,13 +76,18 @@ impl<'a> AssetsQuery<'a> { } } - fn coins_iter(&self) -> impl Iterator> + '_ { + fn coins_iter(mut self) -> impl Stream> + 'a { + let assets = self.assets.take(); self.database .owned_coins_ids(self.owner, None, IterDirection::Forward) .map(|id| id.map(CoinId::from)) - .filter_ok(|id| { - if let Some(exclude) = self.exclude { - !exclude.coin_ids.contains(id) + .filter(move |result| { + if let Ok(id) = result { + if let Some(exclude) = self.exclude { + !exclude.coin_ids.contains(id) + } else { + true + } } else { true } @@ -91,27 +99,34 @@ impl<'a> AssetsQuery<'a> { } else { return Err(anyhow::anyhow!("The coin is not UTXO").into()); }; + // TODO: Fetch coin in a separate thread let coin = self.database.coin(id)?; Ok(CoinType::Coin(coin)) }) }) - .filter_ok(|coin| { - if let CoinType::Coin(coin) = coin { - self.has_asset(&coin.asset_id) + .filter(move |result| { + if let Ok(CoinType::Coin(coin)) = result { + has_asset(&assets, &coin.asset_id) } else { true } }) } - fn messages_iter(&self) -> impl Iterator> + '_ { + fn messages_iter(&self) -> impl Stream> + 'a { + let exclude = self.exclude; + let database = self.database; self.database .owned_message_ids(self.owner, None, IterDirection::Forward) .map(|id| id.map(CoinId::from)) - .filter_ok(|id| { - if let Some(exclude) = self.exclude { - !exclude.coin_ids.contains(id) + .filter(move |result| { + if let Ok(id) = result { + if let Some(e) = exclude { + !e.coin_ids.contains(id) + } else { + true + } } else { true } @@ -123,11 +138,18 @@ impl<'a> AssetsQuery<'a> { } else { return Err(anyhow::anyhow!("The coin is not a message").into()); }; - let message = self.database.message(&id)?; + // TODO: Fetch message in a separate thread + let message = database.message(&id)?; Ok(message) }) }) - .filter_ok(|message| message.data().is_empty()) + .filter(|result| { + if let Ok(message) = result { + message.data().is_empty() + } else { + true + } + }) .map(|result| { result.map(|message| { CoinType::MessageCoin( @@ -139,28 +161,23 @@ impl<'a> AssetsQuery<'a> { }) } - fn has_asset(&self, asset_id: &AssetId) -> bool { - self.assets - .as_ref() - .map(|assets| assets.contains(asset_id)) - .unwrap_or(true) - } - /// Returns the iterator over all valid(spendable, allowed by `exclude`) coins of the `owner`. /// /// # Note: The coins of different type are not grouped by the `asset_id`. // TODO: Optimize this by creating an index // https://github.com/FuelLabs/fuel-core/issues/588 - pub fn coins(&self) -> impl Iterator> + '_ { - let has_base_asset = self.has_asset(self.base_asset_id); - let messages_iter = has_base_asset - .then(|| self.messages_iter()) - .into_iter() - .flatten(); - self.coins_iter().chain(messages_iter) + pub fn coins(self) -> impl Stream> + 'a { + let has_base_asset = has_asset(&self.assets, self.base_asset_id); + if has_base_asset { + let message_iter = self.messages_iter(); + self.coins_iter().chain(message_iter).into_boxed_ref() + } else { + self.coins_iter().into_boxed_ref() + } } } +#[derive(Clone)] pub struct AssetQuery<'a> { pub owner: &'a Address, pub asset: &'a AssetSpendTarget, @@ -196,7 +213,14 @@ impl<'a> AssetQuery<'a> { /// Returns the iterator over all valid(spendable, allowed by `exclude`) coins of the `owner` /// for the `asset_id`. - pub fn coins(&self) -> impl Iterator> + '_ { + pub fn coins(self) -> impl Stream> + 'a { self.query.coins() } } + +fn has_asset(assets: &Option>, asset_id: &AssetId) -> bool { + assets + .as_ref() + .map(|assets| assets.contains(asset_id)) + .unwrap_or(true) +} diff --git a/crates/fuel-core/src/query/block.rs b/crates/fuel-core/src/query/block.rs index 6ef7cf38afa..3b725ab8b49 100644 --- a/crates/fuel-core/src/query/block.rs +++ b/crates/fuel-core/src/query/block.rs @@ -1,15 +1,13 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ - iter::{ - BoxedIter, - IterDirection, - }, + iter::IterDirection, Result as StorageResult, }; use fuel_core_types::{ blockchain::block::CompressedBlock, fuel_types::BlockHeight, }; +use futures::Stream; impl ReadView { pub fn latest_block_height(&self) -> StorageResult { @@ -24,7 +22,7 @@ impl ReadView { &self, height: Option, direction: IterDirection, - ) -> BoxedIter> { - self.blocks(height, direction) + ) -> impl Stream> + '_ { + futures::stream::iter(self.blocks(height, direction)) } } diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index fc944a623f5..c6d52001ddd 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,10 +1,6 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ - iter::{ - BoxedIter, - IntoBoxedIter, - IterDirection, - }, + iter::IterDirection, not_found, tables::Coins, Result as StorageResult, @@ -15,6 +11,10 @@ use fuel_core_types::{ fuel_tx::UtxoId, fuel_types::Address, }; +use futures::{ + Stream, + StreamExt, +}; impl ReadView { pub fn coin(&self, utxo_id: UtxoId) -> StorageResult { @@ -34,9 +34,13 @@ impl ReadView { owner: &Address, start_coin: Option, direction: IterDirection, - ) -> BoxedIter> { + ) -> impl Stream> + '_ { self.owned_coins_ids(owner, start_coin, direction) - .map(|res| res.and_then(|id| self.coin(id))) - .into_boxed() + .map(|res| { + res.and_then(|id| { + // TODO: Move fetching of the coin to a separate thread + self.coin(id) + }) + }) } } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index aedadd47949..89cd21b8ae7 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -5,7 +5,6 @@ use crate::fuel_core_graphql_api::{ use fuel_core_storage::{ iter::{ BoxedIter, - IntoBoxedIter, IterDirection, }, not_found, @@ -38,6 +37,10 @@ use fuel_core_types::{ }, services::txpool::TransactionStatus, }; +use futures::{ + Stream, + StreamExt, +}; use itertools::Itertools; use std::borrow::Cow; @@ -78,15 +81,19 @@ impl ReadView { .map(Cow::into_owned) } - pub fn owned_messages( - &self, - owner: &Address, + pub fn owned_messages<'a>( + &'a self, + owner: &'a Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter> { + ) -> impl Stream> + 'a { self.owned_message_ids(owner, start_message_id, direction) - .map(|result| result.and_then(|id| self.message(&id))) - .into_boxed() + .map(|result| { + result.and_then(|id| { + // TODO: Move `message` fetching to a separate thread + self.message(&id) + }) + }) } } diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 894d05d9c6f..8989efc9bd2 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -1,11 +1,6 @@ use crate::fuel_core_graphql_api::database::ReadView; - use fuel_core_storage::{ - iter::{ - BoxedIter, - IntoBoxedIter, - IterDirection, - }, + iter::IterDirection, not_found, tables::Transactions, Result as StorageResult, @@ -20,6 +15,10 @@ use fuel_core_types::{ fuel_types::Address, services::txpool::TransactionStatus, }; +use futures::{ + Stream, + StreamExt, +}; impl ReadView { pub fn receipts(&self, tx_id: &TxId) -> StorageResult> { @@ -42,15 +41,15 @@ impl ReadView { owner: Address, start: Option, direction: IterDirection, - ) -> BoxedIter> { + ) -> impl Stream> + '_ { self.owned_transactions_ids(owner, start, direction) .map(|result| { result.and_then(|(tx_pointer, tx_id)| { + // TODO: Fetch transactions in a separate thread let tx = self.transaction(&tx_id)?; Ok((tx_pointer, tx)) }) }) - .into_boxed() } } diff --git a/crates/fuel-core/src/schema.rs b/crates/fuel-core/src/schema.rs index bd9e550d448..bcbc5b5c970 100644 --- a/crates/fuel-core/src/schema.rs +++ b/crates/fuel-core/src/schema.rs @@ -23,8 +23,12 @@ use fuel_core_storage::{ iter::IterDirection, Result as StorageResult, }; -use itertools::Itertools; +use futures::{ + Stream, + TryStreamExt, +}; use std::borrow::Cow; +use tokio_stream::StreamExt; pub mod balance; pub mod blob; @@ -99,7 +103,7 @@ where // It means also returning `has_previous_page` and `has_next_page` values. // entries(start_key: Option) F: FnOnce(&Option, IterDirection) -> StorageResult, - Entries: Iterator>, + Entries: Stream>, SchemaKey: Eq, { match (after.as_ref(), before.as_ref(), first, last) { @@ -192,7 +196,7 @@ where } }); - let entries: Vec<_> = entries.try_collect()?; + let entries: Vec<_> = entries.try_collect().await?; let entries = entries.into_iter(); let mut connection = Connection::new(has_previous_page, has_next_page); diff --git a/crates/fuel-core/src/schema/balance.rs b/crates/fuel-core/src/schema/balance.rs index 7c43e3a9509..4e68ddbe894 100644 --- a/crates/fuel-core/src/schema/balance.rs +++ b/crates/fuel-core/src/schema/balance.rs @@ -23,6 +23,7 @@ use async_graphql::{ Object, }; use fuel_core_types::services::graphql_api; +use futures::StreamExt; pub struct Balance(graphql_api::AddressBalance); @@ -64,7 +65,10 @@ impl BalanceQuery { .data_unchecked::() .latest_consensus_params() .base_asset_id(); - let balance = query.balance(owner.0, asset_id.0, base_asset_id)?.into(); + let balance = query + .balance(owner.0, asset_id.0, base_asset_id) + .await? + .into(); Ok(balance) } @@ -85,14 +89,14 @@ impl BalanceQuery { return Err(anyhow!("pagination is not yet supported").into()) } let query = ctx.read_view()?; + let base_asset_id = *ctx + .data_unchecked::() + .latest_consensus_params() + .base_asset_id(); + let owner = filter.owner.into(); crate::schema::query_pagination(after, before, first, last, |_, direction| { - let owner = filter.owner.into(); - let base_asset_id = *ctx - .data_unchecked::() - .latest_consensus_params() - .base_asset_id(); Ok(query - .balances(owner, direction, base_asset_id) + .balances(&owner, direction, &base_asset_id) .map(|result| { result.map(|balance| (balance.asset_id.into(), balance.into())) })) diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index cebd04989b6..d7bc8366282 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -36,11 +36,7 @@ use async_graphql::{ Union, }; use fuel_core_storage::{ - iter::{ - BoxedIter, - IntoBoxedIter, - IterDirection, - }, + iter::IterDirection, Result as StorageResult, }; use fuel_core_types::{ @@ -51,6 +47,10 @@ use fuel_core_types::{ fuel_types, fuel_types::BlockHeight, }; +use futures::{ + Stream, + StreamExt, +}; pub struct Block(pub(crate) CompressedBlock); @@ -339,16 +339,14 @@ fn blocks_query( query: &ReadView, height: Option, direction: IterDirection, -) -> BoxedIter> +) -> impl Stream> + '_ where T: async_graphql::OutputType, T: From, { - let blocks = query.compressed_blocks(height, direction).map(|result| { + query.compressed_blocks(height, direction).map(|result| { result.map(|block| ((*block.header().height()).into(), block.into())) - }); - - blocks.into_boxed() + }) } #[derive(Default)] diff --git a/crates/fuel-core/src/schema/coins.rs b/crates/fuel-core/src/schema/coins.rs index 3610c152e6b..bed54afe676 100644 --- a/crates/fuel-core/src/schema/coins.rs +++ b/crates/fuel-core/src/schema/coins.rs @@ -40,6 +40,7 @@ use fuel_core_types::{ fuel_tx, }; use itertools::Itertools; +use tokio_stream::StreamExt; pub struct Coin(pub(crate) CoinModel); @@ -174,8 +175,8 @@ impl CoinQuery { before: Option, ) -> async_graphql::Result> { let query = ctx.read_view()?; + let owner: fuel_tx::Address = filter.owner.into(); crate::schema::query_pagination(after, before, first, last, |start, direction| { - let owner: fuel_tx::Address = filter.owner.into(); let coins = query .owned_coins(&owner, (*start).map(Into::into), direction) .filter_map(|result| { @@ -222,6 +223,14 @@ impl CoinQuery { let params = ctx .data_unchecked::() .latest_consensus_params(); + let max_input = params.tx_params().max_inputs(); + + if query_per_asset.len() > max_input as usize { + return Err(anyhow::anyhow!( + "The number of assets to spend is greater than the maximum number of inputs." + ) + .into()); + } let owner: fuel_tx::Address = owner.0; let query_per_asset = query_per_asset @@ -230,7 +239,10 @@ impl CoinQuery { AssetSpendTarget::new( e.asset_id.0, e.amount.0, - e.max.map(|max| max.0 as usize).unwrap_or(usize::MAX), + e.max + .and_then(|max| u16::try_from(max.0).ok()) + .unwrap_or(max_input) + .min(max_input), ) }) .collect_vec(); @@ -252,7 +264,8 @@ impl CoinQuery { let query = ctx.read_view()?; - let coins = random_improve(query.as_ref(), &spend_query)? + let coins = random_improve(query.as_ref(), &spend_query) + .await? .into_iter() .map(|coins| { coins diff --git a/crates/fuel-core/src/schema/contract.rs b/crates/fuel-core/src/schema/contract.rs index a8fe6dea693..ac4f8b67a6b 100644 --- a/crates/fuel-core/src/schema/contract.rs +++ b/crates/fuel-core/src/schema/contract.rs @@ -31,6 +31,7 @@ use fuel_core_types::{ fuel_types, services::graphql_api, }; +use futures::StreamExt; pub struct Contract(pub(crate) fuel_types::ContractId); @@ -168,7 +169,7 @@ impl ContractBalanceQuery { (*start).map(Into::into), direction, ) - .map(move |balance| { + .map(|balance| { let balance = balance?; let asset_id = balance.asset_id; diff --git a/crates/fuel-core/src/schema/message.rs b/crates/fuel-core/src/schema/message.rs index 05416623934..36f83cd11c9 100644 --- a/crates/fuel-core/src/schema/message.rs +++ b/crates/fuel-core/src/schema/message.rs @@ -28,7 +28,9 @@ use async_graphql::{ Enum, Object, }; +use fuel_core_services::stream::IntoBoxStream; use fuel_core_types::entities; +use futures::StreamExt; pub struct Message(pub(crate) entities::relayer::message::Message); @@ -91,6 +93,8 @@ impl MessageQuery { ) -> async_graphql::Result> { let query = ctx.read_view()?; + let owner = owner.map(|owner| owner.0); + let owner_ref = owner.as_ref(); crate::schema::query_pagination( after, before, @@ -103,10 +107,12 @@ impl MessageQuery { None }; - let messages = if let Some(owner) = owner { - query.owned_messages(&owner.0, start, direction) + let messages = if let Some(owner) = owner_ref { + query + .owned_messages(owner, start, direction) + .into_boxed_ref() } else { - query.all_messages(start, direction) + query.all_messages(start, direction).into_boxed_ref() }; let messages = messages.map(|result| { diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index 89b20101db0..877aaa9df91 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -67,7 +67,6 @@ use futures::{ Stream, TryStreamExt, }; -use itertools::Itertools; use std::{ borrow::Cow, iter, @@ -133,41 +132,40 @@ impl TxQuery { |start: &Option, direction| { let start = *start; let block_id = start.map(|sorted| sorted.block_height); - let all_block_ids = query.compressed_blocks(block_id, direction); + let compressed_blocks = query.compressed_blocks(block_id, direction); - let all_txs = all_block_ids - .map(move |block| { - block.map(|fuel_block| { - let (header, mut txs) = fuel_block.into_inner(); + let all_txs = compressed_blocks + .map_ok(move |fuel_block| { + let (header, mut txs) = fuel_block.into_inner(); - if direction == IterDirection::Reverse { - txs.reverse(); - } + if direction == IterDirection::Reverse { + txs.reverse(); + } - txs.into_iter().zip(iter::repeat(*header.height())) - }) + let iter = txs.into_iter().zip(iter::repeat(*header.height())); + futures::stream::iter(iter).map(Ok) }) - .flatten_ok() - .map(|result| { - result.map(|(tx_id, block_height)| { - SortedTxCursor::new(block_height, tx_id.into()) - }) + .try_flatten() + .map_ok(|(tx_id, block_height)| { + SortedTxCursor::new(block_height, tx_id.into()) }) - .skip_while(move |result| { - if let Ok(sorted) = result { - if let Some(start) = start { - return sorted != &start - } - } - false - }); - let all_txs = all_txs.map(|result: StorageResult| { - result.and_then(|sorted| { - let tx = query.transaction(&sorted.tx_id.0)?; - - Ok((sorted, Transaction::from_tx(sorted.tx_id.0, tx))) + .try_skip_while(move |sorted| { + let skip = if let Some(start) = start { + sorted != &start + } else { + false + }; + + async move { Ok(skip) } }) - }); + .map(|result: StorageResult| { + result.and_then(|sorted| { + // TODO: Request transactions in a separate thread + let tx = query.transaction(&sorted.tx_id.0)?; + + Ok((sorted, Transaction::from_tx(sorted.tx_id.0, tx))) + }) + }); Ok(all_txs) }, diff --git a/crates/fuel-core/src/state/rocks_db_key_iterator.rs b/crates/fuel-core/src/state/rocks_db_key_iterator.rs index b77e2cb7179..432ab6b41ae 100644 --- a/crates/fuel-core/src/state/rocks_db_key_iterator.rs +++ b/crates/fuel-core/src/state/rocks_db_key_iterator.rs @@ -18,9 +18,9 @@ pub struct RocksDBKeyIterator<'a, D: DBAccess, R> { _marker: core::marker::PhantomData, } -pub trait ExtractItem: 'static { +pub trait ExtractItem: Send + Sync + 'static { /// The item type returned by the iterator. - type Item; + type Item: Send + Sync; /// Extracts the item from the raw iterator. fn extract_item( diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 7162389e082..10da0bc75f7 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -21,26 +21,37 @@ pub mod stream { Stream, }; - /// A Send + Sync BoxStream + /// A `Send` + `Sync` BoxStream with static lifetime. pub type BoxStream = core::pin::Pin + Send + Sync + 'static>>; + /// A `Send` BoxStream with a lifetime. + pub type RefBoxStream<'a, T> = core::pin::Pin + Send + 'a>>; + /// A Send + Sync BoxFuture pub type BoxFuture<'a, T> = core::pin::Pin + Send + Sync + 'a>>; /// Helper trait to create a BoxStream from a Stream pub trait IntoBoxStream: Stream { - /// Convert this stream into a BoxStream. + /// Convert this stream into a [`BoxStream`]. fn into_boxed(self) -> BoxStream where Self: Sized + Send + Sync + 'static, { Box::pin(self) } + + /// Convert this stream into a [`RefBoxStream`]. + fn into_boxed_ref<'a>(self) -> RefBoxStream<'a, Self::Item> + where + Self: Sized + Send + 'a, + { + Box::pin(self) + } } - impl IntoBoxStream for S where S: Stream + Send + Sync + 'static {} + impl IntoBoxStream for S where S: Stream + Send {} } /// Helper trait to trace errors diff --git a/crates/storage/src/iter.rs b/crates/storage/src/iter.rs index 37c38463cc8..35c550851fb 100644 --- a/crates/storage/src/iter.rs +++ b/crates/storage/src/iter.rs @@ -29,7 +29,7 @@ pub mod changes_iterator; // TODO: BoxedIter to be used until RPITIT lands in stable rust. /// A boxed variant of the iterator that can be used as a return type of the traits. pub struct BoxedIter<'a, T> { - iter: Box + 'a>, + iter: Box + 'a + Send>, } impl<'a, T> Iterator for BoxedIter<'a, T> { @@ -48,7 +48,7 @@ pub trait IntoBoxedIter<'a, T> { impl<'a, T, I> IntoBoxedIter<'a, T> for I where - I: Iterator + 'a, + I: Iterator + 'a + Send, { fn into_boxed(self) -> BoxedIter<'a, T> { BoxedIter { @@ -346,7 +346,10 @@ pub fn iterator<'a, V>( prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, -) -> impl Iterator + 'a { +) -> impl Iterator + 'a +where + V: Send + Sync, +{ match (prefix, start) { (None, None) => { if direction == IterDirection::Forward { @@ -401,7 +404,10 @@ pub fn keys_iterator<'a, V>( prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, -) -> impl Iterator + 'a { +) -> impl Iterator + 'a +where + V: Send + Sync, +{ match (prefix, start) { (None, None) => { if direction == IterDirection::Forward { diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index e9d4eb22a52..67ab9493c7f 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -17,13 +17,8 @@ use core::ops::Deref; /// The key of the storage. pub type Key = Vec; -#[cfg(feature = "std")] /// The value of the storage. It is wrapped into the `Arc` to provide less cloning of massive objects. -pub type Value = std::sync::Arc>; - -#[cfg(not(feature = "std"))] -/// The value of the storage. It is wrapped into the `Rc` to provide less cloning of massive objects. -pub type Value = alloc::rc::Rc>; +pub type Value = alloc::sync::Arc>; /// The pair of key and value from the storage. pub type KVItem = StorageResult<(Key, Value)>; From 5192c951ef868b222f62527789a47ad7528947dd Mon Sep 17 00:00:00 2001 From: green Date: Sat, 12 Oct 2024 17:40:50 +0200 Subject: [PATCH 04/19] Making change non-breaking for now --- crates/fuel-core/src/schema/coins.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/fuel-core/src/schema/coins.rs b/crates/fuel-core/src/schema/coins.rs index bed54afe676..a1066762380 100644 --- a/crates/fuel-core/src/schema/coins.rs +++ b/crates/fuel-core/src/schema/coins.rs @@ -214,8 +214,8 @@ impl CoinQuery { #[graphql(desc = "\ The list of requested assets` coins with asset ids, `target` amount the user wants \ to reach, and the `max` number of coins in the selection. Several entries with the \ - same asset id are not allowed.")] - query_per_asset: Vec, + same asset id are not allowed. The result can't contain more coins than `max_inputs`.")] + mut query_per_asset: Vec, #[graphql(desc = "The excluded coins from the selection.")] excluded_ids: Option< ExcludeInput, >, @@ -225,12 +225,12 @@ impl CoinQuery { .latest_consensus_params(); let max_input = params.tx_params().max_inputs(); - if query_per_asset.len() > max_input as usize { - return Err(anyhow::anyhow!( - "The number of assets to spend is greater than the maximum number of inputs." - ) - .into()); - } + // `coins_to_spend` exists to help select inputs for the transactions. + // It doesn't make sense to allow the user to request more than the maximum number + // of inputs. + // TODO: To avoid breaking changes, we will truncate request for now. + // In the future, we should return an error if the input is too large. + query_per_asset.truncate(max_input as usize); let owner: fuel_tx::Address = owner.0; let query_per_asset = query_per_asset From feeb81680f796769335e1ba01b1a6c1001a956a7 Mon Sep 17 00:00:00 2001 From: green Date: Sat, 12 Oct 2024 17:45:19 +0200 Subject: [PATCH 05/19] Updated CHANGELOG.md --- CHANGELOG.md | 4 ++++ crates/client/assets/schema.sdl | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b21a83e99..09cd4c1b624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed - [2334](https://github.com/FuelLabs/fuel-core/pull/2334): Prepare the GraphQL service for the switching to `async` methods. +- [2337](https://github.com/FuelLabs/fuel-core/pull/2337): Updated all pagination queries to work with the async stream instead of the sync iterator. + +#### Breaking +- [2337](https://github.com/FuelLabs/fuel-core/pull/2337): The maximum number of processed coins from the `coins_to_spend` query is limited to `max_inputs`. ## [Version 0.39.0] diff --git a/crates/client/assets/schema.sdl b/crates/client/assets/schema.sdl index 67287341c38..b9048362caa 100644 --- a/crates/client/assets/schema.sdl +++ b/crates/client/assets/schema.sdl @@ -938,7 +938,7 @@ type Query { """ owner: Address!, """ - The list of requested assets` coins with asset ids, `target` amount the user wants to reach, and the `max` number of coins in the selection. Several entries with the same asset id are not allowed. + The list of requested assets` coins with asset ids, `target` amount the user wants to reach, and the `max` number of coins in the selection. Several entries with the same asset id are not allowed. The result can't contain more coins than `max_inputs`. """ queryPerAsset: [SpendQueryElementInput!]!, """ From 4237feb7824997ab35377ca75f4731815f50c875 Mon Sep 17 00:00:00 2001 From: green Date: Sun, 13 Oct 2024 18:24:54 +0200 Subject: [PATCH 06/19] Avoid long heavy tasks in the GraphQL service --- benches/benches/transaction_throughput.rs | 1 + bin/fuel-core/src/cli/run.rs | 1 + bin/fuel-core/src/cli/run/graphql.rs | 4 ++ crates/fuel-core/src/coins_query.rs | 13 ++-- crates/fuel-core/src/database/block.rs | 2 +- crates/fuel-core/src/graphql_api.rs | 1 + .../fuel-core/src/graphql_api/api_service.rs | 8 ++- crates/fuel-core/src/graphql_api/database.rs | 60 +++++++++++-------- crates/fuel-core/src/query/balance.rs | 7 +++ .../src/query/balance/asset_query.rs | 46 +++++++++++--- crates/fuel-core/src/query/block.rs | 12 +++- crates/fuel-core/src/query/coin.rs | 29 +++++++-- crates/fuel-core/src/query/message.rs | 35 ++++++----- crates/fuel-core/src/query/message/test.rs | 13 ---- crates/fuel-core/src/query/tx.rs | 29 +++++---- crates/fuel-core/src/schema/block.rs | 29 ++++++--- crates/fuel-core/src/schema/tx.rs | 44 +++++++++++--- crates/fuel-core/src/schema/tx/types.rs | 2 +- crates/fuel-core/src/service/config.rs | 1 + tests/test-helpers/src/builder.rs | 9 +++ tests/tests/dos.rs | 42 ++++++++++++- 21 files changed, 281 insertions(+), 107 deletions(-) diff --git a/benches/benches/transaction_throughput.rs b/benches/benches/transaction_throughput.rs index 23454dda359..5d78818e8da 100644 --- a/benches/benches/transaction_throughput.rs +++ b/benches/benches/transaction_throughput.rs @@ -89,6 +89,7 @@ where test_builder.trigger = Trigger::Never; test_builder.utxo_validation = true; test_builder.gas_limit = Some(10_000_000_000); + test_builder.block_size_limit = Some(1_000_000_000_000); // spin up node let transactions: Vec = diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index ab3523dec1a..1a76fb18a68 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -488,6 +488,7 @@ impl Command { let config = Config { graphql_config: GraphQLConfig { addr, + database_butch_size: graphql.database_butch_size, max_queries_depth: graphql.graphql_max_depth, max_queries_complexity: graphql.graphql_max_complexity, max_queries_recursive_depth: graphql.graphql_max_recursive_depth, diff --git a/bin/fuel-core/src/cli/run/graphql.rs b/bin/fuel-core/src/cli/run/graphql.rs index 95cfb2852f5..73c682d0643 100644 --- a/bin/fuel-core/src/cli/run/graphql.rs +++ b/bin/fuel-core/src/cli/run/graphql.rs @@ -12,6 +12,10 @@ pub struct GraphQLArgs { #[clap(long = "port", default_value = "4000", env)] pub port: u16, + /// The size of the butch fetched from the database by GraphQL service. + #[clap(long = "graphql-database-butch-size", default_value = "100", env)] + pub database_butch_size: usize, + /// The max depth of GraphQL queries. #[clap(long = "graphql-max-depth", default_value = "16", env)] pub graphql_max_depth: usize, diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index e07630333f8..ea7c2b34c0d 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -279,10 +279,7 @@ mod tests { fuel_asm::Word, fuel_tx::*, }; - use futures::{ - StreamExt, - TryStreamExt, - }; + use futures::TryStreamExt; use itertools::Itertools; use rand::{ rngs::StdRng, @@ -987,7 +984,7 @@ mod tests { fn service_database(&self) -> ServiceDatabase { let on_chain = self.database.on_chain().clone(); let off_chain = self.database.off_chain().clone(); - ServiceDatabase::new(0u32.into(), on_chain, off_chain) + ServiceDatabase::new(100, 0u32.into(), on_chain, off_chain) } } @@ -1044,8 +1041,7 @@ mod tests { let query = self.service_database(); let query = query.test_view(); query - .owned_coins_ids(owner, None, IterDirection::Forward) - .map(|res| res.map(|id| query.coin(id).unwrap())) + .owned_coins(owner, None, IterDirection::Forward) .try_collect() .await .unwrap() @@ -1055,8 +1051,7 @@ mod tests { let query = self.service_database(); let query = query.test_view(); query - .owned_message_ids(owner, None, IterDirection::Forward) - .map(|res| res.map(|id| query.message(&id).unwrap())) + .owned_messages(owner, None, IterDirection::Forward) .try_collect() .await .unwrap() diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index c6295e62017..9202bd04043 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -67,7 +67,7 @@ impl OnChainIterableKeyValueView { let db_block = self.storage::().get(height)?; if let Some(block) = db_block { // fetch all the transactions - // TODO: optimize with multi-key get + // TODO: Use multiget when it's implemented. let txs = block .transactions() .iter() diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 6d0d280fd3f..8b8706b75e5 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -20,6 +20,7 @@ pub mod worker_service; #[derive(Clone, Debug)] pub struct ServiceConfig { pub addr: SocketAddr, + pub database_butch_size: usize, pub max_queries_depth: usize, pub max_queries_complexity: usize, pub max_queries_recursive_depth: usize, diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index ec79918e09c..ce0fdbc9f6c 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -221,8 +221,12 @@ where OffChain::LatestView: OffChainDatabase, { let network_addr = config.config.addr; - let combined_read_database = - ReadDatabase::new(genesis_block_height, on_database, off_database); + let combined_read_database = ReadDatabase::new( + config.config.database_butch_size, + genesis_block_height, + on_database, + off_database, + ); let request_timeout = config.config.api_request_timeout; let concurrency_limit = config.config.max_concurrent_queries; let body_limit = config.config.request_body_bytes_limit; diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 3f254032b0c..bd08761870c 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -5,6 +5,7 @@ use crate::fuel_core_graphql_api::{ OnChainDatabase, }, }; +use async_graphql::futures_util::StreamExt; use fuel_core_storage::{ iter::{ BoxedIter, @@ -77,6 +78,8 @@ pub type OffChainView = Arc; /// The container of the on-chain and off-chain database view provides. /// It is used only by `ViewExtension` to create a [`ReadView`]. pub struct ReadDatabase { + /// The size of the butch during fetching from the database. + butch_size: usize, /// The height of the genesis block. genesis_height: BlockHeight, /// The on-chain database view provider. @@ -88,6 +91,7 @@ pub struct ReadDatabase { impl ReadDatabase { /// Creates a new [`ReadDatabase`] with the given on-chain and off-chain database view providers. pub fn new( + butch_size: usize, genesis_height: BlockHeight, on_chain: OnChain, off_chain: OffChain, @@ -99,6 +103,7 @@ impl ReadDatabase { OffChain::LatestView: OffChainDatabase, { Self { + butch_size, genesis_height, on_chain: Box::new(ArcWrapper::new(on_chain)), off_chain: Box::new(ArcWrapper::new(off_chain)), @@ -111,6 +116,7 @@ impl ReadDatabase { // It is not possible to implement until `view_at` is implemented for the `AtomicView`. // https://github.com/FuelLabs/fuel-core/issues/1582 Ok(ReadView { + butch_size: self.butch_size, genesis_height: self.genesis_height, on_chain: self.on_chain.latest_view()?, off_chain: self.off_chain.latest_view()?, @@ -125,6 +131,7 @@ impl ReadDatabase { #[derive(Clone)] pub struct ReadView { + pub(crate) butch_size: usize, pub(crate) genesis_height: BlockHeight, pub(crate) on_chain: OnChainView, pub(crate) off_chain: OffChainView, @@ -134,7 +141,7 @@ impl ReadView { pub fn transaction(&self, tx_id: &TxId) -> StorageResult { let result = self.on_chain.transaction(tx_id); if result.is_not_found() { - if let Some(tx) = self.old_transaction(tx_id)? { + if let Some(tx) = self.off_chain.old_transaction(tx_id)? { Ok(tx) } else { Err(not_found!(Transactions)) @@ -144,6 +151,20 @@ impl ReadView { } } + pub async fn transactions( + &self, + tx_ids: Vec, + ) -> Vec> { + // TODO: Use multiget when it's implemented. + let result = tx_ids + .iter() + .map(|tx_id| self.transaction(tx_id)) + .collect::>(); + // Give a chance to other tasks to run. + tokio::task::yield_now().await; + result + } + pub fn block(&self, height: &BlockHeight) -> StorageResult { if *height >= self.genesis_height { self.on_chain.block(height) @@ -252,6 +273,13 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { futures::stream::iter(self.on_chain.all_messages(start_message_id, direction)) + .chunks(self.butch_size) + .filter_map(|chunk| async move { + // Give a chance to other tasks to run. + tokio::task::yield_now().await; + Some(futures::stream::iter(chunk)) + }) + .flatten() } pub fn message_exists(&self, nonce: &Nonce) -> StorageResult { @@ -276,6 +304,13 @@ impl ReadView { start_asset, direction, )) + .chunks(self.butch_size) + .filter_map(|chunk| async move { + // Give a chance to other tasks to run. + tokio::task::yield_now().await; + Some(futures::stream::iter(chunk)) + }) + .flatten() } pub fn da_height(&self) -> StorageResult { @@ -345,29 +380,6 @@ impl ReadView { self.off_chain.contract_salt(contract_id) } - pub fn old_block(&self, height: &BlockHeight) -> StorageResult { - self.off_chain.old_block(height) - } - - pub fn old_blocks( - &self, - height: Option, - direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { - self.off_chain.old_blocks(height, direction) - } - - pub fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult { - self.off_chain.old_block_consensus(height) - } - - pub fn old_transaction( - &self, - id: &TxId, - ) -> StorageResult> { - self.off_chain.old_transaction(id) - } - pub fn relayed_tx_status( &self, id: Bytes32, diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 1a715b74522..0638dc768b9 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -106,5 +106,12 @@ impl ReadView { }) .map_ok(|stream| stream.map(Ok)) .try_flatten() + .chunks(self.butch_size) + .filter_map(|chunk| async move { + // Give a chance to other tasks to run. + tokio::task::yield_now().await; + Some(futures::stream::iter(chunk)) + }) + .flatten() } } diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index bbed21c5568..3c9ba49cee6 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -1,4 +1,5 @@ use crate::graphql_api::database::ReadView; +use async_graphql::futures_util::TryStreamExt; use fuel_core_services::stream::IntoBoxStream; use fuel_core_storage::{ iter::IterDirection, @@ -78,7 +79,9 @@ impl<'a> AssetsQuery<'a> { fn coins_iter(mut self) -> impl Stream> + 'a { let assets = self.assets.take(); - self.database + let database = self.database; + let stream = self + .database .owned_coins_ids(self.owner, None, IterDirection::Forward) .map(|id| id.map(CoinId::from)) .filter(move |result| { @@ -99,12 +102,25 @@ impl<'a> AssetsQuery<'a> { } else { return Err(anyhow::anyhow!("The coin is not UTXO").into()); }; - // TODO: Fetch coin in a separate thread - let coin = self.database.coin(id)?; - - Ok(CoinType::Coin(coin)) + Ok(id) }) + }); + + futures::stream::StreamExt::chunks(stream, database.butch_size) + .map(|chunk| { + use itertools::Itertools; + + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok::<_, StorageError>(chunk) }) + .try_filter_map(move |chunk| async move { + let chunk = database + .coins(chunk) + .await + .map(|result| result.map(CoinType::Coin)); + Ok(Some(futures::stream::iter(chunk))) + }) + .try_flatten() .filter(move |result| { if let Ok(CoinType::Coin(coin)) = result { has_asset(&assets, &coin.asset_id) @@ -117,7 +133,8 @@ impl<'a> AssetsQuery<'a> { fn messages_iter(&self) -> impl Stream> + 'a { let exclude = self.exclude; let database = self.database; - self.database + let stream = self + .database .owned_message_ids(self.owner, None, IterDirection::Forward) .map(|id| id.map(CoinId::from)) .filter(move |result| { @@ -138,11 +155,22 @@ impl<'a> AssetsQuery<'a> { } else { return Err(anyhow::anyhow!("The coin is not a message").into()); }; - // TODO: Fetch message in a separate thread - let message = database.message(&id)?; - Ok(message) + Ok(id) }) + }); + + futures::stream::StreamExt::chunks(stream, database.butch_size) + .map(|chunk| { + use itertools::Itertools; + + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok(chunk) + }) + .try_filter_map(move |chunk| async move { + let chunk = database.messages(chunk).await; + Ok::<_, StorageError>(Some(futures::stream::iter(chunk))) }) + .try_flatten() .filter(|result| { if let Ok(message) = result { message.data().is_empty() diff --git a/crates/fuel-core/src/query/block.rs b/crates/fuel-core/src/query/block.rs index 3b725ab8b49..60f98838879 100644 --- a/crates/fuel-core/src/query/block.rs +++ b/crates/fuel-core/src/query/block.rs @@ -7,7 +7,10 @@ use fuel_core_types::{ blockchain::block::CompressedBlock, fuel_types::BlockHeight, }; -use futures::Stream; +use futures::{ + Stream, + StreamExt, +}; impl ReadView { pub fn latest_block_height(&self) -> StorageResult { @@ -24,5 +27,12 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { futures::stream::iter(self.blocks(height, direction)) + .chunks(self.butch_size) + .filter_map(|chunk| async move { + // Give a chance to other tasks to run. + tokio::task::yield_now().await; + Some(futures::stream::iter(chunk)) + }) + .flatten() } } diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index c6d52001ddd..7de4a5211d6 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,8 +1,10 @@ use crate::fuel_core_graphql_api::database::ReadView; +use async_graphql::futures_util::TryStreamExt; use fuel_core_storage::{ iter::IterDirection, not_found, tables::Coins, + Error as StorageError, Result as StorageResult, StorageAsRef, }; @@ -29,6 +31,17 @@ impl ReadView { Ok(coin.uncompress(utxo_id)) } + pub async fn coins( + &self, + utxo_ids: Vec, + ) -> impl Iterator> + '_ { + // TODO: Use multiget when it's implemented. + let coins = utxo_ids.into_iter().map(|id| self.coin(id)); + // Yield to the runtime to allow other tasks to run. + tokio::task::yield_now().await; + coins + } + pub fn owned_coins( &self, owner: &Address, @@ -36,11 +49,17 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { self.owned_coins_ids(owner, start_coin, direction) - .map(|res| { - res.and_then(|id| { - // TODO: Move fetching of the coin to a separate thread - self.coin(id) - }) + .chunks(self.butch_size) + .map(|chunk| { + use itertools::Itertools; + + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok::<_, StorageError>(chunk) + }) + .try_filter_map(move |chunk| async move { + let chunk = self.coins(chunk).await; + Ok(Some(futures::stream::iter(chunk))) }) + .try_flatten() } } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 89cd21b8ae7..51550af3555 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -25,7 +25,6 @@ use fuel_core_types::{ fuel_tx::{ input::message::compute_message_id, Receipt, - Transaction, TxId, }, fuel_types::{ @@ -40,6 +39,7 @@ use fuel_core_types::{ use futures::{ Stream, StreamExt, + TryStreamExt, }; use itertools::Itertools; use std::borrow::Cow; @@ -81,6 +81,16 @@ impl ReadView { .map(Cow::into_owned) } + pub async fn messages( + &self, + ids: Vec, + ) -> impl Iterator> + '_ { + let messages = ids.into_iter().map(|id| self.message(&id)); + // Yield to the runtime to allow other tasks to run. + tokio::task::yield_now().await; + messages + } + pub fn owned_messages<'a>( &'a self, owner: &'a Address, @@ -88,12 +98,16 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + 'a { self.owned_message_ids(owner, start_message_id, direction) - .map(|result| { - result.and_then(|id| { - // TODO: Move `message` fetching to a separate thread - self.message(&id) - }) + .chunks(self.butch_size) + .map(|chunk| { + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok(chunk) + }) + .try_filter_map(move |chunk| async move { + let chunk = self.messages(chunk).await; + Ok::<_, StorageError>(Some(futures::stream::iter(chunk))) }) + .try_flatten() } } @@ -102,9 +116,6 @@ pub trait MessageProofData { /// Get the block. fn block(&self, id: &BlockHeight) -> StorageResult; - /// Get the transaction. - fn transaction(&self, transaction_id: &TxId) -> StorageResult; - /// Return all receipts in the given transaction. fn receipts(&self, transaction_id: &TxId) -> StorageResult>; @@ -128,10 +139,6 @@ impl MessageProofData for ReadView { self.block(id) } - fn transaction(&self, transaction_id: &TxId) -> StorageResult { - self.transaction(transaction_id) - } - fn receipts(&self, transaction_id: &TxId) -> StorageResult> { self.receipts(transaction_id) } @@ -140,7 +147,7 @@ impl MessageProofData for ReadView { &self, transaction_id: &TxId, ) -> StorageResult { - self.status(transaction_id) + self.tx_status(transaction_id) } fn block_history_proof( diff --git a/crates/fuel-core/src/query/message/test.rs b/crates/fuel-core/src/query/message/test.rs index d49f173370b..43d6ecbef1a 100644 --- a/crates/fuel-core/src/query/message/test.rs +++ b/crates/fuel-core/src/query/message/test.rs @@ -10,8 +10,6 @@ use fuel_core_types::{ fuel_tx::{ AssetId, ContractId, - Script, - Transaction, }, fuel_types::BlockHeight, tai64::Tai64, @@ -64,7 +62,6 @@ mockall::mock! { message_block_height: &BlockHeight, commit_block_height: &BlockHeight, ) -> StorageResult; - fn transaction(&self, transaction_id: &TxId) -> StorageResult; fn receipts(&self, transaction_id: &TxId) -> StorageResult>; fn transaction_status(&self, transaction_id: &TxId) -> StorageResult; } @@ -107,16 +104,6 @@ async fn can_build_message_proof() { } }); - data.expect_transaction().returning(move |txn_id| { - let tx = TXNS - .iter() - .find(|t| *t == txn_id) - .map(|_| Script::default().into()) - .ok_or(not_found!("Transaction in `TXNS`"))?; - - Ok(tx) - }); - let commit_block_header = PartialBlockHeader { application: ApplicationHeader { da_height: 0u64.into(), diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 8989efc9bd2..cfeaf00f8d1 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -3,6 +3,7 @@ use fuel_core_storage::{ iter::IterDirection, not_found, tables::Transactions, + Error as StorageError, Result as StorageResult, }; use fuel_core_types::{ @@ -18,11 +19,12 @@ use fuel_core_types::{ use futures::{ Stream, StreamExt, + TryStreamExt, }; impl ReadView { pub fn receipts(&self, tx_id: &TxId) -> StorageResult> { - let status = self.status(tx_id)?; + let status = self.tx_status(tx_id)?; let receipts = match status { TransactionStatus::Success { receipts, .. } @@ -32,10 +34,6 @@ impl ReadView { receipts.ok_or(not_found!(Transactions)) } - pub fn status(&self, tx_id: &TxId) -> StorageResult { - self.tx_status(tx_id) - } - pub fn owned_transactions( &self, owner: Address, @@ -43,13 +41,22 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { self.owned_transactions_ids(owner, start, direction) - .map(|result| { - result.and_then(|(tx_pointer, tx_id)| { - // TODO: Fetch transactions in a separate thread - let tx = self.transaction(&tx_id)?; + .chunks(self.butch_size) + .map(|chunk| { + use itertools::Itertools; - Ok((tx_pointer, tx)) - }) + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok::<_, StorageError>(chunk) + }) + .try_filter_map(move |chunk| async move { + let tx_ids = chunk.iter().map(|(_, tx_id)| *tx_id).collect::>(); + let txs = self.transactions(tx_ids).await; + let txs = txs + .into_iter() + .zip(chunk) + .map(|(result, (tx_pointer, _))| result.map(|tx| (tx_pointer, tx))); + Ok(Some(futures::stream::iter(txs))) }) + .try_flatten() } } diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index d7bc8366282..0dd83103545 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -44,12 +44,14 @@ use fuel_core_types::{ block::CompressedBlock, header::BlockHeader, }, + fuel_tx::TxId, fuel_types, fuel_types::BlockHeight, }; use futures::{ Stream, StreamExt, + TryStreamExt, }; pub struct Block(pub(crate) CompressedBlock); @@ -135,14 +137,27 @@ impl Block { ctx: &Context<'_>, ) -> async_graphql::Result> { let query = ctx.read_view()?; - self.0 - .transactions() - .iter() - .map(|tx_id| { - let tx = query.transaction(tx_id)?; - Ok(Transaction::from_tx(*tx_id, tx)) + let tx_ids = futures::stream::iter(self.0.transactions().iter().copied()); + + let result = tx_ids + .chunks(query.butch_size) + .filter_map(move |tx_ids: Vec| { + let async_query = query.as_ref().clone(); + async move { + let txs = async_query.transactions(tx_ids.clone()).await; + let txs = txs + .into_iter() + .zip(tx_ids.into_iter()) + .map(|(r, tx_id)| r.map(|tx| Transaction::from_tx(tx_id, tx))); + + Some(futures::stream::iter(txs)) + } }) - .collect() + .flatten() + .try_collect() + .await?; + + Ok(result) } } diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index 877aaa9df91..e58ef7ea911 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -71,7 +71,6 @@ use std::{ borrow::Cow, iter, }; -use tokio_stream::StreamExt; use types::{ DryRunTransactionExecutionStatus, Transaction, @@ -123,7 +122,9 @@ impl TxQuery { ) -> async_graphql::Result< Connection, > { + use futures::stream::StreamExt; let query = ctx.read_view()?; + let query_ref = query.as_ref(); crate::schema::query_pagination( after, before, @@ -158,14 +159,36 @@ impl TxQuery { async move { Ok(skip) } }) - .map(|result: StorageResult| { - result.and_then(|sorted| { - // TODO: Request transactions in a separate thread - let tx = query.transaction(&sorted.tx_id.0)?; - - Ok((sorted, Transaction::from_tx(sorted.tx_id.0, tx))) - }) - }); + .chunks(query_ref.butch_size) + .filter_map(move |chunk: Vec>| { + let async_query = query_ref.clone(); + async move { + use itertools::Itertools; + let result = chunk.into_iter().try_collect::<_, Vec<_>, _>(); + + let chunk = match result { + Ok(chunk) => chunk, + Err(err) => { + return Some(Err(err)); + } + }; + + let tx_ids = chunk + .iter() + .map(|sorted| sorted.tx_id.0) + .collect::>(); + let txs = async_query.transactions(tx_ids).await; + let txs = txs.into_iter().zip(chunk.into_iter()).map( + |(result, sorted)| { + result.map(|tx| { + (sorted, Transaction::from_tx(sorted.tx_id.0, tx)) + }) + }, + ); + Some(Ok(futures::stream::iter(txs))) + } + }) + .try_flatten(); Ok(all_txs) }, @@ -188,6 +211,7 @@ impl TxQuery { before: Option, ) -> async_graphql::Result> { + use futures::stream::StreamExt; let query = ctx.read_view()?; let params = ctx .data_unchecked::() @@ -382,6 +406,7 @@ impl TxStatusSubscription { ) -> async_graphql::Result< impl Stream> + 'a, > { + use tokio_stream::StreamExt; let subscription = submit_and_await_status(ctx, tx).await?; Ok(subscription @@ -410,6 +435,7 @@ async fn submit_and_await_status<'a>( ) -> async_graphql::Result< impl Stream> + 'a, > { + use tokio_stream::StreamExt; let txpool = ctx.data_unchecked::(); let params = ctx .data_unchecked::() diff --git a/crates/fuel-core/src/schema/tx/types.rs b/crates/fuel-core/src/schema/tx/types.rs index 097d6f06fdd..48c78e3509e 100644 --- a/crates/fuel-core/src/schema/tx/types.rs +++ b/crates/fuel-core/src/schema/tx/types.rs @@ -987,7 +987,7 @@ pub(crate) async fn get_tx_status( txpool: &TxPool, ) -> Result, StorageError> { match query - .status(&id) + .tx_status(&id) .into_api_result::()? { Some(status) => { diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index e5473135d6d..ed75edae477 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -138,6 +138,7 @@ impl Config { std::net::Ipv4Addr::new(127, 0, 0, 1).into(), 0, ), + database_butch_size: 100, max_queries_depth: 16, max_queries_complexity: 80000, max_queries_recursive_depth: 16, diff --git a/tests/test-helpers/src/builder.rs b/tests/test-helpers/src/builder.rs index 91134ceb7c3..0d16bc48dc6 100644 --- a/tests/test-helpers/src/builder.rs +++ b/tests/test-helpers/src/builder.rs @@ -93,6 +93,7 @@ pub struct TestSetupBuilder { pub initial_coins: Vec, pub starting_gas_price: u64, pub gas_limit: Option, + pub block_size_limit: Option, pub starting_block: Option, pub utxo_validation: bool, pub privileged_address: Address, @@ -201,6 +202,13 @@ impl TestSetupBuilder { .set_block_gas_limit(gas_limit); } + if let Some(block_size_limit) = self.block_size_limit { + chain_conf + .consensus_parameters + .set_block_transaction_size_limit(block_size_limit) + .expect("Should set new block size limit"); + } + chain_conf .consensus_parameters .set_privileged_address(self.privileged_address); @@ -251,6 +259,7 @@ impl Default for TestSetupBuilder { initial_coins: vec![], starting_gas_price: 0, gas_limit: None, + block_size_limit: None, starting_block: None, utxo_validation: true, privileged_address: Default::default(), diff --git a/tests/tests/dos.rs b/tests/tests/dos.rs index b9067738a5d..52109c46c6e 100644 --- a/tests/tests/dos.rs +++ b/tests/tests/dos.rs @@ -1,6 +1,9 @@ #![allow(non_snake_case)] -use std::time::Instant; +use std::time::{ + Duration, + Instant, +}; use fuel_core::service::{ Config, @@ -666,3 +669,40 @@ async fn schema_is_retrievable() { let result = send_graph_ql_query(&url, query).await; assert!(result.contains("__schema"), "{:?}", result); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 8)] +async fn heavy_tasks_doesnt_block_graphql() { + let mut config = Config::local_node(); + + const NUM_OF_BLOCKS: u32 = 4000; + config.graphql_config.max_queries_complexity = 10_000_000; + + let query = FULL_BLOCK_QUERY.to_string(); + let query = query.replace("$NUMBER_OF_BLOCKS", NUM_OF_BLOCKS.to_string().as_str()); + + let node = FuelService::new_node(config).await.unwrap(); + let url = format!("http://{}/v1/graphql", node.bound_address); + let client = FuelClient::new(url.clone()).unwrap(); + client.produce_blocks(NUM_OF_BLOCKS, None).await.unwrap(); + + // Given + for _ in 0..50 { + let url = url.clone(); + let query = query.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + let result = send_graph_ql_query(&url, &query).await; + assert!(result.contains("transactions")); + }); + } + // Wait for all queries to start be processed on the node. + tokio::time::sleep(Duration::from_secs(1)).await; + + // When + let result = tokio::time::timeout(Duration::from_secs(5), client.health()).await; + + // Then + let result = result.expect("Health check timed out"); + let health = result.expect("Health check failed"); + assert!(health); +} From 142cd1d2f1dc48c650efd4344247af3f8010f2b8 Mon Sep 17 00:00:00 2001 From: green Date: Sun, 13 Oct 2024 18:26:15 +0200 Subject: [PATCH 07/19] Use correct naming --- bin/fuel-core/src/cli/run.rs | 2 +- bin/fuel-core/src/cli/run/graphql.rs | 6 +++--- crates/fuel-core/src/graphql_api.rs | 2 +- crates/fuel-core/src/graphql_api/api_service.rs | 2 +- crates/fuel-core/src/graphql_api/database.rs | 16 ++++++++-------- crates/fuel-core/src/query/balance.rs | 2 +- .../fuel-core/src/query/balance/asset_query.rs | 4 ++-- crates/fuel-core/src/query/block.rs | 2 +- crates/fuel-core/src/query/coin.rs | 2 +- crates/fuel-core/src/query/message.rs | 2 +- crates/fuel-core/src/query/tx.rs | 2 +- crates/fuel-core/src/schema/block.rs | 2 +- crates/fuel-core/src/schema/tx.rs | 2 +- crates/fuel-core/src/service/config.rs | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bin/fuel-core/src/cli/run.rs b/bin/fuel-core/src/cli/run.rs index 1a76fb18a68..93964f7c10e 100644 --- a/bin/fuel-core/src/cli/run.rs +++ b/bin/fuel-core/src/cli/run.rs @@ -488,7 +488,7 @@ impl Command { let config = Config { graphql_config: GraphQLConfig { addr, - database_butch_size: graphql.database_butch_size, + database_batch_size: graphql.database_batch_size, max_queries_depth: graphql.graphql_max_depth, max_queries_complexity: graphql.graphql_max_complexity, max_queries_recursive_depth: graphql.graphql_max_recursive_depth, diff --git a/bin/fuel-core/src/cli/run/graphql.rs b/bin/fuel-core/src/cli/run/graphql.rs index 73c682d0643..90a1ad9508c 100644 --- a/bin/fuel-core/src/cli/run/graphql.rs +++ b/bin/fuel-core/src/cli/run/graphql.rs @@ -12,9 +12,9 @@ pub struct GraphQLArgs { #[clap(long = "port", default_value = "4000", env)] pub port: u16, - /// The size of the butch fetched from the database by GraphQL service. - #[clap(long = "graphql-database-butch-size", default_value = "100", env)] - pub database_butch_size: usize, + /// The size of the batch fetched from the database by GraphQL service. + #[clap(long = "graphql-database-batch-size", default_value = "100", env)] + pub database_batch_size: usize, /// The max depth of GraphQL queries. #[clap(long = "graphql-max-depth", default_value = "16", env)] diff --git a/crates/fuel-core/src/graphql_api.rs b/crates/fuel-core/src/graphql_api.rs index 8b8706b75e5..247a79bb59a 100644 --- a/crates/fuel-core/src/graphql_api.rs +++ b/crates/fuel-core/src/graphql_api.rs @@ -20,7 +20,7 @@ pub mod worker_service; #[derive(Clone, Debug)] pub struct ServiceConfig { pub addr: SocketAddr, - pub database_butch_size: usize, + pub database_batch_size: usize, pub max_queries_depth: usize, pub max_queries_complexity: usize, pub max_queries_recursive_depth: usize, diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index ce0fdbc9f6c..0c185d472a7 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -222,7 +222,7 @@ where { let network_addr = config.config.addr; let combined_read_database = ReadDatabase::new( - config.config.database_butch_size, + config.config.database_batch_size, genesis_block_height, on_database, off_database, diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index bd08761870c..52b1cef5e27 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -78,8 +78,8 @@ pub type OffChainView = Arc; /// The container of the on-chain and off-chain database view provides. /// It is used only by `ViewExtension` to create a [`ReadView`]. pub struct ReadDatabase { - /// The size of the butch during fetching from the database. - butch_size: usize, + /// The size of the batch during fetching from the database. + batch_size: usize, /// The height of the genesis block. genesis_height: BlockHeight, /// The on-chain database view provider. @@ -91,7 +91,7 @@ pub struct ReadDatabase { impl ReadDatabase { /// Creates a new [`ReadDatabase`] with the given on-chain and off-chain database view providers. pub fn new( - butch_size: usize, + batch_size: usize, genesis_height: BlockHeight, on_chain: OnChain, off_chain: OffChain, @@ -103,7 +103,7 @@ impl ReadDatabase { OffChain::LatestView: OffChainDatabase, { Self { - butch_size, + batch_size, genesis_height, on_chain: Box::new(ArcWrapper::new(on_chain)), off_chain: Box::new(ArcWrapper::new(off_chain)), @@ -116,7 +116,7 @@ impl ReadDatabase { // It is not possible to implement until `view_at` is implemented for the `AtomicView`. // https://github.com/FuelLabs/fuel-core/issues/1582 Ok(ReadView { - butch_size: self.butch_size, + batch_size: self.batch_size, genesis_height: self.genesis_height, on_chain: self.on_chain.latest_view()?, off_chain: self.off_chain.latest_view()?, @@ -131,7 +131,7 @@ impl ReadDatabase { #[derive(Clone)] pub struct ReadView { - pub(crate) butch_size: usize, + pub(crate) batch_size: usize, pub(crate) genesis_height: BlockHeight, pub(crate) on_chain: OnChainView, pub(crate) off_chain: OffChainView, @@ -273,7 +273,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { futures::stream::iter(self.on_chain.all_messages(start_message_id, direction)) - .chunks(self.butch_size) + .chunks(self.batch_size) .filter_map(|chunk| async move { // Give a chance to other tasks to run. tokio::task::yield_now().await; @@ -304,7 +304,7 @@ impl ReadView { start_asset, direction, )) - .chunks(self.butch_size) + .chunks(self.batch_size) .filter_map(|chunk| async move { // Give a chance to other tasks to run. tokio::task::yield_now().await; diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 0638dc768b9..74c0099cf28 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -106,7 +106,7 @@ impl ReadView { }) .map_ok(|stream| stream.map(Ok)) .try_flatten() - .chunks(self.butch_size) + .chunks(self.batch_size) .filter_map(|chunk| async move { // Give a chance to other tasks to run. tokio::task::yield_now().await; diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index 3c9ba49cee6..f404d22ae12 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -106,7 +106,7 @@ impl<'a> AssetsQuery<'a> { }) }); - futures::stream::StreamExt::chunks(stream, database.butch_size) + futures::stream::StreamExt::chunks(stream, database.batch_size) .map(|chunk| { use itertools::Itertools; @@ -159,7 +159,7 @@ impl<'a> AssetsQuery<'a> { }) }); - futures::stream::StreamExt::chunks(stream, database.butch_size) + futures::stream::StreamExt::chunks(stream, database.batch_size) .map(|chunk| { use itertools::Itertools; diff --git a/crates/fuel-core/src/query/block.rs b/crates/fuel-core/src/query/block.rs index 60f98838879..bf5bf91c206 100644 --- a/crates/fuel-core/src/query/block.rs +++ b/crates/fuel-core/src/query/block.rs @@ -27,7 +27,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { futures::stream::iter(self.blocks(height, direction)) - .chunks(self.butch_size) + .chunks(self.batch_size) .filter_map(|chunk| async move { // Give a chance to other tasks to run. tokio::task::yield_now().await; diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 7de4a5211d6..641cbf99d3b 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -49,7 +49,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { self.owned_coins_ids(owner, start_coin, direction) - .chunks(self.butch_size) + .chunks(self.batch_size) .map(|chunk| { use itertools::Itertools; diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 51550af3555..5d90f559d43 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -98,7 +98,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + 'a { self.owned_message_ids(owner, start_message_id, direction) - .chunks(self.butch_size) + .chunks(self.batch_size) .map(|chunk| { let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; Ok(chunk) diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index cfeaf00f8d1..0bceeef8809 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -41,7 +41,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { self.owned_transactions_ids(owner, start, direction) - .chunks(self.butch_size) + .chunks(self.batch_size) .map(|chunk| { use itertools::Itertools; diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index 0dd83103545..8bf87a96e5c 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -140,7 +140,7 @@ impl Block { let tx_ids = futures::stream::iter(self.0.transactions().iter().copied()); let result = tx_ids - .chunks(query.butch_size) + .chunks(query.batch_size) .filter_map(move |tx_ids: Vec| { let async_query = query.as_ref().clone(); async move { diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index e58ef7ea911..c90b25fed89 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -159,7 +159,7 @@ impl TxQuery { async move { Ok(skip) } }) - .chunks(query_ref.butch_size) + .chunks(query_ref.batch_size) .filter_map(move |chunk: Vec>| { let async_query = query_ref.clone(); async move { diff --git a/crates/fuel-core/src/service/config.rs b/crates/fuel-core/src/service/config.rs index ed75edae477..e990a978949 100644 --- a/crates/fuel-core/src/service/config.rs +++ b/crates/fuel-core/src/service/config.rs @@ -138,7 +138,7 @@ impl Config { std::net::Ipv4Addr::new(127, 0, 0, 1).into(), 0, ), - database_butch_size: 100, + database_batch_size: 100, max_queries_depth: 16, max_queries_complexity: 80000, max_queries_recursive_depth: 16, From 671e80edfb636ee0e8e6c5dbf0ea89d33f1738b9 Mon Sep 17 00:00:00 2001 From: green Date: Sun, 13 Oct 2024 18:28:37 +0200 Subject: [PATCH 08/19] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09cd4c1b624..7daf30ecf62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2334](https://github.com/FuelLabs/fuel-core/pull/2334): Prepare the GraphQL service for the switching to `async` methods. - [2337](https://github.com/FuelLabs/fuel-core/pull/2337): Updated all pagination queries to work with the async stream instead of the sync iterator. +- [2340](https://github.com/FuelLabs/fuel-core/pull/2340): Avoid long heavy tasks in the GraphQL service by splitting work into batches. #### Breaking - [2337](https://github.com/FuelLabs/fuel-core/pull/2337): The maximum number of processed coins from the `coins_to_spend` query is limited to `max_inputs`. From 2354d1099465164df24da106891757e21fec8ef0 Mon Sep 17 00:00:00 2001 From: green Date: Sun, 13 Oct 2024 18:33:48 +0200 Subject: [PATCH 09/19] Make closure more readable --- crates/fuel-core/src/schema/tx.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index c90b25fed89..1f211da72d2 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -157,22 +157,18 @@ impl TxQuery { false }; - async move { Ok(skip) } + async move { Ok::<_, StorageError>(skip) } }) .chunks(query_ref.batch_size) - .filter_map(move |chunk: Vec>| { + .map(|chunk| { + use itertools::Itertools; + + let chunk = chunk.into_iter().try_collect::<_, Vec<_>, _>()?; + Ok::<_, StorageError>(chunk) + }) + .try_filter_map(move |chunk| { let async_query = query_ref.clone(); async move { - use itertools::Itertools; - let result = chunk.into_iter().try_collect::<_, Vec<_>, _>(); - - let chunk = match result { - Ok(chunk) => chunk, - Err(err) => { - return Some(Err(err)); - } - }; - let tx_ids = chunk .iter() .map(|sorted| sorted.tx_id.0) @@ -185,7 +181,7 @@ impl TxQuery { }) }, ); - Some(Ok(futures::stream::iter(txs))) + Ok(Some(futures::stream::iter(txs))) } }) .try_flatten(); From d982fbc2ba14f3ac72249c0fad7ec129db33f727 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 08:59:31 +0200 Subject: [PATCH 10/19] Apply comment from parent PR --- crates/fuel-core/src/graphql_api/database.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 52b1cef5e27..6c1c173fca4 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -351,12 +351,12 @@ impl ReadView { futures::stream::iter(iter) } - pub fn owned_message_ids<'a>( - &'a self, - owner: &'a Address, + pub fn owned_message_ids( + &self, + owner: &Address, start_message_id: Option, direction: IterDirection, - ) -> impl Stream> + 'a { + ) -> impl Stream> + '_ { futures::stream::iter(self.off_chain.owned_message_ids( owner, start_message_id, From c355cd1569c785379018912aa3ab1a1661d7ee8b Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 09:03:37 +0200 Subject: [PATCH 11/19] Use more clear naming for the logic --- .../fuel-core/src/query/balance/asset_query.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index bbed21c5568..b1da4a7b821 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -53,7 +53,7 @@ impl Exclude { #[derive(Clone)] pub struct AssetsQuery<'a> { pub owner: &'a Address, - pub assets: Option>, + pub allowed_assets: Option>, pub exclude: Option<&'a Exclude>, pub database: &'a ReadView, pub base_asset_id: &'a AssetId, @@ -62,14 +62,14 @@ pub struct AssetsQuery<'a> { impl<'a> AssetsQuery<'a> { pub fn new( owner: &'a Address, - assets: Option>, + allowed_assets: Option>, exclude: Option<&'a Exclude>, database: &'a ReadView, base_asset_id: &'a AssetId, ) -> Self { Self { owner, - assets, + allowed_assets, exclude, database, base_asset_id, @@ -77,7 +77,7 @@ impl<'a> AssetsQuery<'a> { } fn coins_iter(mut self) -> impl Stream> + 'a { - let assets = self.assets.take(); + let allowed_assets = self.allowed_assets.take(); self.database .owned_coins_ids(self.owner, None, IterDirection::Forward) .map(|id| id.map(CoinId::from)) @@ -107,7 +107,7 @@ impl<'a> AssetsQuery<'a> { }) .filter(move |result| { if let Ok(CoinType::Coin(coin)) = result { - has_asset(&assets, &coin.asset_id) + allowed_asset(&allowed_assets, &coin.asset_id) } else { true } @@ -167,7 +167,7 @@ impl<'a> AssetsQuery<'a> { // TODO: Optimize this by creating an index // https://github.com/FuelLabs/fuel-core/issues/588 pub fn coins(self) -> impl Stream> + 'a { - let has_base_asset = has_asset(&self.assets, self.base_asset_id); + let has_base_asset = allowed_asset(&self.allowed_assets, self.base_asset_id); if has_base_asset { let message_iter = self.messages_iter(); self.coins_iter().chain(message_iter).into_boxed_ref() @@ -218,9 +218,9 @@ impl<'a> AssetQuery<'a> { } } -fn has_asset(assets: &Option>, asset_id: &AssetId) -> bool { - assets +fn allowed_asset(allowed_assets: &Option>, asset_id: &AssetId) -> bool { + allowed_assets .as_ref() - .map(|assets| assets.contains(asset_id)) + .map(|allowed_assets| allowed_assets.contains(asset_id)) .unwrap_or(true) } From 5a752bc0b7f08b507077aa0bb1953c7faee80da0 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 09:08:39 +0200 Subject: [PATCH 12/19] Fix flakiness --- crates/services/src/async_processor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/services/src/async_processor.rs b/crates/services/src/async_processor.rs index 6a5b43f395f..383bfc150c2 100644 --- a/crates/services/src/async_processor.rs +++ b/crates/services/src/async_processor.rs @@ -287,6 +287,7 @@ mod tests { // Then while broadcast_receiver.recv().await.is_ok() {} + tokio::task::yield_now().await; assert!(instant.elapsed() <= Duration::from_secs(2)); let duration = Duration::from_nanos(heavy_task_processor.metric.busy.get()); assert_eq!(duration.as_secs(), 0); From a8c19425233446c47f39677d713f15b3a6420498 Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Mon, 14 Oct 2024 10:09:08 +0200 Subject: [PATCH 13/19] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mårten Blankfors --- crates/fuel-core/src/query/balance/asset_query.rs | 2 +- crates/fuel-core/src/query/coin.rs | 2 +- crates/fuel-core/src/query/message.rs | 2 +- crates/fuel-core/src/query/tx.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index b1da4a7b821..5482f6b5272 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -138,7 +138,7 @@ impl<'a> AssetsQuery<'a> { } else { return Err(anyhow::anyhow!("The coin is not a message").into()); }; - // TODO: Fetch message in a separate thread + // TODO: Fetch message in a separate thread (https://github.com/FuelLabs/fuel-core/pull/2340) let message = database.message(&id)?; Ok(message) }) diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index c6d52001ddd..94a2fbafe54 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -38,7 +38,7 @@ impl ReadView { self.owned_coins_ids(owner, start_coin, direction) .map(|res| { res.and_then(|id| { - // TODO: Move fetching of the coin to a separate thread + // TODO: Move fetching of the coin to a separate thread (https://github.com/FuelLabs/fuel-core/pull/2340) self.coin(id) }) }) diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 89cd21b8ae7..56c83431ee5 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -90,7 +90,7 @@ impl ReadView { self.owned_message_ids(owner, start_message_id, direction) .map(|result| { result.and_then(|id| { - // TODO: Move `message` fetching to a separate thread + // TODO: Move `message` fetching to a separate thread (https://github.com/FuelLabs/fuel-core/pull/2340) self.message(&id) }) }) diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 8989efc9bd2..6898e9155a4 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -45,7 +45,7 @@ impl ReadView { self.owned_transactions_ids(owner, start, direction) .map(|result| { result.and_then(|(tx_pointer, tx_id)| { - // TODO: Fetch transactions in a separate thread + // TODO: Fetch transactions in a separate thread (https://github.com/FuelLabs/fuel-core/pull/2340) let tx = self.transaction(&tx_id)?; Ok((tx_pointer, tx)) From df80801202d272702a211c7243642fc98851f807 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 10:09:44 +0200 Subject: [PATCH 14/19] Apply comments from PR --- CHANGELOG.md | 4 ++-- crates/fuel-core/src/coins_query.rs | 1 + crates/fuel-core/src/schema/coins.rs | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09cd4c1b624..6d34ab078bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed - [2334](https://github.com/FuelLabs/fuel-core/pull/2334): Prepare the GraphQL service for the switching to `async` methods. -- [2337](https://github.com/FuelLabs/fuel-core/pull/2337): Updated all pagination queries to work with the async stream instead of the sync iterator. +- [2341](https://github.com/FuelLabs/fuel-core/pull/2341): Updated all pagination queries to work with the async stream instead of the sync iterator. #### Breaking -- [2337](https://github.com/FuelLabs/fuel-core/pull/2337): The maximum number of processed coins from the `coins_to_spend` query is limited to `max_inputs`. +- [2341](https://github.com/FuelLabs/fuel-core/pull/2341): The maximum number of processed coins from the `coins_to_spend` query is limited to `max_inputs`. ## [Version 0.39.0] diff --git a/crates/fuel-core/src/coins_query.rs b/crates/fuel-core/src/coins_query.rs index e07630333f8..c878c071335 100644 --- a/crates/fuel-core/src/coins_query.rs +++ b/crates/fuel-core/src/coins_query.rs @@ -164,6 +164,7 @@ pub async fn largest_first( // target and the part that does not(by choosing the most expensive coins). // When the target is satisfied, we can select random coins from the remaining // coins not used in the target. +// https://github.com/FuelLabs/fuel-core/issues/1965 pub async fn random_improve( db: &ReadView, spend_query: &SpendQuery, diff --git a/crates/fuel-core/src/schema/coins.rs b/crates/fuel-core/src/schema/coins.rs index a1066762380..f474c1c79d6 100644 --- a/crates/fuel-core/src/schema/coins.rs +++ b/crates/fuel-core/src/schema/coins.rs @@ -230,6 +230,7 @@ impl CoinQuery { // of inputs. // TODO: To avoid breaking changes, we will truncate request for now. // In the future, we should return an error if the input is too large. + // https://github.com/FuelLabs/fuel-core/issues/2343 query_per_asset.truncate(max_input as usize); let owner: fuel_tx::Address = owner.0; From 8cb24b013c47824e314275c8760444424a028dbc Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 10:24:33 +0200 Subject: [PATCH 15/19] Add link to issue for TODO --- crates/fuel-core/src/database/block.rs | 1 + crates/fuel-core/src/graphql_api/database.rs | 1 + crates/fuel-core/src/query/coin.rs | 1 + crates/fuel-core/src/query/message.rs | 2 ++ 4 files changed, 5 insertions(+) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 9202bd04043..374a76a9663 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -68,6 +68,7 @@ impl OnChainIterableKeyValueView { if let Some(block) = db_block { // fetch all the transactions // TODO: Use multiget when it's implemented. + // https://github.com/FuelLabs/fuel-core/issues/2344 let txs = block .transactions() .iter() diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 6c1c173fca4..574a9e1cc9e 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -156,6 +156,7 @@ impl ReadView { tx_ids: Vec, ) -> Vec> { // TODO: Use multiget when it's implemented. + // https://github.com/FuelLabs/fuel-core/issues/2344 let result = tx_ids .iter() .map(|tx_id| self.transaction(tx_id)) diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 641cbf99d3b..352b477bb87 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -36,6 +36,7 @@ impl ReadView { utxo_ids: Vec, ) -> impl Iterator> + '_ { // TODO: Use multiget when it's implemented. + // https://github.com/FuelLabs/fuel-core/issues/2344 let coins = utxo_ids.into_iter().map(|id| self.coin(id)); // Yield to the runtime to allow other tasks to run. tokio::task::yield_now().await; diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 5d90f559d43..0c89564196c 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -85,6 +85,8 @@ impl ReadView { &self, ids: Vec, ) -> impl Iterator> + '_ { + // TODO: Use multiget when it's implemented. + // https://github.com/FuelLabs/fuel-core/issues/2344 let messages = ids.into_iter().map(|id| self.message(&id)); // Yield to the runtime to allow other tasks to run. tokio::task::yield_now().await; From 678e0be9b47fdc7cec5a022f4295abdb7b58544c Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 10:38:09 +0200 Subject: [PATCH 16/19] Use `futures::TryStreamExt` --- crates/fuel-core/src/query/balance/asset_query.rs | 6 ++++-- crates/fuel-core/src/query/coin.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index f28ae6e08d4..13a289ec1e4 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -1,5 +1,4 @@ use crate::graphql_api::database::ReadView; -use async_graphql::futures_util::TryStreamExt; use fuel_core_services::stream::IntoBoxStream; use fuel_core_storage::{ iter::IterDirection, @@ -16,7 +15,10 @@ use fuel_core_types::{ AssetId, }, }; -use futures::Stream; +use futures::{ + Stream, + TryStreamExt, +}; use std::collections::HashSet; use tokio_stream::StreamExt; diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 352b477bb87..14a2a7c61ca 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,5 +1,4 @@ use crate::fuel_core_graphql_api::database::ReadView; -use async_graphql::futures_util::TryStreamExt; use fuel_core_storage::{ iter::IterDirection, not_found, @@ -16,6 +15,7 @@ use fuel_core_types::{ use futures::{ Stream, StreamExt, + TryStreamExt, }; impl ReadView { From 727b2da6e460dc2fdb03e56ac01bef7f04f4379d Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 12:06:11 +0200 Subject: [PATCH 17/19] Implemented `yield_each` for the stream allowing other tasks to proceed --- crates/fuel-core/src/graphql_api/database.rs | 18 +- crates/fuel-core/src/query/balance.rs | 9 +- crates/fuel-core/src/query/block.rs | 15 +- crates/fuel-core/src/query/coin.rs | 2 +- crates/fuel-core/src/query/message.rs | 2 +- crates/services/Cargo.toml | 1 + crates/services/src/lib.rs | 1 + crates/services/src/yield_stream.rs | 167 +++++++++++++++++++ 8 files changed, 179 insertions(+), 36 deletions(-) create mode 100644 crates/services/src/yield_stream.rs diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 574a9e1cc9e..bf47c8d92a7 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -5,7 +5,7 @@ use crate::fuel_core_graphql_api::{ OnChainDatabase, }, }; -use async_graphql::futures_util::StreamExt; +use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::{ BoxedIter, @@ -274,13 +274,7 @@ impl ReadView { direction: IterDirection, ) -> impl Stream> + '_ { futures::stream::iter(self.on_chain.all_messages(start_message_id, direction)) - .chunks(self.batch_size) - .filter_map(|chunk| async move { - // Give a chance to other tasks to run. - tokio::task::yield_now().await; - Some(futures::stream::iter(chunk)) - }) - .flatten() + .yield_each(self.batch_size) } pub fn message_exists(&self, nonce: &Nonce) -> StorageResult { @@ -305,13 +299,7 @@ impl ReadView { start_asset, direction, )) - .chunks(self.batch_size) - .filter_map(|chunk| async move { - // Give a chance to other tasks to run. - tokio::task::yield_now().await; - Some(futures::stream::iter(chunk)) - }) - .flatten() + .yield_each(self.batch_size) } pub fn da_height(&self) -> StorageResult { diff --git a/crates/fuel-core/src/query/balance.rs b/crates/fuel-core/src/query/balance.rs index 74c0099cf28..161fd64b87e 100644 --- a/crates/fuel-core/src/query/balance.rs +++ b/crates/fuel-core/src/query/balance.rs @@ -4,6 +4,7 @@ use asset_query::{ AssetSpendTarget, AssetsQuery, }; +use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::IterDirection, Result as StorageResult, @@ -106,12 +107,6 @@ impl ReadView { }) .map_ok(|stream| stream.map(Ok)) .try_flatten() - .chunks(self.batch_size) - .filter_map(|chunk| async move { - // Give a chance to other tasks to run. - tokio::task::yield_now().await; - Some(futures::stream::iter(chunk)) - }) - .flatten() + .yield_each(self.batch_size) } } diff --git a/crates/fuel-core/src/query/block.rs b/crates/fuel-core/src/query/block.rs index bf5bf91c206..f4b461639f0 100644 --- a/crates/fuel-core/src/query/block.rs +++ b/crates/fuel-core/src/query/block.rs @@ -1,4 +1,5 @@ use crate::fuel_core_graphql_api::database::ReadView; +use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::IterDirection, Result as StorageResult, @@ -7,10 +8,7 @@ use fuel_core_types::{ blockchain::block::CompressedBlock, fuel_types::BlockHeight, }; -use futures::{ - Stream, - StreamExt, -}; +use futures::Stream; impl ReadView { pub fn latest_block_height(&self) -> StorageResult { @@ -26,13 +24,6 @@ impl ReadView { height: Option, direction: IterDirection, ) -> impl Stream> + '_ { - futures::stream::iter(self.blocks(height, direction)) - .chunks(self.batch_size) - .filter_map(|chunk| async move { - // Give a chance to other tasks to run. - tokio::task::yield_now().await; - Some(futures::stream::iter(chunk)) - }) - .flatten() + futures::stream::iter(self.blocks(height, direction)).yield_each(self.batch_size) } } diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 14a2a7c61ca..c487bdba23c 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -38,7 +38,7 @@ impl ReadView { // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 let coins = utxo_ids.into_iter().map(|id| self.coin(id)); - // Yield to the runtime to allow other tasks to run. + // Give a chance to other tasks to run. tokio::task::yield_now().await; coins } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 0c89564196c..c98b2358b2c 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -88,7 +88,7 @@ impl ReadView { // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 let messages = ids.into_iter().map(|id| self.message(&id)); - // Yield to the runtime to allow other tasks to run. + // Give a chance to other tasks to run. tokio::task::yield_now().await; messages } diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 345bcd64287..82b5a17e998 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -18,6 +18,7 @@ parking_lot = { workspace = true } rayon = { workspace = true, optional = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } +pin-project-lite = { workspace = true } [dev-dependencies] fuel-core-services = { path = ".", features = ["sync-processor"] } diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 10da0bc75f7..e7fa438631f 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -11,6 +11,7 @@ mod state; mod sync; #[cfg(feature = "sync-processor")] mod sync_processor; +pub mod yield_stream; /// Re-exports for streaming utilities pub mod stream { diff --git a/crates/services/src/yield_stream.rs b/crates/services/src/yield_stream.rs new file mode 100644 index 00000000000..24a207b4787 --- /dev/null +++ b/crates/services/src/yield_stream.rs @@ -0,0 +1,167 @@ +//! Stream that yields each `batch_size` items allowing other tasks to work. + +use futures::{ + ready, + stream::Fuse, + Stream, + StreamExt, +}; +use std::{ + pin::Pin, + task::{ + Context, + Poll, + }, +}; + +pin_project_lite::pin_project! { + /// Stream that yields each `batch_size` items. + #[derive(Debug)] + #[must_use = "streams do nothing unless polled"] + pub struct YieldStream { + #[pin] + stream: Fuse, + item: Option, + counter: usize, + batch_size: usize, + } +} + +impl YieldStream { + /// Create a new `YieldStream` with the given `batch_size`. + pub fn new(stream: St, batch_size: usize) -> Self { + assert!(batch_size > 0); + + Self { + stream: stream.fuse(), + item: None, + counter: 0, + batch_size, + } + } +} + +impl Stream for YieldStream { + type Item = St::Item; + + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + let mut this = self.as_mut().project(); + loop { + // If we have a cached item, return it because that means we were woken up. + if let Some(item) = this.item.take() { + *this.counter = 1; + return Poll::Ready(Some(item)); + } + + match ready!(this.stream.as_mut().poll_next(cx)) { + // Return items, unless we reached the batch size. + // after that, we want to yield before returning the next item. + Some(item) => { + if this.counter < this.batch_size { + *this.counter = this.counter.saturating_add(1); + + return Poll::Ready(Some(item)); + } else { + *this.item = Some(item); + + cx.waker().wake_by_ref(); + + return Poll::Pending; + } + } + + // Underlying stream ran out of values, so finish this stream as well. + None => { + return Poll::Ready(None); + } + } + } + } + + fn size_hint(&self) -> (usize, Option) { + let cached_len = usize::from(self.item.is_some()); + let (lower, upper) = self.stream.size_hint(); + let lower = lower.saturating_add(cached_len); + let upper = match upper { + Some(x) => x.checked_add(cached_len), + None => None, + }; + (lower, upper) + } +} + +/// Extension trait for `Stream`. +pub trait StreamYieldExt: Stream { + /// Yields each `batch_size` items allowing other tasks to work. + fn yield_each(self, batch_size: usize) -> YieldStream + where + Self: Sized, + { + YieldStream::new(self, batch_size) + } +} + +impl StreamYieldExt for St where St: Stream {} + +#[cfg(test)] +#[allow(non_snake_case)] +mod tests { + use super::*; + + #[tokio::test] + async fn yield_stream__works_with_10_elements_loop() { + let stream = futures::stream::iter(0..10); + let mut yield_stream = YieldStream::new(stream, 3); + + let mut items = Vec::new(); + while let Some(item) = yield_stream.next().await { + items.push(item); + } + + assert_eq!(items, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + } + + #[tokio::test] + async fn yield_stream__works_with_10_elements__collect() { + let stream = futures::stream::iter(0..10); + let yield_stream = stream.yield_each(3); + + let items = yield_stream.collect::>().await; + + assert_eq!(items, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + } + + #[tokio::test] + async fn yield_stream__passed_control_to_another_future() { + let stream = futures::stream::iter(0..10); + let mut yield_stream = YieldStream::new(stream, 3); + + async fn second_future() -> i32 { + -1 + } + + let mut items = Vec::new(); + loop { + tokio::select! { + biased; + + item = yield_stream.next() => { + if let Some(item) = item { + items.push(item); + } else { + break; + } + } + + item = second_future() => { + items.push(item); + } + } + } + + assert_eq!(items, vec![0, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8, -1, 9]); + } +} From 5e0ec920fa339555590db8b570b1e822c100a6b9 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 12:13:36 +0200 Subject: [PATCH 18/19] Make CI happy --- crates/services/Cargo.toml | 2 +- crates/services/src/yield_stream.rs | 41 +++++++++++++---------------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 82b5a17e998..34d0726cdca 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -15,10 +15,10 @@ async-trait = { workspace = true } fuel-core-metrics = { workspace = true } futures = { workspace = true } parking_lot = { workspace = true } +pin-project-lite = { workspace = true } rayon = { workspace = true, optional = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } -pin-project-lite = { workspace = true } [dev-dependencies] fuel-core-services = { path = ".", features = ["sync-processor"] } diff --git a/crates/services/src/yield_stream.rs b/crates/services/src/yield_stream.rs index 24a207b4787..60331dbf119 100644 --- a/crates/services/src/yield_stream.rs +++ b/crates/services/src/yield_stream.rs @@ -49,35 +49,32 @@ impl Stream for YieldStream { cx: &mut Context<'_>, ) -> Poll> { let mut this = self.as_mut().project(); - loop { - // If we have a cached item, return it because that means we were woken up. - if let Some(item) = this.item.take() { - *this.counter = 1; - return Poll::Ready(Some(item)); - } - match ready!(this.stream.as_mut().poll_next(cx)) { - // Return items, unless we reached the batch size. - // after that, we want to yield before returning the next item. - Some(item) => { - if this.counter < this.batch_size { - *this.counter = this.counter.saturating_add(1); + // If we have a cached item, return it because that means we were woken up. + if let Some(item) = this.item.take() { + *this.counter = 1; + return Poll::Ready(Some(item)); + } - return Poll::Ready(Some(item)); - } else { - *this.item = Some(item); + match ready!(this.stream.as_mut().poll_next(cx)) { + // Return items, unless we reached the batch size. + // after that, we want to yield before returning the next item. + Some(item) => { + if this.counter < this.batch_size { + *this.counter = this.counter.saturating_add(1); - cx.waker().wake_by_ref(); + Poll::Ready(Some(item)) + } else { + *this.item = Some(item); - return Poll::Pending; - } - } + cx.waker().wake_by_ref(); - // Underlying stream ran out of values, so finish this stream as well. - None => { - return Poll::Ready(None); + Poll::Pending } } + + // Underlying stream ran out of values, so finish this stream as well. + None => Poll::Ready(None), } } From 3521a726886a59da4742a894bbe0a1ed45922d99 Mon Sep 17 00:00:00 2001 From: green Date: Mon, 14 Oct 2024 18:17:36 +0200 Subject: [PATCH 19/19] Merge with `master` --- CHANGELOG.md | 2 +- crates/fuel-core/src/graphql_api/api_service.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af11741de9..4cfe04ee094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2334](https://github.com/FuelLabs/fuel-core/pull/2334): Prepare the GraphQL service for the switching to `async` methods. - [2341](https://github.com/FuelLabs/fuel-core/pull/2341): Updated all pagination queries to work with the async stream instead of the sync iterator. +- [2350](https://github.com/FuelLabs/fuel-core/pull/2350): Limited the number of threads used by the GraphQL service. #### Breaking - [2341](https://github.com/FuelLabs/fuel-core/pull/2341): The maximum number of processed coins from the `coins_to_spend` query is limited to `max_inputs`. -- [2350](https://github.com/FuelLabs/fuel-core/pull/2350): Limited the number of threads used by the GraphQL service. ## [Version 0.39.0] diff --git a/crates/fuel-core/src/graphql_api/api_service.rs b/crates/fuel-core/src/graphql_api/api_service.rs index dc6f53a1856..ef19cf269fb 100644 --- a/crates/fuel-core/src/graphql_api/api_service.rs +++ b/crates/fuel-core/src/graphql_api/api_service.rs @@ -132,7 +132,11 @@ where F::Output: Send + 'static, { fn execute(&self, fut: F) { - let _ = self.processor.try_spawn(fut); + let result = self.processor.try_spawn(fut); + + if let Err(err) = result { + tracing::error!("Failed to spawn a task for GraphQL: {:?}", err); + } } }