From 44903332f3c723c3a51b1dec9ccd2e9e4b47d6c8 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2021 20:24:27 +0300 Subject: [PATCH 1/9] CLI options and DB upgrade --- bin/node/testing/src/bench.rs | 4 ++- client/cli/src/config.rs | 29 ++++++++++++++--- client/cli/src/params/database_params.rs | 14 ++++++++ client/cli/src/params/pruning_params.rs | 17 ++++++++-- client/db/src/lib.rs | 41 ++++++++++++++++++++---- client/db/src/upgrade.rs | 41 +++++++++++++++++++++--- client/db/src/utils.rs | 2 +- client/service/src/builder.rs | 8 +++-- client/service/src/config.rs | 13 ++++++-- client/service/src/lib.rs | 1 + client/service/test/src/client/mod.rs | 12 +++++-- client/service/test/src/lib.rs | 5 ++- utils/browser/src/lib.rs | 7 ++-- 13 files changed, 163 insertions(+), 31 deletions(-) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 3bc31c6e414a6..3b227f5e949dd 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -410,8 +410,10 @@ impl BenchDb { let db_config = sc_client_db::DatabaseSettings { state_cache_size: 16*1024*1024, state_cache_child_ratio: Some((0, 100)), - pruning: PruningMode::ArchiveAll, + state_pruning: PruningMode::ArchiveAll, source: database_type.into_settings(dir.into()), + keep_blocks: sc_client_db::KeepBlocks::All, + transaction_storage: sc_client_db::TransactionStorage::BlockBody, }; let task_executor = TaskExecutor::new(); diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 017d2b421683f..db69aa4fff126 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -32,7 +32,7 @@ use sc_service::config::{ NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods, TaskExecutor, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, }; -use sc_service::{ChainSpec, TracingReceiver}; +use sc_service::{ChainSpec, TracingReceiver, KeepBlocks, TransactionStorage }; use std::net::SocketAddr; use std::path::PathBuf; @@ -203,6 +203,13 @@ pub trait CliConfiguration: Sized { .unwrap_or_default()) } + /// Get the database transaction storage scheme. + fn database_transaction_storage(&self) -> Result { + Ok(self.database_params() + .map(|x| x.transaction_storage()) + .unwrap_or(TransactionStorage::BlockBody)) + } + /// Get the database backend variant. /// /// By default this is retrieved from `DatabaseParams` if it is available. Otherwise its `None`. @@ -244,16 +251,26 @@ pub trait CliConfiguration: Sized { Ok(Default::default()) } - /// Get the pruning mode. + /// Get the state pruning mode. /// /// By default this is retrieved from `PruningMode` if it is available. Otherwise its /// `PruningMode::default()`. - fn pruning(&self, unsafe_pruning: bool, role: &Role) -> Result { + fn state_pruning(&self, unsafe_pruning: bool, role: &Role) -> Result { self.pruning_params() - .map(|x| x.pruning(unsafe_pruning, role)) + .map(|x| x.state_pruning(unsafe_pruning, role)) .unwrap_or_else(|| Ok(Default::default())) } + /// Get the block pruning mode. + /// + /// By default this is retrieved from `block_pruning` if it is available. Otherwise its + /// `KeepBlocks::All`. + fn keep_blocks(&self) -> Result { + self.pruning_params() + .map(|x| x.block_pruning()) + .unwrap_or_else(|| Ok(KeepBlocks::All)) + } + /// Get the chain ID (string). /// /// By default this is retrieved from `SharedParams`. @@ -493,7 +510,9 @@ pub trait CliConfiguration: Sized { database: self.database_config(&config_dir, database_cache_size, database)?, state_cache_size: self.state_cache_size()?, state_cache_child_ratio: self.state_cache_child_ratio()?, - pruning: self.pruning(unsafe_pruning, &role)?, + state_pruning: self.state_pruning(unsafe_pruning, &role)?, + keep_blocks: self.keep_blocks()?, + transaction_storage: self.database_transaction_storage()?, wasm_method: self.wasm_method()?, wasm_runtime_overrides: self.wasm_runtime_overrides(), execution_strategies: self.execution_strategies(is_dev, is_validator)?, diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index 21529f65a56b0..8718f2f45e609 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -18,6 +18,7 @@ use crate::arg_enums::Database; use structopt::StructOpt; +use sc_service::TransactionStorage; /// Parameters for block import. #[derive(Debug, StructOpt)] @@ -34,6 +35,10 @@ pub struct DatabaseParams { /// Limit the memory the database cache can use. #[structopt(long = "db-cache", value_name = "MiB")] pub database_cache_size: Option, + + /// Enable storage chain mode + #[structopt(long = "storage-chain")] + pub storage_chain: bool, } impl DatabaseParams { @@ -46,4 +51,13 @@ impl DatabaseParams { pub fn database_cache_size(&self) -> Option { self.database_cache_size } + + /// Transaction storage scheme. + pub fn transaction_storage(&self) -> TransactionStorage { + if self.storage_chain { + TransactionStorage::StorageChain + } else { + TransactionStorage::BlockBody + } + } } diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index 80118cafd8769..29d4b82feb58b 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::error; -use sc_service::{PruningMode, Role}; +use sc_service::{PruningMode, Role, KeepBlocks}; use structopt::StructOpt; /// Parameters to define the pruning mode @@ -30,11 +30,16 @@ pub struct PruningParams { /// 256 blocks. #[structopt(long = "pruning", value_name = "PRUNING_MODE")] pub pruning: Option, + /// Specify the number of finalized blocks to keep in the database. + /// + /// Default is to keep all blocks. + #[structopt(long = "block-pruning", value_name = "COUNT")] + pub block_pruning: Option, } impl PruningParams { /// Get the pruning value from the parameters - pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result { + pub fn state_pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result { // by default we disable pruning if the node is an authority (i.e. // `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the // node is an authority and pruning is enabled explicitly, then we error @@ -58,4 +63,12 @@ impl PruningParams { } }) } + + /// Get the block pruning value from the parameters + pub fn block_pruning(&self) -> error::Result { + Ok(match self.block_pruning { + Some(n) => KeepBlocks::Some(n), + None => KeepBlocks::All, + }) + } } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index e3b94b03c87d8..2aa8c914a79fa 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -264,10 +264,33 @@ pub struct DatabaseSettings { pub state_cache_size: usize, /// Ratio of cache size dedicated to child tries. pub state_cache_child_ratio: Option<(usize, usize)>, - /// Pruning mode. - pub pruning: PruningMode, + /// State pruning mode. + pub state_pruning: PruningMode, /// Where to find the database. pub source: DatabaseSettingsSrc, + /// Block pruning mode. + pub keep_blocks: KeepBlocks, + /// Block body/Transaction storage scheme. + pub transaction_storage: TransactionStorage, +} + +/// Block pruning settings. +#[derive(Debug, Clone)] +pub enum KeepBlocks { + /// Keep full block history. + All, + /// Keep N recent finalized blocks. + Some(u32), +} + +/// Block body storage scheme. +#[derive(Debug, Clone)] +pub enum TransactionStorage { + /// Store block body as deeply encoded list of transactions in the BODY column + BlockBody, + /// Store a list of hashes in the BODY column and each transaction individually + /// in the TRANSACTION column. + StorageChain, } /// Where to find the database.. @@ -334,6 +357,8 @@ pub(crate) mod columns { /// Offchain workers local storage pub const OFFCHAIN: u32 = 9; pub const CACHE: u32 = 10; + /// Transactions + pub const TRANSACTION: u32 = 11; } struct PendingBlock { @@ -876,8 +901,10 @@ impl Backend { let db_setting = DatabaseSettings { state_cache_size: 16777216, state_cache_child_ratio: Some((50, 100)), - pruning: PruningMode::keep_blocks(keep_blocks), + state_pruning: PruningMode::keep_blocks(keep_blocks), source: DatabaseSettingsSrc::Custom(db), + keep_blocks: KeepBlocks::All, + transaction_storage: TransactionStorage::BlockBody, }; Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") @@ -888,12 +915,12 @@ impl Backend { canonicalization_delay: u64, config: &DatabaseSettings, ) -> ClientResult { - let is_archive_pruning = config.pruning.is_archive(); + let is_archive_pruning = config.state_pruning.is_archive(); let blockchain = BlockchainDb::new(db.clone())?; let meta = blockchain.meta.clone(); let map_e = |e: sc_state_db::Error| sp_blockchain::Error::from_state_db(e); let state_db: StateDb<_, _> = StateDb::new( - config.pruning.clone(), + config.state_pruning.clone(), !config.source.supports_ref_counting(), &StateMetaDb(&*db), ).map_err(map_e)?; @@ -1882,8 +1909,10 @@ pub(crate) mod tests { let backend = Backend::::new(DatabaseSettings { state_cache_size: 16777216, state_cache_child_ratio: Some((50, 100)), - pruning: PruningMode::keep_blocks(1), + state_pruning: PruningMode::keep_blocks(1), source: DatabaseSettingsSrc::Custom(backing), + keep_blocks: KeepBlocks::All, + transaction_storage: TransactionStorage::BlockBody, }, 0).unwrap(); assert_eq!(backend.blockchain().info().best_number, 9); for i in 0..10 { diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index e87b11b69660c..7979b11c4e4a7 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -24,21 +24,26 @@ use std::path::{Path, PathBuf}; use sp_runtime::traits::Block as BlockT; use crate::utils::DatabaseType; +use kvdb_rocksdb::{Database, DatabaseConfig}; /// Version file name. const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. -const CURRENT_VERSION: u32 = 1; +const CURRENT_VERSION: u32 = 2; + +/// Number of columns in v1. +const V1_NUM_COLUMNS: u32 = 11; /// Upgrade database to current version. -pub fn upgrade_db(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { +pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> { let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none()); if !is_empty { let db_version = current_version(db_path)?; match db_version { 0 => Err(sp_blockchain::Error::Backend(format!("Unsupported database version: {}", db_version)))?, - 1 => (), + 1 => migrate_1_to_2::(db_path, db_type)?, + CURRENT_VERSION => (), _ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?, } } @@ -46,6 +51,19 @@ pub fn upgrade_db(db_path: &Path, _db_type: DatabaseType) -> sp_b update_version(db_path) } +/// Migration from version1 to version2: +/// 1) the number of columns has changed from 11 to 12; +/// 2) transactions column is added; +fn migrate_1_to_2(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { + { + let db_path = db_path.to_str() + .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; + let db_cfg = DatabaseConfig::with_columns(V1_NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).map_err(db_err)?; + db.add_column().map_err(db_err)?; + } + Ok(()) +} /// Reads current database version from the file at given path. /// If the file does not exist returns 0. @@ -87,7 +105,7 @@ fn version_file_path(path: &Path) -> PathBuf { #[cfg(test)] mod tests { use sc_state_db::PruningMode; - use crate::{DatabaseSettings, DatabaseSettingsSrc}; + use crate::{DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorage}; use crate::tests::Block; use super::*; @@ -103,8 +121,10 @@ mod tests { crate::utils::open_database::(&DatabaseSettings { state_cache_size: 0, state_cache_child_ratio: None, - pruning: PruningMode::ArchiveAll, + state_pruning: PruningMode::ArchiveAll, source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 }, + keep_blocks: KeepBlocks::All, + transaction_storage: TransactionStorage::BlockBody, }, DatabaseType::Full).map(|_| ()) } @@ -122,4 +142,15 @@ mod tests { open_database(db_dir.path()).unwrap(); assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION); } + + #[test] + fn upgrade_from_1_to_2_works() { + for version_from_file in &[None, Some(1)] { + let db_dir = tempfile::TempDir::new().unwrap(); + let db_path = db_dir.path(); + create_db(db_path, *version_from_file); + open_database(db_path).unwrap(); + assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION); + } + } } diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index dfc1e945b3a4c..a1021d6581be7 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -37,7 +37,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc, Database, DbHash}; /// Number of columns in the db. Must be the same for both full && light dbs. /// Otherwise RocksDb will fail to open database && check its type. #[cfg(any(feature = "with-kvdb-rocksdb", feature = "with-parity-db", feature = "test-helpers", test))] -pub const NUM_COLUMNS: u32 = 11; +pub const NUM_COLUMNS: u32 = 12; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: u32 = 0; diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index e3476e625ca55..bac5191f0ccae 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -326,8 +326,10 @@ pub fn new_full_parts( state_cache_size: config.state_cache_size, state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), - pruning: config.pruning.clone(), + state_pruning: config.state_pruning.clone(), source: config.database.clone(), + keep_blocks: config.keep_blocks.clone(), + transaction_storage: config.transaction_storage.clone(), }; let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( @@ -384,8 +386,10 @@ pub fn new_light_parts( state_cache_size: config.state_cache_size, state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), - pruning: config.pruning.clone(), + state_pruning: config.state_pruning.clone(), source: config.database.clone(), + keep_blocks: config.keep_blocks.clone(), + transaction_storage: config.transaction_storage.clone(), }; sc_client_db::light::LightStorage::new(db_settings)? }; diff --git a/client/service/src/config.rs b/client/service/src/config.rs index e253ed97ff3a5..c63753b836dac 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -18,7 +18,10 @@ //! Service configuration. -pub use sc_client_db::{Database, PruningMode, DatabaseSettingsSrc as DatabaseConfig}; +pub use sc_client_db::{ + Database, PruningMode, DatabaseSettingsSrc as DatabaseConfig, + KeepBlocks, TransactionStorage +}; pub use sc_network::Multiaddr; pub use sc_network::config::{ExtTransport, MultiaddrWithPeerId, NetworkConfiguration, Role, NodeKeyConfig}; pub use sc_executor::WasmExecutionMethod; @@ -58,8 +61,12 @@ pub struct Configuration { pub state_cache_size: usize, /// Size in percent of cache size dedicated to child tries pub state_cache_child_ratio: Option, - /// Pruning settings. - pub pruning: PruningMode, + /// State pruning settings. + pub state_pruning: PruningMode, + /// Block pruning settings. + pub keep_blocks: KeepBlocks, + /// Transaction storage scheme. + pub transaction_storage: TransactionStorage, /// Chain configuration. pub chain_spec: Box, /// Wasm execution method. diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 8b26b1a75ddf8..3cb8aacddbb42 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -60,6 +60,7 @@ pub use self::builder::{ }; pub use config::{ BasePath, Configuration, DatabaseConfig, PruningMode, Role, RpcMethods, TaskExecutor, TaskType, + KeepBlocks, TransactionStorage, }; pub use sc_chain_spec::{ ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 6bb09981107a0..a5e84fa338562 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -31,7 +31,9 @@ use substrate_test_runtime_client::{ use sc_client_api::{ StorageProvider, BlockBackend, in_mem, BlockchainEvents, }; -use sc_client_db::{Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode}; +use sc_client_db::{ + Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode, KeepBlocks, TransactionStorage +}; use sc_block_builder::BlockBuilderProvider; use sc_service::client::{self, Client, LocalCallExecutor, new_in_mem}; use sp_runtime::traits::{ @@ -1275,7 +1277,9 @@ fn doesnt_import_blocks_that_revert_finality() { DatabaseSettings { state_cache_size: 1 << 20, state_cache_child_ratio: None, - pruning: PruningMode::ArchiveAll, + state_pruning: PruningMode::ArchiveAll, + keep_blocks: KeepBlocks::All, + transaction_storage: TransactionStorage::BlockBody, source: DatabaseSettingsSrc::RocksDb { path: tmp.path().into(), cache_size: 1024, @@ -1476,7 +1480,9 @@ fn returns_status_for_pruned_blocks() { DatabaseSettings { state_cache_size: 1 << 20, state_cache_child_ratio: None, - pruning: PruningMode::keep_blocks(1), + state_pruning: PruningMode::keep_blocks(1), + keep_blocks: KeepBlocks::All, + transaction_storage: TransactionStorage::BlockBody, source: DatabaseSettingsSrc::RocksDb { path: tmp.path().into(), cache_size: 1024, diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index c30246e91ca05..134990b972f59 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -35,6 +35,7 @@ use sc_service::{ GenericChainSpec, ChainSpecExtension, Configuration, + KeepBlocks, TransactionStorage, config::{BasePath, DatabaseConfig, KeystoreConfig}, RuntimeGenesis, Role, @@ -250,7 +251,9 @@ fn node_config Date: Sun, 10 Jan 2021 22:50:10 +0300 Subject: [PATCH 2/9] Transaction storage --- client/db/src/lib.rs | 75 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 2aa8c914a79fa..1c5544a4994f0 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -66,7 +66,7 @@ use codec::{Decode, Encode}; use hash_db::Prefix; use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key}; use sp_database::Transaction; -use sp_core::ChangesTrieConfiguration; +use sp_core::{Hasher, ChangesTrieConfiguration}; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; @@ -397,10 +397,14 @@ pub struct BlockchainDb { leaves: RwLock>>, header_metadata_cache: Arc>, header_cache: Mutex>>, + transaction_storage: TransactionStorage, } impl BlockchainDb { - fn new(db: Arc>) -> ClientResult { + fn new( + db: Arc>, + transaction_storage: TransactionStorage + ) -> ClientResult { let meta = read_meta::(&*db, columns::HEADER)?; let leaves = LeafSet::read_from_db(&*db, columns::META, meta_keys::LEAF_PREFIX)?; Ok(BlockchainDb { @@ -409,6 +413,7 @@ impl BlockchainDb { meta: Arc::new(RwLock::new(meta)), header_metadata_cache: Arc::new(HeaderMetadataCache::default()), header_cache: Default::default(), + transaction_storage, }) } @@ -443,6 +448,20 @@ impl BlockchainDb { header.digest().log(DigestItem::as_changes_trie_root) .cloned())) } + + fn extrinsic(&self, hash: &Block::Hash) -> ClientResult> { + match self.db.get(columns::TRANSACTION, hash.as_ref()) { + Some(ex) => { + match Decode::decode(&mut &ex[..]) { + Ok(ex) => Ok(Some(ex)), + Err(err) => Err(sp_blockchain::Error::Backend( + format!("Error decoding extrinsic {}: {}", hash, err) + )), + } + }, + None => Ok(None), + } + } } impl sc_client_api::blockchain::HeaderBackend for BlockchainDb { @@ -501,11 +520,31 @@ impl sc_client_api::blockchain::HeaderBackend for Blockcha impl sc_client_api::blockchain::Backend for BlockchainDb { fn body(&self, id: BlockId) -> ClientResult>> { match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { - Some(body) => match Decode::decode(&mut &body[..]) { - Ok(body) => Ok(Some(body)), - Err(err) => return Err(sp_blockchain::Error::Backend( - format!("Error decoding body: {}", err) - )), + Some(body) => { + match self.transaction_storage { + TransactionStorage::BlockBody => match Decode::decode(&mut &body[..]) { + Ok(body) => Ok(Some(body)), + Err(err) => return Err(sp_blockchain::Error::Backend( + format!("Error decoding body: {}", err) + )), + }, + TransactionStorage::StorageChain => { + match Vec::::decode(&mut &body[..]) { + Ok(hashes) => { + let extrinsics: ClientResult> = hashes.into_iter().map( + |h| self.extrinsic(&h) + .and_then(|maybe_ex| maybe_ex.ok_or_else( + || sp_blockchain::Error::Backend( + format!("Missing transaction: {}", h)))) + ).collect(); + Ok(Some(extrinsics?)) + } + Err(err) => return Err(sp_blockchain::Error::Backend( + format!("Error decoding body list: {}", err) + )), + } + } + } } None => Ok(None), } @@ -880,6 +919,8 @@ pub struct Backend { shared_cache: SharedCache, import_lock: Arc>, is_archive: bool, + keep_blocks: KeepBlocks, + transaction_storage: TransactionStorage, io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, } @@ -916,7 +957,7 @@ impl Backend { config: &DatabaseSettings, ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); - let blockchain = BlockchainDb::new(db.clone())?; + let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; let meta = blockchain.meta.clone(); let map_e = |e: sc_state_db::Error| sp_blockchain::Error::from_state_db(e); let state_db: StateDb<_, _> = StateDb::new( @@ -960,6 +1001,8 @@ impl Backend { is_archive: is_archive_pruning, io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), state_usage: Arc::new(StateUsageStats::new()), + keep_blocks: config.keep_blocks.clone(), + transaction_storage: config.transaction_storage.clone(), }) } @@ -1167,7 +1210,21 @@ impl Backend { transaction.set_from_vec(columns::HEADER, &lookup_key, pending_block.header.encode()); if let Some(body) = &pending_block.body { - transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); + match self.transaction_storage { + TransactionStorage::BlockBody => { + transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); + }, + TransactionStorage::StorageChain => { + let mut hashes = Vec::with_capacity(body.len()); + for extrinsic in body { + let extrinsic = extrinsic.encode(); + let hash = HashFor::::hash(&extrinsic); + transaction.set(columns::TRANSACTION, &hash.as_ref(), &extrinsic); + hashes.push(hash); + } + transaction.set_from_vec(columns::BODY, &lookup_key, hashes.encode()); + }, + } } if let Some(justification) = pending_block.justification { transaction.set_from_vec(columns::JUSTIFICATION, &lookup_key, justification.encode()); From 8648239ee57c051b7f34d164bc7e2773fc0446c6 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2021 23:34:41 +0300 Subject: [PATCH 3/9] Block pruning --- client/db/src/lib.rs | 49 ++++++++++++++++++++++++++++++++++++++++++ client/db/src/utils.rs | 17 +++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 1c5544a4994f0..83008080e8c95 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -95,6 +95,8 @@ pub use bench::BenchmarkingState; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; const CACHE_HEADERS: usize = 8; +// Max blocks to prune at once +const MAX_BLOCK_PRUNE: usize = 1024; /// Default value for storage cache child ratio. const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); @@ -1475,6 +1477,7 @@ impl Backend { } } + self.prune_blocks(transaction, f_num)?; let new_displaced = self.blockchain.leaves.write().finalize_height(f_num); match displaced { x @ &mut None => *x = Some(new_displaced), @@ -1483,6 +1486,52 @@ impl Backend { Ok(()) } + + fn prune_blocks( + &self, + transaction: &mut Transaction, + finalized: NumberFor, + ) -> ClientResult<()> { + if let KeepBlocks::Some(keep_blocks) = self.keep_blocks { + let mut removed_blocks = 0; + let mut removed_transactions = 0; + let mut number = finalized.saturating_sub(keep_blocks.into()); + while number > One::one() && removed_blocks < MAX_BLOCK_PRUNE { + number = number.saturating_sub(One::one()); + match read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY, BlockId::::number(number))? { + Some(body) => { + removed_blocks += 1; + utils::remove_db( + transaction, + &*self.storage.db, + columns::KEY_LOOKUP, + columns::BODY, + BlockId::::number(number) + )?; + match self.transaction_storage { + TransactionStorage::BlockBody => {}, + TransactionStorage::StorageChain => { + match Vec::::decode(&mut &body[..]) { + Ok(hashes) => { + removed_transactions += hashes.len(); + for h in hashes { + transaction.remove(columns::TRANSACTION, h.as_ref()); + } + } + Err(err) => return Err(sp_blockchain::Error::Backend( + format!("Error decoding body list: {}", err) + )), + } + } + } + } + None => break, + } + } + debug!(target: "db", "Pruned {} blocks, {} transactions", removed_blocks, removed_transactions); + } + Ok(()) + } } fn apply_state_commit(transaction: &mut Transaction, commit: sc_state_db::CommitSet>) { diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index a1021d6581be7..87f62fc85fd6e 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -327,6 +327,23 @@ pub fn read_db( }) } +/// Remove database column entry for the given block. +pub fn remove_db( + transaction: &mut Transaction, + db: &dyn Database, + col_index: u32, + col: u32, + id: BlockId +) -> sp_blockchain::Result<()> +where + Block: BlockT, +{ + block_id_to_lookup_key(db, col_index, id).and_then(|key| match key { + Some(key) => Ok(transaction.remove(col, key.as_ref())), + None => Ok(()), + }) +} + /// Read a header from the database. pub fn read_header( db: &dyn Database, From 2b974688fd209d8a899871cc05826f52a2995747 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 11 Jan 2021 13:50:42 +0300 Subject: [PATCH 4/9] Block pruning test --- client/db/src/lib.rs | 81 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 16 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 83008080e8c95..d7d9a3952764d 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -95,8 +95,6 @@ pub use bench::BenchmarkingState; const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; const CACHE_HEADERS: usize = 8; -// Max blocks to prune at once -const MAX_BLOCK_PRUNE: usize = 1024; /// Default value for storage cache child ratio. const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); @@ -277,7 +275,7 @@ pub struct DatabaseSettings { } /// Block pruning settings. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum KeepBlocks { /// Keep full block history. All, @@ -286,7 +284,7 @@ pub enum KeepBlocks { } /// Block body storage scheme. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub enum TransactionStorage { /// Store block body as deeply encoded list of transactions in the BODY column BlockBody, @@ -939,6 +937,20 @@ impl Backend { /// Create new memory-backed client backend for tests. #[cfg(any(test, feature = "test-helpers"))] pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self { + Self::new_test_with_tx_storage( + keep_blocks, + canonicalization_delay, + TransactionStorage::BlockBody + ) + } + + /// Create new memory-backed client backend for tests. + #[cfg(any(test, feature = "test-helpers"))] + fn new_test_with_tx_storage( + keep_blocks: u32, + canonicalization_delay: u64, + transaction_storage: TransactionStorage + ) -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); let db_setting = DatabaseSettings { @@ -946,8 +958,8 @@ impl Backend { state_cache_child_ratio: Some((50, 100)), state_pruning: PruningMode::keep_blocks(keep_blocks), source: DatabaseSettingsSrc::Custom(db), - keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorage::BlockBody, + keep_blocks: KeepBlocks::Some(keep_blocks), + transaction_storage, }; Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") @@ -1493,14 +1505,12 @@ impl Backend { finalized: NumberFor, ) -> ClientResult<()> { if let KeepBlocks::Some(keep_blocks) = self.keep_blocks { - let mut removed_blocks = 0; - let mut removed_transactions = 0; - let mut number = finalized.saturating_sub(keep_blocks.into()); - while number > One::one() && removed_blocks < MAX_BLOCK_PRUNE { - number = number.saturating_sub(One::one()); + // Always keep the last finalized block + let keep = std::cmp::max(keep_blocks, 1); + if finalized >= keep.into() { + let number = finalized.saturating_sub(keep.into()); match read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY, BlockId::::number(number))? { Some(body) => { - removed_blocks += 1; utils::remove_db( transaction, &*self.storage.db, @@ -1508,12 +1518,12 @@ impl Backend { columns::BODY, BlockId::::number(number) )?; + debug!(target: "db", "Removing block #{}", number); match self.transaction_storage { TransactionStorage::BlockBody => {}, TransactionStorage::StorageChain => { match Vec::::decode(&mut &body[..]) { Ok(hashes) => { - removed_transactions += hashes.len(); for h in hashes { transaction.remove(columns::TRANSACTION, h.as_ref()); } @@ -1525,10 +1535,9 @@ impl Backend { } } } - None => break, + None => return Ok(()), } } - debug!(target: "db", "Pruned {} blocks, {} transactions", removed_blocks, removed_transactions); } Ok(()) } @@ -1937,6 +1946,17 @@ pub(crate) mod tests { parent_hash: H256, changes: Option, Vec)>>, extrinsics_root: H256, + ) -> H256 { + insert_block(backend, number, parent_hash, changes, extrinsics_root, Vec::new()) + } + + pub fn insert_block( + backend: &Backend, + number: u64, + parent_hash: H256, + changes: Option, Vec)>>, + extrinsics_root: H256, + body: Vec>, ) -> H256 { use sp_runtime::testing::Digest; @@ -1963,7 +1983,7 @@ pub(crate) mod tests { }; let mut op = backend.begin_operation().unwrap(); backend.begin_state_operation(&mut op, block_id).unwrap(); - op.set_block_data(header, Some(Vec::new()), None, NewBlockState::Best).unwrap(); + op.set_block_data(header, Some(body), None, NewBlockState::Best).unwrap(); op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)).unwrap(); backend.commit_operation(op).unwrap(); @@ -2562,4 +2582,33 @@ pub(crate) mod tests { assert_eq!(cht_root_1, cht_root_2); assert_eq!(cht_root_2, cht_root_3); } + + #[test] + fn prune_blocks_on_finalize() { + for storage in &[TransactionStorage::BlockBody, TransactionStorage::StorageChain] { + let backend = Backend::::new_test_with_tx_storage(2, 0, *storage); + let mut blocks = Vec::new(); + let mut prev_hash = Default::default(); + for i in 0 .. 5 { + let hash = insert_block(&backend, i, prev_hash, None, Default::default(), vec![i.into()]); + blocks.push(hash); + prev_hash = hash; + } + + { + let mut op = backend.begin_operation().unwrap(); + backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap(); + for i in 1 .. 5 { + op.mark_finalized(BlockId::Hash(blocks[i]), None).unwrap(); + } + backend.commit_operation(op).unwrap(); + } + let bc = backend.blockchain(); + assert_eq!(None, bc.body(BlockId::hash(blocks[0])).unwrap()); + assert_eq!(None, bc.body(BlockId::hash(blocks[1])).unwrap()); + assert_eq!(None, bc.body(BlockId::hash(blocks[2])).unwrap()); + assert_eq!(Some(vec![3.into()]), bc.body(BlockId::hash(blocks[3])).unwrap()); + assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap()); + } + } } From 65a2708ebf01493fa01c193540c3b85d0f17cb2c Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 12 Jan 2021 14:43:28 +0300 Subject: [PATCH 5/9] Style --- client/db/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d7d9a3952764d..9083c807be3d4 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -455,7 +455,7 @@ impl BlockchainDb { match Decode::decode(&mut &ex[..]) { Ok(ex) => Ok(Some(ex)), Err(err) => Err(sp_blockchain::Error::Backend( - format!("Error decoding extrinsic {}: {}", hash, err) + format!("Error decoding extrinsic {}: {}", hash, err) )), } }, @@ -525,7 +525,7 @@ impl sc_client_api::blockchain::Backend for BlockchainDb match Decode::decode(&mut &body[..]) { Ok(body) => Ok(Some(body)), Err(err) => return Err(sp_blockchain::Error::Backend( - format!("Error decoding body: {}", err) + format!("Error decoding body: {}", err) )), }, TransactionStorage::StorageChain => { @@ -1529,7 +1529,7 @@ impl Backend { } } Err(err) => return Err(sp_blockchain::Error::Backend( - format!("Error decoding body list: {}", err) + format!("Error decoding body list: {}", err) )), } } From fca526bdf53cbe9d01fbdb82c745a8515e97adf1 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 13 Jan 2021 13:18:50 +0300 Subject: [PATCH 6/9] Naming --- bin/node/testing/src/bench.rs | 2 +- client/cli/src/config.rs | 6 ++--- client/cli/src/params/database_params.rs | 8 +++---- client/db/src/lib.rs | 30 ++++++++++++------------ client/db/src/upgrade.rs | 4 ++-- client/service/src/config.rs | 4 ++-- client/service/src/lib.rs | 2 +- client/service/test/src/client/mod.rs | 6 ++--- client/service/test/src/lib.rs | 4 ++-- utils/browser/src/lib.rs | 4 ++-- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index 3b227f5e949dd..a6f65b86a0e27 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -413,7 +413,7 @@ impl BenchDb { state_pruning: PruningMode::ArchiveAll, source: database_type.into_settings(dir.into()), keep_blocks: sc_client_db::KeepBlocks::All, - transaction_storage: sc_client_db::TransactionStorage::BlockBody, + transaction_storage: sc_client_db::TransactionStorageMode::BlockBody, }; let task_executor = TaskExecutor::new(); diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index db69aa4fff126..a9ea16dab3634 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -32,7 +32,7 @@ use sc_service::config::{ NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods, TaskExecutor, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, }; -use sc_service::{ChainSpec, TracingReceiver, KeepBlocks, TransactionStorage }; +use sc_service::{ChainSpec, TracingReceiver, KeepBlocks, TransactionStorageMode }; use std::net::SocketAddr; use std::path::PathBuf; @@ -204,10 +204,10 @@ pub trait CliConfiguration: Sized { } /// Get the database transaction storage scheme. - fn database_transaction_storage(&self) -> Result { + fn database_transaction_storage(&self) -> Result { Ok(self.database_params() .map(|x| x.transaction_storage()) - .unwrap_or(TransactionStorage::BlockBody)) + .unwrap_or(TransactionStorageMode::BlockBody)) } /// Get the database backend variant. diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index 8718f2f45e609..14f6643be3b46 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -18,7 +18,7 @@ use crate::arg_enums::Database; use structopt::StructOpt; -use sc_service::TransactionStorage; +use sc_service::TransactionStorageMode; /// Parameters for block import. #[derive(Debug, StructOpt)] @@ -53,11 +53,11 @@ impl DatabaseParams { } /// Transaction storage scheme. - pub fn transaction_storage(&self) -> TransactionStorage { + pub fn transaction_storage(&self) -> TransactionStorageMode { if self.storage_chain { - TransactionStorage::StorageChain + TransactionStorageMode::StorageChain } else { - TransactionStorage::BlockBody + TransactionStorageMode::BlockBody } } } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 9083c807be3d4..392c23c6ba9ec 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -271,7 +271,7 @@ pub struct DatabaseSettings { /// Block pruning mode. pub keep_blocks: KeepBlocks, /// Block body/Transaction storage scheme. - pub transaction_storage: TransactionStorage, + pub transaction_storage: TransactionStorageMode, } /// Block pruning settings. @@ -285,7 +285,7 @@ pub enum KeepBlocks { /// Block body storage scheme. #[derive(Debug, Clone, Copy)] -pub enum TransactionStorage { +pub enum TransactionStorageMode { /// Store block body as deeply encoded list of transactions in the BODY column BlockBody, /// Store a list of hashes in the BODY column and each transaction individually @@ -397,13 +397,13 @@ pub struct BlockchainDb { leaves: RwLock>>, header_metadata_cache: Arc>, header_cache: Mutex>>, - transaction_storage: TransactionStorage, + transaction_storage: TransactionStorageMode, } impl BlockchainDb { fn new( db: Arc>, - transaction_storage: TransactionStorage + transaction_storage: TransactionStorageMode ) -> ClientResult { let meta = read_meta::(&*db, columns::HEADER)?; let leaves = LeafSet::read_from_db(&*db, columns::META, meta_keys::LEAF_PREFIX)?; @@ -522,13 +522,13 @@ impl sc_client_api::blockchain::Backend for BlockchainDb { match self.transaction_storage { - TransactionStorage::BlockBody => match Decode::decode(&mut &body[..]) { + TransactionStorageMode::BlockBody => match Decode::decode(&mut &body[..]) { Ok(body) => Ok(Some(body)), Err(err) => return Err(sp_blockchain::Error::Backend( format!("Error decoding body: {}", err) )), }, - TransactionStorage::StorageChain => { + TransactionStorageMode::StorageChain => { match Vec::::decode(&mut &body[..]) { Ok(hashes) => { let extrinsics: ClientResult> = hashes.into_iter().map( @@ -920,7 +920,7 @@ pub struct Backend { import_lock: Arc>, is_archive: bool, keep_blocks: KeepBlocks, - transaction_storage: TransactionStorage, + transaction_storage: TransactionStorageMode, io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, } @@ -940,7 +940,7 @@ impl Backend { Self::new_test_with_tx_storage( keep_blocks, canonicalization_delay, - TransactionStorage::BlockBody + TransactionStorageMode::BlockBody ) } @@ -949,7 +949,7 @@ impl Backend { fn new_test_with_tx_storage( keep_blocks: u32, canonicalization_delay: u64, - transaction_storage: TransactionStorage + transaction_storage: TransactionStorageMode ) -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); @@ -1225,10 +1225,10 @@ impl Backend { transaction.set_from_vec(columns::HEADER, &lookup_key, pending_block.header.encode()); if let Some(body) = &pending_block.body { match self.transaction_storage { - TransactionStorage::BlockBody => { + TransactionStorageMode::BlockBody => { transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); }, - TransactionStorage::StorageChain => { + TransactionStorageMode::StorageChain => { let mut hashes = Vec::with_capacity(body.len()); for extrinsic in body { let extrinsic = extrinsic.encode(); @@ -1520,8 +1520,8 @@ impl Backend { )?; debug!(target: "db", "Removing block #{}", number); match self.transaction_storage { - TransactionStorage::BlockBody => {}, - TransactionStorage::StorageChain => { + TransactionStorageMode::BlockBody => {}, + TransactionStorageMode::StorageChain => { match Vec::::decode(&mut &body[..]) { Ok(hashes) => { for h in hashes { @@ -2038,7 +2038,7 @@ pub(crate) mod tests { state_pruning: PruningMode::keep_blocks(1), source: DatabaseSettingsSrc::Custom(backing), keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorage::BlockBody, + transaction_storage: TransactionStorageMode::BlockBody, }, 0).unwrap(); assert_eq!(backend.blockchain().info().best_number, 9); for i in 0..10 { @@ -2585,7 +2585,7 @@ pub(crate) mod tests { #[test] fn prune_blocks_on_finalize() { - for storage in &[TransactionStorage::BlockBody, TransactionStorage::StorageChain] { + for storage in &[TransactionStorageMode::BlockBody, TransactionStorageMode::StorageChain] { let backend = Backend::::new_test_with_tx_storage(2, 0, *storage); let mut blocks = Vec::new(); let mut prev_hash = Default::default(); diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 7979b11c4e4a7..fcc53c3462c48 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -105,7 +105,7 @@ fn version_file_path(path: &Path) -> PathBuf { #[cfg(test)] mod tests { use sc_state_db::PruningMode; - use crate::{DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorage}; + use crate::{DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorageMode}; use crate::tests::Block; use super::*; @@ -124,7 +124,7 @@ mod tests { state_pruning: PruningMode::ArchiveAll, source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 }, keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorage::BlockBody, + transaction_storage: TransactionStorageMode::BlockBody, }, DatabaseType::Full).map(|_| ()) } diff --git a/client/service/src/config.rs b/client/service/src/config.rs index c63753b836dac..75c8cb3cdcaf7 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -20,7 +20,7 @@ pub use sc_client_db::{ Database, PruningMode, DatabaseSettingsSrc as DatabaseConfig, - KeepBlocks, TransactionStorage + KeepBlocks, TransactionStorageMode }; pub use sc_network::Multiaddr; pub use sc_network::config::{ExtTransport, MultiaddrWithPeerId, NetworkConfiguration, Role, NodeKeyConfig}; @@ -66,7 +66,7 @@ pub struct Configuration { /// Block pruning settings. pub keep_blocks: KeepBlocks, /// Transaction storage scheme. - pub transaction_storage: TransactionStorage, + pub transaction_storage: TransactionStorageMode, /// Chain configuration. pub chain_spec: Box, /// Wasm execution method. diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 3cb8aacddbb42..df1cd47db0f7e 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -60,7 +60,7 @@ pub use self::builder::{ }; pub use config::{ BasePath, Configuration, DatabaseConfig, PruningMode, Role, RpcMethods, TaskExecutor, TaskType, - KeepBlocks, TransactionStorage, + KeepBlocks, TransactionStorageMode, }; pub use sc_chain_spec::{ ChainSpec, GenericChainSpec, Properties, RuntimeGenesis, Extension as ChainSpecExtension, diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index a5e84fa338562..9712e84e60496 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -32,7 +32,7 @@ use sc_client_api::{ StorageProvider, BlockBackend, in_mem, BlockchainEvents, }; use sc_client_db::{ - Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode, KeepBlocks, TransactionStorage + Backend, DatabaseSettings, DatabaseSettingsSrc, PruningMode, KeepBlocks, TransactionStorageMode }; use sc_block_builder::BlockBuilderProvider; use sc_service::client::{self, Client, LocalCallExecutor, new_in_mem}; @@ -1279,7 +1279,7 @@ fn doesnt_import_blocks_that_revert_finality() { state_cache_child_ratio: None, state_pruning: PruningMode::ArchiveAll, keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorage::BlockBody, + transaction_storage: TransactionStorageMode::BlockBody, source: DatabaseSettingsSrc::RocksDb { path: tmp.path().into(), cache_size: 1024, @@ -1482,7 +1482,7 @@ fn returns_status_for_pruned_blocks() { state_cache_child_ratio: None, state_pruning: PruningMode::keep_blocks(1), keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorage::BlockBody, + transaction_storage: TransactionStorageMode::BlockBody, source: DatabaseSettingsSrc::RocksDb { path: tmp.path().into(), cache_size: 1024, diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 134990b972f59..bd4f325908b0a 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -35,7 +35,7 @@ use sc_service::{ GenericChainSpec, ChainSpecExtension, Configuration, - KeepBlocks, TransactionStorage, + KeepBlocks, TransactionStorageMode, config::{BasePath, DatabaseConfig, KeystoreConfig}, RuntimeGenesis, Role, @@ -253,7 +253,7 @@ fn node_config Date: Thu, 14 Jan 2021 17:32:56 +0300 Subject: [PATCH 7/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- client/cli/src/params/database_params.rs | 7 ++++++- client/db/src/lib.rs | 6 +++--- client/db/src/upgrade.rs | 13 +++++-------- client/db/src/utils.rs | 2 +- client/service/src/config.rs | 2 +- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index 14f6643be3b46..23d2adc07f9de 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -37,7 +37,12 @@ pub struct DatabaseParams { pub database_cache_size: Option, /// Enable storage chain mode - #[structopt(long = "storage-chain")] + /// + /// This changes the storage format for blocks bodys. + /// If this is enabled, each transaction is stored separately in the + /// transaction database column and is only referenced by hash + /// in the block body column. + #[structopt(long)] pub storage_chain: bool, } diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 392c23c6ba9ec..47317681e37ce 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -940,7 +940,7 @@ impl Backend { Self::new_test_with_tx_storage( keep_blocks, canonicalization_delay, - TransactionStorageMode::BlockBody + TransactionStorageMode::BlockBody, ) } @@ -949,7 +949,7 @@ impl Backend { fn new_test_with_tx_storage( keep_blocks: u32, canonicalization_delay: u64, - transaction_storage: TransactionStorageMode + transaction_storage: TransactionStorageMode, ) -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); @@ -1516,7 +1516,7 @@ impl Backend { &*self.storage.db, columns::KEY_LOOKUP, columns::BODY, - BlockId::::number(number) + BlockId::::number(number), )?; debug!(target: "db", "Removing block #{}", number); match self.transaction_storage { diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index fcc53c3462c48..b6e49edba1978 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -55,14 +55,11 @@ pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> sp_bl /// 1) the number of columns has changed from 11 to 12; /// 2) transactions column is added; fn migrate_1_to_2(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> { - { - let db_path = db_path.to_str() - .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; - let db_cfg = DatabaseConfig::with_columns(V1_NUM_COLUMNS); - let db = Database::open(&db_cfg, db_path).map_err(db_err)?; - db.add_column().map_err(db_err)?; - } - Ok(()) + let db_path = db_path.to_str() + .ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?; + let db_cfg = DatabaseConfig::with_columns(V1_NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path).map_err(db_err)?; + db.add_column().map_err(db_err) } /// Reads current database version from the file at given path. diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index 87f62fc85fd6e..4a4c68b6c2a2f 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -333,7 +333,7 @@ pub fn remove_db( db: &dyn Database, col_index: u32, col: u32, - id: BlockId + id: BlockId, ) -> sp_blockchain::Result<()> where Block: BlockT, diff --git a/client/service/src/config.rs b/client/service/src/config.rs index 75c8cb3cdcaf7..5197aa655b249 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -63,7 +63,7 @@ pub struct Configuration { pub state_cache_child_ratio: Option, /// State pruning settings. pub state_pruning: PruningMode, - /// Block pruning settings. + /// Number of blocks to keep in the db. pub keep_blocks: KeepBlocks, /// Transaction storage scheme. pub transaction_storage: TransactionStorageMode, From 3e90eedce4e42ac64d7018d041b32e064009aeec Mon Sep 17 00:00:00 2001 From: Arkadiy Paronyan Date: Thu, 14 Jan 2021 17:38:16 +0300 Subject: [PATCH 8/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- client/cli/src/params/pruning_params.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index 29d4b82feb58b..0351af92e07fe 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -33,7 +33,7 @@ pub struct PruningParams { /// Specify the number of finalized blocks to keep in the database. /// /// Default is to keep all blocks. - #[structopt(long = "block-pruning", value_name = "COUNT")] + #[structopt(long, value_name = "COUNT")] pub block_pruning: Option, } From 95ec11e0e021c0774cf3b0f14ef6b0f3c34c4fc4 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 14 Jan 2021 17:45:42 +0300 Subject: [PATCH 9/9] Style --- client/cli/src/config.rs | 2 +- client/cli/src/params/pruning_params.rs | 6 +-- client/db/src/lib.rs | 54 ++++++++++++------------- client/db/src/utils.rs | 2 +- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index a9ea16dab3634..854e73ae78125 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -267,7 +267,7 @@ pub trait CliConfiguration: Sized { /// `KeepBlocks::All`. fn keep_blocks(&self) -> Result { self.pruning_params() - .map(|x| x.block_pruning()) + .map(|x| x.keep_blocks()) .unwrap_or_else(|| Ok(KeepBlocks::All)) } diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index 0351af92e07fe..467ca253531f1 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -34,7 +34,7 @@ pub struct PruningParams { /// /// Default is to keep all blocks. #[structopt(long, value_name = "COUNT")] - pub block_pruning: Option, + pub keep_blocks: Option, } impl PruningParams { @@ -65,8 +65,8 @@ impl PruningParams { } /// Get the block pruning value from the parameters - pub fn block_pruning(&self) -> error::Result { - Ok(match self.block_pruning { + pub fn keep_blocks(&self) -> error::Result { + Ok(match self.keep_blocks { Some(n) => KeepBlocks::Some(n), None => KeepBlocks::All, }) diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 47317681e37ce..3fc95d5cdf975 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -286,7 +286,7 @@ pub enum KeepBlocks { /// Block body storage scheme. #[derive(Debug, Clone, Copy)] pub enum TransactionStorageMode { - /// Store block body as deeply encoded list of transactions in the BODY column + /// Store block body as an encoded list of full transactions in the BODY column BlockBody, /// Store a list of hashes in the BODY column and each transaction individually /// in the TRANSACTION column. @@ -532,8 +532,7 @@ impl sc_client_api::blockchain::Backend for BlockchainDb::decode(&mut &body[..]) { Ok(hashes) => { let extrinsics: ClientResult> = hashes.into_iter().map( - |h| self.extrinsic(&h) - .and_then(|maybe_ex| maybe_ex.ok_or_else( + |h| self.extrinsic(&h) .and_then(|maybe_ex| maybe_ex.ok_or_else( || sp_blockchain::Error::Backend( format!("Missing transaction: {}", h)))) ).collect(); @@ -1507,36 +1506,37 @@ impl Backend { if let KeepBlocks::Some(keep_blocks) = self.keep_blocks { // Always keep the last finalized block let keep = std::cmp::max(keep_blocks, 1); - if finalized >= keep.into() { - let number = finalized.saturating_sub(keep.into()); - match read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY, BlockId::::number(number))? { - Some(body) => { - utils::remove_db( - transaction, - &*self.storage.db, - columns::KEY_LOOKUP, - columns::BODY, - BlockId::::number(number), - )?; - debug!(target: "db", "Removing block #{}", number); - match self.transaction_storage { - TransactionStorageMode::BlockBody => {}, - TransactionStorageMode::StorageChain => { - match Vec::::decode(&mut &body[..]) { - Ok(hashes) => { - for h in hashes { - transaction.remove(columns::TRANSACTION, h.as_ref()); - } + if finalized < keep.into() { + return Ok(()) + } + let number = finalized.saturating_sub(keep.into()); + match read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY, BlockId::::number(number))? { + Some(body) => { + debug!(target: "db", "Removing block #{}", number); + utils::remove_from_db( + transaction, + &*self.storage.db, + columns::KEY_LOOKUP, + columns::BODY, + BlockId::::number(number), + )?; + match self.transaction_storage { + TransactionStorageMode::BlockBody => {}, + TransactionStorageMode::StorageChain => { + match Vec::::decode(&mut &body[..]) { + Ok(hashes) => { + for h in hashes { + transaction.remove(columns::TRANSACTION, h.as_ref()); } - Err(err) => return Err(sp_blockchain::Error::Backend( - format!("Error decoding body list: {}", err) - )), } + Err(err) => return Err(sp_blockchain::Error::Backend( + format!("Error decoding body list: {}", err) + )), } } } - None => return Ok(()), } + None => return Ok(()), } } Ok(()) diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index 4a4c68b6c2a2f..baea6aab69fa0 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -328,7 +328,7 @@ pub fn read_db( } /// Remove database column entry for the given block. -pub fn remove_db( +pub fn remove_from_db( transaction: &mut Transaction, db: &dyn Database, col_index: u32,