From 4616e775e66b35a13677caf68c3f0c0bb69dd365 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 24 Aug 2020 15:54:09 +0200 Subject: [PATCH 1/4] Move subcommands from sc-cli to nodes --- bin/node-template/node/src/cli.rs | 26 +- bin/node-template/node/src/command.rs | 56 +++- bin/node/cli/src/cli.rs | 25 +- bin/node/cli/src/command.rs | 52 +++- client/cli/src/commands/mod.rs | 403 -------------------------- client/cli/src/runner.rs | 51 +--- 6 files changed, 141 insertions(+), 472 deletions(-) diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs index 46ab9bc3dafac..f3667fa79d19e 100644 --- a/bin/node-template/node/src/cli.rs +++ b/bin/node-template/node/src/cli.rs @@ -1,5 +1,5 @@ use structopt::StructOpt; -use sc_cli::{RunCmd, Subcommand}; +use sc_cli::RunCmd; #[derive(Debug, StructOpt)] pub struct Cli { @@ -9,3 +9,27 @@ pub struct Cli { #[structopt(flatten)] pub run: RunCmd, } + +#[derive(Debug, StructOpt)] +pub enum Subcommand { + /// Build a chain specification. + BuildSpec(sc_cli::BuildSpecCmd), + + /// Validate blocks. + CheckBlock(sc_cli::CheckBlockCmd), + + /// Export blocks. + ExportBlocks(sc_cli::ExportBlocksCmd), + + /// Export the state of a given block into a chain spec. + ExportState(sc_cli::ExportStateCmd), + + /// Import blocks. + ImportBlocks(sc_cli::ImportBlocksCmd), + + /// Remove the whole chain. + PurgeChain(sc_cli::PurgeChainCmd), + + /// Revert the chain to a previous state. + Revert(sc_cli::RevertCmd), +} diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 9cd2248d6547a..98c56e948300e 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -16,7 +16,7 @@ // limitations under the License. use crate::chain_spec; -use crate::cli::Cli; +use crate::cli::{Cli, Subcommand}; use crate::service; use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec}; use sc_service::PartialComponents; @@ -66,15 +66,55 @@ impl SubstrateCli for Cli { pub fn run() -> sc_cli::Result<()> { let cli = Cli::from_args(); - match cli.subcommand { - Some(ref subcommand) => { - let runner = cli.create_runner(subcommand)?; - runner.run_subcommand(subcommand, |config| { - let PartialComponents { client, backend, task_manager, import_queue, .. } + match &cli.subcommand { + Some(Subcommand::BuildSpec(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|config| cmd.run(config.chain_spec, config.network)) + }, + Some(Subcommand::CheckBlock(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, import_queue, ..} = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) + Ok((cmd.run(client, import_queue), task_manager)) }) - } + }, + Some(Subcommand::ExportBlocks(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, ..} + = new_partial(&config)?; + Ok((cmd.run(client, config.database), task_manager)) + }) + }, + Some(Subcommand::ExportState(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, ..} + = new_partial(&config)?; + Ok((cmd.run(client, config.chain_spec), task_manager)) + }) + }, + Some(Subcommand::ImportBlocks(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, import_queue, ..} + = new_partial(&config)?; + Ok((cmd.run(client, import_queue), task_manager)) + }) + }, + Some(Subcommand::PurgeChain(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|config| cmd.run(config.database)) + }, + Some(Subcommand::Revert(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, backend, ..} + = new_partial(&config)?; + Ok((cmd.run(client, backend), task_manager)) + }) + }, None => { let runner = cli.create_runner(&cli.run)?; runner.run_node_until_exit(|config| match config.role { diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 42a13fcb39070..2130ff1e4b106 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -33,10 +33,6 @@ pub struct Cli { /// Possible subcommands of the main binary. #[derive(Debug, StructOpt)] pub enum Subcommand { - /// A set of base subcommands handled by `sc_cli`. - #[structopt(flatten)] - Base(sc_cli::Subcommand), - /// Key management cli utilities Key(KeySubcommand), @@ -59,4 +55,25 @@ pub enum Subcommand { /// Sign a message, with a given (secret) key. Sign(SignCmd), + + /// Build a chain specification. + BuildSpec(sc_cli::BuildSpecCmd), + + /// Validate blocks. + CheckBlock(sc_cli::CheckBlockCmd), + + /// Export blocks. + ExportBlocks(sc_cli::ExportBlocksCmd), + + /// Export the state of a given block into a chain spec. + ExportState(sc_cli::ExportStateCmd), + + /// Import blocks. + ImportBlocks(sc_cli::ImportBlocksCmd), + + /// Remove the whole chain. + PurgeChain(sc_cli::PurgeChainCmd), + + /// Revert the chain to a previous state. + Revert(sc_cli::RevertCmd), } diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 10e9413702b81..a715b2ecaa091 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -97,13 +97,53 @@ pub fn run() -> Result<()> { Some(Subcommand::Sign(cmd)) => cmd.run(), Some(Subcommand::Verify(cmd)) => cmd.run(), Some(Subcommand::Vanity(cmd)) => cmd.run(), - Some(Subcommand::Base(subcommand)) => { - let runner = cli.create_runner(subcommand)?; - runner.run_subcommand(subcommand, |config| { - let PartialComponents { client, backend, task_manager, import_queue, ..} + Some(Subcommand::BuildSpec(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|config| cmd.run(config.chain_spec, config.network)) + }, + Some(Subcommand::CheckBlock(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, import_queue, ..} = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) + Ok((cmd.run(client, import_queue), task_manager)) }) - } + }, + Some(Subcommand::ExportBlocks(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, ..} + = new_partial(&config)?; + Ok((cmd.run(client, config.database), task_manager)) + }) + }, + Some(Subcommand::ExportState(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, ..} + = new_partial(&config)?; + Ok((cmd.run(client, config.chain_spec), task_manager)) + }) + }, + Some(Subcommand::ImportBlocks(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, import_queue, ..} + = new_partial(&config)?; + Ok((cmd.run(client, import_queue), task_manager)) + }) + }, + Some(Subcommand::PurgeChain(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|config| cmd.run(config.database)) + }, + Some(Subcommand::Revert(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, backend, ..} + = new_partial(&config)?; + Ok((cmd.run(client, backend), task_manager)) + }) + }, } } diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index 108c38b19db30..7b740d1003238 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -34,9 +34,6 @@ mod inspect; mod key; pub mod utils; -use std::fmt::Debug; -use structopt::StructOpt; - pub use self::{ build_spec_cmd::BuildSpecCmd, check_block_cmd::CheckBlockCmd, @@ -56,403 +53,3 @@ pub use self::{ revert_cmd::RevertCmd, run_cmd::RunCmd, }; - -/// All core commands that are provided by default. -/// -/// The core commands are split into multiple subcommands and `Run` is the default subcommand. From -/// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of -/// `Run` are exported as main executable parameters. -#[derive(Debug, StructOpt)] -pub enum Subcommand { - /// Build a spec.json file, outputs to stdout. - BuildSpec(BuildSpecCmd), - - /// Export blocks to a file. - ExportBlocks(ExportBlocksCmd), - - /// Import blocks from file. - ImportBlocks(ImportBlocksCmd), - - /// Validate a single block. - CheckBlock(CheckBlockCmd), - - /// Export state as raw chain spec. - ExportState(ExportStateCmd), - - /// Revert chain to the previous state. - Revert(RevertCmd), - - /// Remove the whole chain data. - PurgeChain(PurgeChainCmd), -} - -/// Macro that helps implement CliConfiguration on an enum of subcommand automatically -/// -/// # Example -/// -/// ``` -/// # #[macro_use] extern crate sc_cli; -/// -/// # struct EmptyVariant {} -/// -/// # impl sc_cli::CliConfiguration for EmptyVariant { -/// # fn shared_params(&self) -> &sc_cli::SharedParams { unimplemented!() } -/// # fn chain_id(&self, _: bool) -> sc_cli::Result { Ok("test-chain-id".to_string()) } -/// # } -/// -/// # fn main() { -/// enum Subcommand { -/// Variant1(EmptyVariant), -/// Variant2(EmptyVariant), -/// } -/// -/// substrate_cli_subcommands!( -/// Subcommand => Variant1, Variant2 -/// ); -/// -/// # use sc_cli::CliConfiguration; -/// # assert_eq!(Subcommand::Variant1(EmptyVariant {}).chain_id(false).unwrap(), "test-chain-id"); -/// -/// # } -/// ``` -/// -/// Which will expand to: -/// -/// ```ignore -/// impl CliConfiguration for Subcommand { -/// fn base_path(&self) -> Result> { -/// match self { -/// Subcommand::Variant1(cmd) => cmd.base_path(), -/// Subcommand::Variant2(cmd) => cmd.base_path(), -/// } -/// } -/// -/// fn is_dev(&self) -> Result { -/// match self { -/// Subcommand::Variant1(cmd) => cmd.is_dev(), -/// Subcommand::Variant2(cmd) => cmd.is_dev(), -/// } -/// } -/// -/// // ... -/// } -/// ``` -#[macro_export] -macro_rules! substrate_cli_subcommands { - ($enum:ident => $($variant:ident),*) => { - impl $crate::CliConfiguration for $enum { - fn shared_params(&self) -> &$crate::SharedParams { - match self { - $($enum::$variant(cmd) => cmd.shared_params()),* - } - } - - fn import_params(&self) -> Option<&$crate::ImportParams> { - match self { - $($enum::$variant(cmd) => cmd.import_params()),* - } - } - - fn pruning_params(&self) -> Option<&$crate::PruningParams> { - match self { - $($enum::$variant(cmd) => cmd.pruning_params()),* - } - } - - fn keystore_params(&self) -> Option<&$crate::KeystoreParams> { - match self { - $($enum::$variant(cmd) => cmd.keystore_params()),* - } - } - - fn network_params(&self) -> Option<&$crate::NetworkParams> { - match self { - $($enum::$variant(cmd) => cmd.network_params()),* - } - } - - fn offchain_worker_params(&self) -> Option<&$crate::OffchainWorkerParams> { - match self { - $($enum::$variant(cmd) => cmd.offchain_worker_params()),* - } - } - - fn database_params(&self) -> Option<&$crate::DatabaseParams> { - match self { - $($enum::$variant(cmd) => cmd.database_params()),* - } - } - - fn base_path(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.base_path()),* - } - } - - fn is_dev(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.is_dev()),* - } - } - - fn role(&self, is_dev: bool) -> $crate::Result<::sc_service::Role> { - match self { - $($enum::$variant(cmd) => cmd.role(is_dev)),* - } - } - - fn transaction_pool(&self) - -> $crate::Result<::sc_service::config::TransactionPoolOptions> { - match self { - $($enum::$variant(cmd) => cmd.transaction_pool()),* - } - } - - fn network_config( - &self, - chain_spec: &std::boxed::Box, - is_dev: bool, - net_config_dir: std::path::PathBuf, - client_id: &str, - node_name: &str, - node_key: sc_service::config::NodeKeyConfig, - default_listen_port: u16, - ) -> $crate::Result<::sc_service::config::NetworkConfiguration> { - match self { - $( - $enum::$variant(cmd) => cmd.network_config( - chain_spec, - is_dev, - net_config_dir, - client_id, - node_name, - node_key, - default_listen_port, - ) - ),* - } - } - - fn keystore_config(&self, base_path: &::std::path::PathBuf) - -> $crate::Result<::sc_service::config::KeystoreConfig> { - match self { - $($enum::$variant(cmd) => cmd.keystore_config(base_path)),* - } - } - - fn database_cache_size(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.database_cache_size()),* - } - } - - fn database_config( - &self, - base_path: &::std::path::PathBuf, - cache_size: usize, - database: $crate::Database, - ) -> $crate::Result<::sc_service::config::DatabaseConfig> { - match self { - $($enum::$variant(cmd) => cmd.database_config(base_path, cache_size, database)),* - } - } - - fn database(&self) -> $crate::Result<::std::option::Option<$crate::Database>> { - match self { - $($enum::$variant(cmd) => cmd.database()),* - } - } - - fn state_cache_size(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.state_cache_size()),* - } - } - - fn state_cache_child_ratio(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.state_cache_child_ratio()),* - } - } - - fn pruning(&self, unsafe_pruning: bool, role: &::sc_service::Role) - -> $crate::Result<::sc_service::config::PruningMode> { - match self { - $($enum::$variant(cmd) => cmd.pruning(unsafe_pruning, role)),* - } - } - - fn chain_id(&self, is_dev: bool) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.chain_id(is_dev)),* - } - } - - fn init(&self) -> $crate::Result<()> { - match self { - $($enum::$variant(cmd) => cmd.init::()),* - } - } - - fn node_name(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.node_name()),* - } - } - - fn wasm_method(&self) -> $crate::Result<::sc_service::config::WasmExecutionMethod> { - match self { - $($enum::$variant(cmd) => cmd.wasm_method()),* - } - } - - fn execution_strategies(&self, is_dev: bool, is_validator: bool) - -> $crate::Result<::sc_client_api::execution_extensions::ExecutionStrategies> { - match self { - $($enum::$variant(cmd) => cmd.execution_strategies(is_dev, is_validator)),* - } - } - - fn rpc_ipc(&self) -> $crate::Result<::std::option::Option<::std::string::String>> { - match self { - $($enum::$variant(cmd) => cmd.rpc_ipc()),* - } - } - - fn rpc_http( - &self, - default_listen_port: u16, - ) -> $crate::Result> { - match self { - $($enum::$variant(cmd) => cmd.rpc_http(default_listen_port)),* - } - } - - fn rpc_ws( - &self, - default_listen_port: u16, - ) -> $crate::Result> { - match self { - $($enum::$variant(cmd) => cmd.rpc_ws(default_listen_port)),* - } - } - - fn rpc_methods(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.rpc_methods()),* - } - } - - fn rpc_ws_max_connections(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.rpc_ws_max_connections()),* - } - } - - fn rpc_cors(&self, is_dev: bool) - -> $crate::Result>> { - match self { - $($enum::$variant(cmd) => cmd.rpc_cors(is_dev)),* - } - } - - fn prometheus_config(&self, default_listen_port: u16) - -> $crate::Result> { - match self { - $($enum::$variant(cmd) => cmd.prometheus_config(default_listen_port)),* - } - } - - fn telemetry_endpoints( - &self, - chain_spec: &Box, - ) -> $crate::Result> { - match self { - $($enum::$variant(cmd) => cmd.telemetry_endpoints(chain_spec)),* - } - } - - fn telemetry_external_transport(&self) - -> $crate::Result<::std::option::Option<::sc_service::config::ExtTransport>> { - match self { - $($enum::$variant(cmd) => cmd.telemetry_external_transport()),* - } - } - - fn default_heap_pages(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.default_heap_pages()),* - } - } - - fn offchain_worker( - &self, - role: &::sc_service::Role, - ) -> $crate::Result<::sc_service::config::OffchainWorkerConfig> { - match self { - $($enum::$variant(cmd) => cmd.offchain_worker(role)),* - } - } - - fn force_authoring(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.force_authoring()),* - } - } - - fn disable_grandpa(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.disable_grandpa()),* - } - } - - fn dev_key_seed(&self, is_dev: bool) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.dev_key_seed(is_dev)),* - } - } - - fn tracing_targets(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.tracing_targets()),* - } - } - - fn tracing_receiver(&self) -> $crate::Result<::sc_service::TracingReceiver> { - match self { - $($enum::$variant(cmd) => cmd.tracing_receiver()),* - } - } - - fn node_key(&self, net_config_dir: &::std::path::PathBuf) - -> $crate::Result<::sc_service::config::NodeKeyConfig> { - match self { - $($enum::$variant(cmd) => cmd.node_key(net_config_dir)),* - } - } - - fn max_runtime_instances(&self) -> $crate::Result<::std::option::Option> { - match self { - $($enum::$variant(cmd) => cmd.max_runtime_instances()),* - } - } - - fn log_filters(&self) -> $crate::Result { - match self { - $($enum::$variant(cmd) => cmd.log_filters()),* - } - } - } - } -} - -substrate_cli_subcommands!( - Subcommand => - BuildSpec, - ExportBlocks, - ExportState, - ImportBlocks, - CheckBlock, - Revert, - PurgeChain -); diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index f2558b1bb6070..64bd88d63130b 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -18,7 +18,6 @@ use crate::CliConfiguration; use crate::Result; -use crate::Subcommand; use crate::SubstrateCli; use chrono::prelude::*; use futures::pin_mut; @@ -26,10 +25,8 @@ use futures::select; use futures::{future, future::FutureExt, Future}; use log::info; use sc_service::{Configuration, TaskType, TaskManager}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; -use std::{fmt::Debug, marker::PhantomData, str::FromStr, sync::Arc}; -use sc_client_api::{UsageProvider, BlockBackend, StorageProvider}; +use std::marker::PhantomData; #[cfg(target_family = "unix")] async fn main(func: F) -> std::result::Result<(), Box> @@ -173,52 +170,6 @@ impl Runner { info!("⛓ Native runtime: {}", C::native_runtime_version(&self.config.chain_spec)); } - /// A helper function that runs a future with tokio and stops if the process receives the signal - /// `SIGTERM` or `SIGINT`. - pub fn run_subcommand(self, subcommand: &Subcommand, builder: BU) - -> Result<()> - where - BU: FnOnce(Configuration) - -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, - B: BlockT + for<'de> serde::Deserialize<'de>, - BA: sc_client_api::backend::Backend + 'static, - IQ: sc_service::ImportQueue + 'static, - ::Hash: FromStr, - <::Hash as FromStr>::Err: Debug, - <<::Header as HeaderT>::Number as FromStr>::Err: Debug, - CL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + - 'static, - { - let chain_spec = self.config.chain_spec.cloned_box(); - let network_config = self.config.network.clone(); - let db_config = self.config.database.clone(); - - match subcommand { - Subcommand::BuildSpec(cmd) => cmd.run(chain_spec, network_config), - Subcommand::ExportBlocks(cmd) => { - let (client, _, _, task_manager) = builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(client, db_config), task_manager) - } - Subcommand::ImportBlocks(cmd) => { - let (client, _, import_queue, task_manager) = builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) - } - Subcommand::CheckBlock(cmd) => { - let (client, _, import_queue, task_manager) = builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) - } - Subcommand::Revert(cmd) => { - let (client, backend, _, task_manager) = builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(client, backend), task_manager) - }, - Subcommand::PurgeChain(cmd) => cmd.run(db_config), - Subcommand::ExportState(cmd) => { - let (client, _, _, task_manager) = builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(client, chain_spec), task_manager) - }, - } - } - /// A helper function that runs a node with tokio and stops if the process receives the signal /// `SIGTERM` or `SIGINT`. pub fn run_node_until_exit( From 101a439ca34a278c4fbf2e16872a3af147c735d3 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Thu, 27 Aug 2020 17:16:40 +0200 Subject: [PATCH 2/4] Add --build-sync-spec subcommand --- bin/node/cli/src/chain_spec.rs | 7 +- bin/node/cli/src/cli.rs | 3 + bin/node/cli/src/command.rs | 13 +- bin/node/cli/src/service.rs | 53 ++++---- .../cli/src/commands/build_sync_spec_cmd.rs | 116 ++++++++++++++++++ client/cli/src/commands/mod.rs | 2 + 6 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 client/cli/src/commands/build_sync_spec_cmd.rs diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index e323f7956f169..90824a5572f12 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -380,7 +380,7 @@ pub fn local_testnet_config() -> ChainSpec { #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::service::{new_full_base, new_light_base}; + use crate::service::{new_full_base, new_light_base, NewFullBase}; use sc_service_test; use sp_runtime::BuildStorage; @@ -431,8 +431,9 @@ pub(crate) mod tests { sc_service_test::connectivity( integration_test_config_with_two_authorities(), |config| { - let (keep_alive, _, client, network, transaction_pool) = new_full_base(config,|_, _| ())?; - Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + let NewFullBase { task_manager, client, network, transaction_pool, .. } + = new_full_base(config,|_, _| ())?; + Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool)) }, |config| { let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 2130ff1e4b106..6e51dae93793f 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -59,6 +59,9 @@ pub enum Subcommand { /// Build a chain specification. BuildSpec(sc_cli::BuildSpecCmd), + /// Build a chain specification with a light client sync state. + BuildSyncSpec(sc_cli::BuildSyncSpecCmd), + /// Validate blocks. CheckBlock(sc_cli::CheckBlockCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index a715b2ecaa091..07913f6690359 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -21,7 +21,7 @@ use node_executor::Executor; use node_runtime::{Block, RuntimeApi}; use sc_cli::{Result, SubstrateCli, RuntimeVersion, Role, ChainSpec}; use sc_service::PartialComponents; -use crate::service::new_partial; +use crate::service::{new_partial, new_full_base, NewFullBase}; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -101,6 +101,17 @@ pub fn run() -> Result<()> { let runner = cli.create_runner(cmd)?; runner.sync_run(|config| cmd.run(config.chain_spec, config.network)) }, + Some(Subcommand::BuildSyncSpec(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let chain_spec = config.chain_spec.cloned_box(); + let network_config = config.network.clone(); + let NewFullBase { task_manager, client, backend, network_status_sinks, .. } + = new_full_base(config, |_, _| ())?; + + Ok((cmd.run(chain_spec, network_config, client, backend, network_status_sinks), task_manager)) + }) + }, Some(Subcommand::CheckBlock(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index d91696ab7d6bc..734a8f43f2ff8 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -151,6 +151,16 @@ pub fn new_partial(config: &Configuration) -> Result, + pub backend: Arc, + pub network: Arc::Hash>>, + pub network_status_sinks: sc_service::NetworkStatusSinks, + pub transaction_pool: Arc>, +} + /// Creates a full service from the configuration. pub fn new_full_base( config: Configuration, @@ -158,11 +168,7 @@ pub fn new_full_base( &sc_consensus_babe::BabeBlockImport, &sc_consensus_babe::BabeLink, ) -) -> Result<( - TaskManager, InherentDataProviders, Arc, - Arc::Hash>>, - Arc>, -), ServiceError> { +) -> Result { let sc_service::PartialComponents { client, backend, mut task_manager, import_queue, keystore, select_chain, transaction_pool, inherent_data_providers, @@ -210,7 +216,7 @@ pub fn new_full_base( on_demand: None, remote_blockchain: None, telemetry_connection_sinks: telemetry_connection_sinks.clone(), - network_status_sinks, + network_status_sinks: network_status_sinks.clone(), system_rpc_tx, })?; @@ -330,13 +336,16 @@ pub fn new_full_base( } network_starter.start_network(); - Ok((task_manager, inherent_data_providers, client, network, transaction_pool)) + Ok(NewFullBase { + task_manager, inherent_data_providers, client, backend, network, network_status_sinks, + transaction_pool, + }) } /// Builds a new service for a full client. pub fn new_full(config: Configuration) -> Result { - new_full_base(config, |_, _| ()).map(|(task_manager, _, _, _, _)| { + new_full_base(config, |_, _| ()).map(|NewFullBase { task_manager, .. }| { task_manager }) } @@ -467,7 +476,7 @@ mod tests { use sp_finality_tracker; use sp_keyring::AccountKeyring; use sc_service_test::TestNetNode; - use crate::service::{new_full_base, new_light_base}; + use crate::service::{new_full_base, new_light_base, NewFullBase}; use sp_runtime::traits::IdentifyAccount; use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sc_client_api::BlockBackend; @@ -499,18 +508,19 @@ mod tests { chain_spec, |config| { let mut setup_handles = None; - let (keep_alive, inherent_data_providers, client, network, transaction_pool) = - new_full_base(config, - | - block_import: &sc_consensus_babe::BabeBlockImport, - babe_link: &sc_consensus_babe::BabeLink, - | { - setup_handles = Some((block_import.clone(), babe_link.clone())); - } - )?; + let NewFullBase { + task_manager, inherent_data_providers, client, network, transaction_pool, .. + } = new_full_base(config, + | + block_import: &sc_consensus_babe::BabeBlockImport, + babe_link: &sc_consensus_babe::BabeLink, + | { + setup_handles = Some((block_import.clone(), babe_link.clone())); + } + )?; let node = sc_service_test::TestNetComponents::new( - keep_alive, client, network, transaction_pool + task_manager, client, network, transaction_pool ); Ok((node, (inherent_data_providers, setup_handles.unwrap()))) }, @@ -661,8 +671,9 @@ mod tests { sc_service_test::consensus( crate::chain_spec::tests::integration_test_config_with_two_authorities(), |config| { - let (keep_alive, _, client, network, transaction_pool) = new_full_base(config, |_, _| ())?; - Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) + let NewFullBase { task_manager, client, network, transaction_pool, .. } + = new_full_base(config,|_, _| ())?; + Ok(sc_service_test::TestNetComponents::new(task_manager, client, network, transaction_pool)) }, |config| { let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; diff --git a/client/cli/src/commands/build_sync_spec_cmd.rs b/client/cli/src/commands/build_sync_spec_cmd.rs new file mode 100644 index 0000000000000..ee158172a383f --- /dev/null +++ b/client/cli/src/commands/build_sync_spec_cmd.rs @@ -0,0 +1,116 @@ +// This file is part of Substrate. + +// Copyright (C) 2018-2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::error; +use crate::params::{SharedParams, NetworkParams}; +use crate::CliConfiguration; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec}; +use structopt::StructOpt; +use std::io::Write; +use std::sync::Arc; +use sp_runtime::traits::Block as BlockT; +use sc_service::chain_ops::build_light_sync_state; +use sc_service::NetworkStatusSinks; +use futures::{FutureExt, StreamExt}; +use futures::future::ready; + +/// The `build-sync-spec` command used to build a chain spec that contains a light client state +/// so that light clients can sync faster. +#[derive(Debug, StructOpt)] +pub struct BuildSyncSpecCmd { + /// Force raw genesis storage output. + #[structopt(long = "raw")] + pub raw: bool, + + /// Sync the chain using a full client first. + #[structopt(long)] + pub sync_first: bool, + + /// Disable adding the default bootnode to the specification. + /// + /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the + /// specification when no bootnode exists. + #[structopt(long = "disable-default-bootnode")] + pub disable_default_bootnode: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub network_params: NetworkParams, +} + +impl BuildSyncSpecCmd { + /// Run the build-sync-spec command + pub async fn run( + &self, + mut spec: Box, + network_config: NetworkConfiguration, + client: Arc, + backend: Arc, + network_status_sinks: NetworkStatusSinks, + ) -> error::Result<()> + where + B: BlockT, + CL: sp_blockchain::HeaderBackend, + BA: sc_client_api::Backend, + >::Blockchain: sc_client_api::ProvideChtRoots, + { + if self.sync_first { + network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { + ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0) + }).into_future().map(drop).await; + } + + let light_sync_state = build_light_sync_state(client, backend)?; + spec.set_light_sync_state(light_sync_state.to_serializable()); + + info!("Building chain spec"); + let raw_output = self.raw; + + if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { + let keys = network_config.node_key.into_keypair()?; + let peer_id = keys.public().into_peer_id(); + let addr = MultiaddrWithPeerId { + multiaddr: build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(30333u16)], + peer_id, + }; + spec.add_boot_node(addr) + } + + let json = sc_service::chain_ops::build_spec(&*spec, raw_output)?; + if std::io::stdout().write_all(json.as_bytes()).is_err() { + let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); + } + Ok(()) + } +} + +impl CliConfiguration for BuildSyncSpecCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn network_params(&self) -> Option<&NetworkParams> { + Some(&self.network_params) + } +} diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index 7b740d1003238..899abf0c3d437 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . mod build_spec_cmd; +mod build_sync_spec_cmd; mod check_block_cmd; mod export_blocks_cmd; mod export_state_cmd; @@ -36,6 +37,7 @@ pub mod utils; pub use self::{ build_spec_cmd::BuildSpecCmd, + build_sync_spec_cmd::BuildSyncSpecCmd, check_block_cmd::CheckBlockCmd, export_blocks_cmd::ExportBlocksCmd, export_state_cmd::ExportStateCmd, From 2b7fd858a34f760f258a2b55ddb0c5f81b8f3ab3 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Thu, 27 Aug 2020 17:38:43 +0200 Subject: [PATCH 3/4] Remove CHTs from snapshots --- bin/node/cli/src/command.rs | 4 +- bin/node/cli/src/service.rs | 3 +- client/api/src/backend.rs | 18 ----- client/api/src/in_mem.rs | 20 +++--- client/api/src/light.rs | 20 ++++-- client/chain-spec/src/chain_spec.rs | 7 -- .../cli/src/commands/build_sync_spec_cmd.rs | 7 +- client/db/src/lib.rs | 66 +------------------ client/db/src/light.rs | 38 +++++------ client/light/src/blockchain.rs | 20 +----- .../{build_spec.rs => build_sync_spec.rs} | 33 ++-------- client/service/src/chain_ops/mod.rs | 4 +- client/service/test/src/client/light.rs | 20 +++--- 13 files changed, 66 insertions(+), 194 deletions(-) rename client/service/src/chain_ops/{build_spec.rs => build_sync_spec.rs} (52%) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 07913f6690359..4772d6e4be605 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -106,10 +106,10 @@ pub fn run() -> Result<()> { runner.async_run(|config| { let chain_spec = config.chain_spec.cloned_box(); let network_config = config.network.clone(); - let NewFullBase { task_manager, client, backend, network_status_sinks, .. } + let NewFullBase { task_manager, client, network_status_sinks, .. } = new_full_base(config, |_, _| ())?; - Ok((cmd.run(chain_spec, network_config, client, backend, network_status_sinks), task_manager)) + Ok((cmd.run(chain_spec, network_config, client, network_status_sinks), task_manager)) }) }, Some(Subcommand::CheckBlock(cmd)) => { diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 734a8f43f2ff8..51232df9b82b1 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -155,7 +155,6 @@ pub struct NewFullBase { pub task_manager: TaskManager, pub inherent_data_providers: InherentDataProviders, pub client: Arc, - pub backend: Arc, pub network: Arc::Hash>>, pub network_status_sinks: sc_service::NetworkStatusSinks, pub transaction_pool: Arc>, @@ -337,7 +336,7 @@ pub fn new_full_base( network_starter.start_network(); Ok(NewFullBase { - task_manager, inherent_data_providers, client, backend, network, network_status_sinks, + task_manager, inherent_data_providers, client, network, network_status_sinks, transaction_pool, }) } diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 47fec977f5e82..efc5ca4ee8ca0 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -536,21 +536,3 @@ pub fn changes_tries_state_at_block<'a, Block: BlockT>( None => Ok(None), } } - -/// Provide CHT roots. These are stored on a light client and generated dynamically on a full -/// client. -pub trait ProvideChtRoots { - /// Get headers CHT root for given block. Returns None if the block is not a part of any CHT. - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result>; - - /// Get changes trie CHT root for given block. Returns None if the block is not a part of any CHT. - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result>; -} diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index ded030fb8046f..306c3c2b2f10c 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -35,7 +35,7 @@ use sp_state_machine::{ use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata}; use crate::{ - backend::{self, NewBlockState, ProvideChtRoots}, + backend::{self, NewBlockState}, blockchain::{ self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId }, @@ -447,16 +447,6 @@ impl light::Storage for Blockchain Blockchain::finalize_header(self, id, None) } - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } -} - -impl ProvideChtRoots for Blockchain { fn header_cht_root( &self, _cht_size: NumberFor, @@ -476,6 +466,14 @@ impl ProvideChtRoots for Blockchain { .ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block))) .map(Some) } + + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } } /// In-memory operation. diff --git a/client/api/src/light.rs b/client/api/src/light.rs index 144851dac0075..b359c1149eea6 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -32,7 +32,7 @@ use sp_blockchain::{ HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache, Error as ClientError, Result as ClientResult, }; -use crate::{backend::{AuxStore, NewBlockState}, UsageInfo, ProvideChtRoots}; +use crate::{backend::{AuxStore, NewBlockState}, UsageInfo}; /// Remote call request. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -232,9 +232,7 @@ pub trait FetchChecker: Send + Sync { /// Light client blockchain storage. -pub trait Storage: AuxStore + HeaderBackend - + HeaderMetadata + ProvideChtRoots -{ +pub trait Storage: AuxStore + HeaderBackend + HeaderMetadata { /// Store new header. Should refuse to revert any finalized blocks. /// /// Takes new authorities, the leaf state of the new block, and @@ -256,6 +254,20 @@ pub trait Storage: AuxStore + HeaderBackend /// Get last finalized header. fn last_finalized(&self) -> ClientResult; + /// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT). + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult>; + + /// Get changes trie CHT root for given block. Returns None if the block is not pruned (not a part of any CHT). + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult>; + /// Get storage cache. fn cache(&self) -> Option>>; diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 20811394c56d7..1fbf0419e2001 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -401,8 +401,6 @@ where pub struct LightSyncState { /// The header of the best finalized block. pub header: ::Header, - /// A list of all CHTs in the chain. - pub chts: Vec<::Hash>, } impl LightSyncState { @@ -412,7 +410,6 @@ impl LightSyncState { SerializableLightSyncState { header: StorageData(self.header.encode()), - chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(), } } @@ -420,9 +417,6 @@ impl LightSyncState { pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result { Ok(Self { header: codec::Decode::decode(&mut &serialized.header.0[..])?, - chts: serialized.chts.iter() - .map(|cht| codec::Decode::decode(&mut &cht.0[..])) - .collect::>()?, }) } } @@ -433,7 +427,6 @@ impl LightSyncState { #[serde(deny_unknown_fields)] pub struct SerializableLightSyncState { header: StorageData, - chts: Vec, } #[cfg(test)] diff --git a/client/cli/src/commands/build_sync_spec_cmd.rs b/client/cli/src/commands/build_sync_spec_cmd.rs index ee158172a383f..4d87e2b063ac6 100644 --- a/client/cli/src/commands/build_sync_spec_cmd.rs +++ b/client/cli/src/commands/build_sync_spec_cmd.rs @@ -61,19 +61,16 @@ pub struct BuildSyncSpecCmd { impl BuildSyncSpecCmd { /// Run the build-sync-spec command - pub async fn run( + pub async fn run( &self, mut spec: Box, network_config: NetworkConfiguration, client: Arc, - backend: Arc, network_status_sinks: NetworkStatusSinks, ) -> error::Result<()> where B: BlockT, CL: sp_blockchain::HeaderBackend, - BA: sc_client_api::Backend, - >::Blockchain: sc_client_api::ProvideChtRoots, { if self.sync_first { network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { @@ -81,7 +78,7 @@ impl BuildSyncSpecCmd { }).into_future().map(drop).await; } - let light_sync_state = build_light_sync_state(client, backend)?; + let light_sync_state = build_light_sync_state(client)?; spec.set_light_sync_state(light_sync_state.to_serializable()); info!("Building chain spec"); diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index bd438f4dd71b2..d76816d962511 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -54,8 +54,8 @@ use std::collections::{HashMap, HashSet}; use sc_client_api::{ UsageInfo, MemoryInfo, IoInfo, MemorySize, - backend::{NewBlockState, PrunableStateChangesTrieStorage, ProvideChtRoots}, - leaves::{LeafSet, FinalizationDisplaced}, cht, + backend::{NewBlockState, PrunableStateChangesTrieStorage}, + leaves::{LeafSet, FinalizationDisplaced}, }; use sp_blockchain::{ Result as ClientResult, Error as ClientError, @@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; -use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage}; +use sp_runtime::{generic::BlockId, Justification, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; @@ -405,14 +405,6 @@ impl BlockchainDb { meta.finalized_hash = hash; } } - - // Get block changes trie root, if available. - fn changes_trie_root(&self, block: BlockId) -> ClientResult> { - self.header(block) - .map(|header| header.and_then(|header| - header.digest().log(DigestItem::as_changes_trie_root) - .cloned())) - } } impl sc_client_api::blockchain::HeaderBackend for BlockchainDb { @@ -533,58 +525,6 @@ impl HeaderMetadata for BlockchainDb { } } -impl ProvideChtRoots for BlockchainDb { - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - let cht_number = match cht::block_to_cht_number(cht_size, block) { - Some(number) => number, - None => return Ok(None), - }; - - let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); - - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - - cht::compute_root::, _>( - cht::size(), cht_number, cht_range.map(|num| self.hash(num)) - ).map(Some) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - let cht_number = match cht::block_to_cht_number(cht_size, block) { - Some(number) => number, - None => return Ok(None), - }; - - let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); - - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - - cht::compute_root::, _>( - cht::size(), - cht_number, - cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))), - ).map(Some) - } -} - /// Database transaction pub struct BlockImportOperation { old_state: SyncingCachingState, Block>, diff --git a/client/db/src/light.rs b/client/db/src/light.rs index acfb6217ce9e0..b7cf30901bf31 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -23,7 +23,7 @@ use std::convert::TryInto; use parking_lot::RwLock; use sc_client_api::{ - cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo, + cht, backend::{AuxStore, NewBlockState}, UsageInfo, blockchain::{ BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo, }, @@ -523,6 +523,22 @@ impl Storage for LightStorage } } + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) + } + fn finalize_header(&self, id: BlockId) -> ClientResult<()> { if let Some(header) = self.header(id)? { let mut transaction = Transaction::new(); @@ -596,26 +612,6 @@ impl Storage for LightStorage } } -impl ProvideChtRoots for LightStorage - where Block: BlockT, -{ - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) - } -} - /// Build the key for inserting header-CHT at given block. fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { let mut key = [cht_type; 5]; diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 3b5753f2849d5..39354aa45b0fb 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -29,7 +29,7 @@ use sp_blockchain::{ }; pub use sc_client_api::{ backend::{ - AuxStore, NewBlockState, ProvideChtRoots, + AuxStore, NewBlockState, }, blockchain::{ Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache, @@ -173,21 +173,3 @@ impl RemoteBlockchain for Blockchain })) } } - -impl, Block: BlockT> ProvideChtRoots for Blockchain { - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage().header_cht_root(cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage().changes_trie_cht_root(cht_size, block) - } -} diff --git a/client/service/src/chain_ops/build_spec.rs b/client/service/src/chain_ops/build_sync_spec.rs similarity index 52% rename from client/service/src/chain_ops/build_spec.rs rename to client/service/src/chain_ops/build_sync_spec.rs index 40d591d81f02b..9553ea21a6965 100644 --- a/client/service/src/chain_ops/build_spec.rs +++ b/client/service/src/chain_ops/build_sync_spec.rs @@ -14,48 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One}; +use sp_runtime::traits::Block as BlockT; use sp_blockchain::HeaderBackend; use std::sync::Arc; use sp_runtime::generic::BlockId; -use sc_client_api::ProvideChtRoots; /// Build a `LightSyncState` from the CHT roots stored in a backend. -pub fn build_light_sync_state( +pub fn build_light_sync_state( client: Arc, - backend: Arc, ) -> Result, sp_blockchain::Error> where TBl: BlockT, TCl: HeaderBackend, - TBackend: sc_client_api::Backend, - >::Blockchain: ProvideChtRoots, { - let cht_root_provider = backend.blockchain(); - let finalized_hash = client.info().finalized_hash; - let finalized_number = client.info().finalized_number; - - use sc_client_api::cht; - - let mut chts = Vec::new(); - - // We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. - let cht_size_x_2 = cht::size::>() * NumberFor::::from(2); - - let mut number = NumberFor::::one(); - - while number <= finalized_number.saturating_sub(cht_size_x_2) { - match cht_root_provider.header_cht_root(cht::size(), number)? { - Some(cht_root) => chts.push(cht_root), - None => log::error!("No CHT found for block {}", number), - } - - number += cht::size(); - } + let header = client.header(BlockId::Hash(finalized_hash))?.unwrap(); Ok(sc_chain_spec::LightSyncState { - header: client.header(BlockId::Hash(finalized_hash))?.unwrap(), - chts, + header }) } diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs index 19f5e346820aa..e6b2fdfb8e0e6 100644 --- a/client/service/src/chain_ops/mod.rs +++ b/client/service/src/chain_ops/mod.rs @@ -21,11 +21,11 @@ mod export_blocks; mod export_raw_state; mod import_blocks; mod revert_chain; -mod build_spec; +mod build_sync_spec; pub use check_block::*; pub use export_blocks::*; pub use export_raw_state::*; pub use import_blocks::*; pub use revert_chain::*; -pub use build_spec::*; +pub use build_sync_spec::*; diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index f38aef008e11c..ffc84ad47b8f3 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor}; use sc_client_api::{ blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, - in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ProvideChtRoots, + in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend, @@ -164,16 +164,6 @@ impl Storage for DummyStorage { Err(ClientError::Backend("Test error".into())) } - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } -} - -impl ProvideChtRoots for DummyStorage { fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { Err(ClientError::Backend("Test error".into())) } @@ -187,6 +177,14 @@ impl ProvideChtRoots for DummyStorage { ).into()) .map(Some) } + + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } } struct DummyCallExecutor; From ca61ff2a007e4a5411fe953a3ec96833742c24d8 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Tue, 1 Sep 2020 12:18:08 +0200 Subject: [PATCH 4/4] Keep ProvideChtRoots --- client/api/src/backend.rs | 18 +++++++ client/api/src/in_mem.rs | 20 ++++---- client/api/src/light.rs | 20 ++------ client/db/src/lib.rs | 66 +++++++++++++++++++++++-- client/db/src/light.rs | 38 +++++++------- client/light/src/blockchain.rs | 20 +++++++- client/service/test/src/client/light.rs | 20 ++++---- 7 files changed, 147 insertions(+), 55 deletions(-) diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index efc5ca4ee8ca0..47fec977f5e82 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -536,3 +536,21 @@ pub fn changes_tries_state_at_block<'a, Block: BlockT>( None => Ok(None), } } + +/// Provide CHT roots. These are stored on a light client and generated dynamically on a full +/// client. +pub trait ProvideChtRoots { + /// Get headers CHT root for given block. Returns None if the block is not a part of any CHT. + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result>; + + /// Get changes trie CHT root for given block. Returns None if the block is not a part of any CHT. + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result>; +} diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 306c3c2b2f10c..ded030fb8046f 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -35,7 +35,7 @@ use sp_state_machine::{ use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata}; use crate::{ - backend::{self, NewBlockState}, + backend::{self, NewBlockState, ProvideChtRoots}, blockchain::{ self, BlockStatus, HeaderBackend, well_known_cache_keys::Id as CacheKeyId }, @@ -447,6 +447,16 @@ impl light::Storage for Blockchain Blockchain::finalize_header(self, id, None) } + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } +} + +impl ProvideChtRoots for Blockchain { fn header_cht_root( &self, _cht_size: NumberFor, @@ -466,14 +476,6 @@ impl light::Storage for Blockchain .ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block))) .map(Some) } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } } /// In-memory operation. diff --git a/client/api/src/light.rs b/client/api/src/light.rs index b359c1149eea6..144851dac0075 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -32,7 +32,7 @@ use sp_blockchain::{ HeaderMetadata, well_known_cache_keys, HeaderBackend, Cache as BlockchainCache, Error as ClientError, Result as ClientResult, }; -use crate::{backend::{AuxStore, NewBlockState}, UsageInfo}; +use crate::{backend::{AuxStore, NewBlockState}, UsageInfo, ProvideChtRoots}; /// Remote call request. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -232,7 +232,9 @@ pub trait FetchChecker: Send + Sync { /// Light client blockchain storage. -pub trait Storage: AuxStore + HeaderBackend + HeaderMetadata { +pub trait Storage: AuxStore + HeaderBackend + + HeaderMetadata + ProvideChtRoots +{ /// Store new header. Should refuse to revert any finalized blocks. /// /// Takes new authorities, the leaf state of the new block, and @@ -254,20 +256,6 @@ pub trait Storage: AuxStore + HeaderBackend + HeaderMetada /// Get last finalized header. fn last_finalized(&self) -> ClientResult; - /// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT). - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult>; - - /// Get changes trie CHT root for given block. Returns None if the block is not pruned (not a part of any CHT). - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult>; - /// Get storage cache. fn cache(&self) -> Option>>; diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index d76816d962511..bd438f4dd71b2 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -54,8 +54,8 @@ use std::collections::{HashMap, HashSet}; use sc_client_api::{ UsageInfo, MemoryInfo, IoInfo, MemorySize, - backend::{NewBlockState, PrunableStateChangesTrieStorage}, - leaves::{LeafSet, FinalizationDisplaced}, + backend::{NewBlockState, PrunableStateChangesTrieStorage, ProvideChtRoots}, + leaves::{LeafSet, FinalizationDisplaced}, cht, }; use sp_blockchain::{ Result as ClientResult, Error as ClientError, @@ -70,7 +70,7 @@ use sp_core::ChangesTrieConfiguration; use sp_core::offchain::storage::{OffchainOverlayedChange, OffchainOverlayedChanges}; use sp_core::storage::{well_known_keys, ChildInfo}; use sp_arithmetic::traits::Saturating; -use sp_runtime::{generic::BlockId, Justification, Storage}; +use sp_runtime::{generic::{DigestItem, BlockId}, Justification, Storage}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, SaturatedConversion, HashFor, }; @@ -405,6 +405,14 @@ impl BlockchainDb { meta.finalized_hash = hash; } } + + // Get block changes trie root, if available. + fn changes_trie_root(&self, block: BlockId) -> ClientResult> { + self.header(block) + .map(|header| header.and_then(|header| + header.digest().log(DigestItem::as_changes_trie_root) + .cloned())) + } } impl sc_client_api::blockchain::HeaderBackend for BlockchainDb { @@ -525,6 +533,58 @@ impl HeaderMetadata for BlockchainDb { } } +impl ProvideChtRoots for BlockchainDb { + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result> { + let cht_number = match cht::block_to_cht_number(cht_size, block) { + Some(number) => number, + None => return Ok(None), + }; + + let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); + + let mut current_num = cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); + + cht::compute_root::, _>( + cht::size(), cht_number, cht_range.map(|num| self.hash(num)) + ).map(Some) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result> { + let cht_number = match cht::block_to_cht_number(cht_size, block) { + Some(number) => number, + None => return Ok(None), + }; + + let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); + + let mut current_num = cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); + + cht::compute_root::, _>( + cht::size(), + cht_number, + cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))), + ).map(Some) + } +} + /// Database transaction pub struct BlockImportOperation { old_state: SyncingCachingState, Block>, diff --git a/client/db/src/light.rs b/client/db/src/light.rs index b7cf30901bf31..acfb6217ce9e0 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -23,7 +23,7 @@ use std::convert::TryInto; use parking_lot::RwLock; use sc_client_api::{ - cht, backend::{AuxStore, NewBlockState}, UsageInfo, + cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo, blockchain::{ BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo, }, @@ -523,22 +523,6 @@ impl Storage for LightStorage } } - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) - } - fn finalize_header(&self, id: BlockId) -> ClientResult<()> { if let Some(header) = self.header(id)? { let mut transaction = Transaction::new(); @@ -612,6 +596,26 @@ impl Storage for LightStorage } } +impl ProvideChtRoots for LightStorage + where Block: BlockT, +{ + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) + } +} + /// Build the key for inserting header-CHT at given block. fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { let mut key = [cht_type; 5]; diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs index 39354aa45b0fb..3b5753f2849d5 100644 --- a/client/light/src/blockchain.rs +++ b/client/light/src/blockchain.rs @@ -29,7 +29,7 @@ use sp_blockchain::{ }; pub use sc_client_api::{ backend::{ - AuxStore, NewBlockState, + AuxStore, NewBlockState, ProvideChtRoots, }, blockchain::{ Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache, @@ -173,3 +173,21 @@ impl RemoteBlockchain for Blockchain })) } } + +impl, Block: BlockT> ProvideChtRoots for Blockchain { + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result> { + self.storage().header_cht_root(cht_size, block) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> sp_blockchain::Result> { + self.storage().changes_trie_cht_root(cht_size, block) + } +} diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index ffc84ad47b8f3..f38aef008e11c 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor}; use sc_client_api::{ blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, - in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, + in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ProvideChtRoots, AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend, @@ -164,6 +164,16 @@ impl Storage for DummyStorage { Err(ClientError::Backend("Test error".into())) } + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } +} + +impl ProvideChtRoots for DummyStorage { fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { Err(ClientError::Backend("Test error".into())) } @@ -177,14 +187,6 @@ impl Storage for DummyStorage { ).into()) .map(Some) } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } } struct DummyCallExecutor;