diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 8868f28557e70..799fc5397ced6 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -858,95 +858,34 @@ impl_runtime_apis! { #[cfg(feature = "runtime-benchmarks")] impl frame_benchmarking::Benchmark for Runtime { fn dispatch_benchmark( - module: Vec, - extrinsic: Vec, + pallet: Vec, + benchmark: Vec, lowest_range_values: Vec, highest_range_values: Vec, steps: Vec, repeat: u32, - ) -> Result, sp_runtime::RuntimeString> { - use frame_benchmarking::Benchmarking; + ) -> Result, sp_runtime::RuntimeString> { + use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark}; // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues. // To get around that, we separated the Session benchmarks into its own crate, which is why // we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; impl pallet_session_benchmarking::Trait for Runtime {} - let result = match module.as_slice() { - b"pallet-balances" | b"balances" => Balances::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-identity" | b"identity" => Identity::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-session" | b"session" => SessionBench::::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-staking" | b"staking" => Staking::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-treasury" | b"treasury" => Treasury::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-vesting" | b"vesting" => Vesting::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-democracy" | b"democracy" => Democracy::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - b"pallet-collective" | b"collective" => Council::run_benchmark( - extrinsic, - lowest_range_values, - highest_range_values, - steps, - repeat, - ), - _ => Err("Benchmark not found for this pallet."), - }; - - result.map_err(|e| e.into()) + let mut batches = Vec::::new(); + let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); + add_benchmark!(params, batches, b"balances", Balances); + add_benchmark!(params, batches, b"im-online", ImOnline); + add_benchmark!(params, batches, b"identity", Identity); + add_benchmark!(params, batches, b"session", SessionBench::); + add_benchmark!(params, batches, b"staking", Staking); + add_benchmark!(params, batches, b"timestamp", Timestamp); + add_benchmark!(params, batches, b"treasury", Treasury); + add_benchmark!(params, batches, b"vesting", Vesting); + add_benchmark!(params, batches, b"democracy", Democracy); + add_benchmark!(params, batches, b"collective", Council); + if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } + Ok(batches) } } } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 4254edcac57cf..4cd6072ce4555 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -505,15 +505,19 @@ macro_rules! impl_benchmark { NO_INSTANCE $( $name:ident ),* ) => { impl $crate::Benchmarking<$crate::BenchmarkResults> for Module { + fn benchmarks() -> Vec<&'static [u8]> { + vec![ $( stringify!($name).as_ref() ),* ] + } + fn run_benchmark( - extrinsic: Vec, - lowest_range_values: Vec, - highest_range_values: Vec, - steps: Vec, + extrinsic: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], repeat: u32, ) -> Result, &'static str> { // Map the input to the selected benchmark. - let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice()) + let extrinsic = sp_std::str::from_utf8(extrinsic) .map_err(|_| "`extrinsic` is not a valid utf8 string!")?; let selected_benchmark = match extrinsic { $( stringify!($name) => SelectedBenchmark::$name, )* @@ -597,15 +601,19 @@ macro_rules! impl_benchmark { INSTANCE $( $name:ident ),* ) => { impl, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module { + fn benchmarks() -> Vec<&'static [u8]> { + vec![ $( stringify!($name).as_ref() ),* ] + } + fn run_benchmark( - extrinsic: Vec, - lowest_range_values: Vec, - highest_range_values: Vec, - steps: Vec, + extrinsic: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], repeat: u32, ) -> Result, &'static str> { // Map the input to the selected benchmark. - let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice()) + let extrinsic = sp_std::str::from_utf8(extrinsic) .map_err(|_| "`extrinsic` is not a valid utf8 string!")?; let selected_benchmark = match extrinsic { $( stringify!($name) => SelectedBenchmark::$name, )* @@ -686,3 +694,65 @@ macro_rules! impl_benchmark { } } } + + +/// This macro adds pallet benchmarks to a `Vec` object. +/// +/// First create an object that holds in the input parameters for the benchmark: +/// +/// ```ignore +/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); +/// ``` +/// +/// Then define a mutable local variable to hold your `BenchmarkBatch` object: +/// +/// ```ignore +/// let mut batches = Vec::::new(); +/// ```` +/// +/// Then add the pallets you want to benchmark to this object, including the string +/// you want to use target a particular pallet: +/// +/// ```ignore +/// add_benchmark!(params, batches, b"balances", Balances); +/// add_benchmark!(params, batches, b"identity", Identity); +/// add_benchmark!(params, batches, b"session", SessionBench::); +/// ... +/// ``` +/// +/// At the end of `dispatch_benchmark`, you should return this batches object. +#[macro_export] +macro_rules! add_benchmark { + ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( + let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params; + if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { + if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { + for benchmark in $( $location )*::benchmarks().into_iter() { + $batches.push($crate::BenchmarkBatch { + results: $( $location )*::run_benchmark( + benchmark, + &lowest_range_values[..], + &highest_range_values[..], + &steps[..], + repeat, + )?, + pallet: pallet.to_vec(), + benchmark: benchmark.to_vec(), + }); + } + } else { + $batches.push($crate::BenchmarkBatch { + results: $( $location )*::run_benchmark( + &benchmark[..], + &lowest_range_values[..], + &highest_range_values[..], + &steps[..], + repeat, + )?, + pallet: pallet.to_vec(), + benchmark: benchmark.clone(), + }); + } + } + ) +} diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 122ef029979f5..a6f262eab99ec 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256; use sp_runtime::RuntimeString; /// An alphabet of possible parameters to use for benchmarking. -#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)] +#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)] #[allow(missing_docs)] #[allow(non_camel_case_types)] pub enum BenchmarkParameter { a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, } +/// The results of a single of benchmark. +#[derive(Encode, Decode, Clone, PartialEq, Debug)] +pub struct BenchmarkBatch { + /// The pallet containing this benchmark. + pub pallet: Vec, + /// The extrinsic (or benchmark name) of this benchmark. + pub benchmark: Vec, + /// The results from this benchmark. + pub results: Vec, +} + /// Results from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. @@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! { pub trait Benchmark { /// Dispatch the given benchmark. fn dispatch_benchmark( - module: Vec, - extrinsic: Vec, + pallet: Vec, + benchmark: Vec, lowest_range_values: Vec, highest_range_values: Vec, steps: Vec, repeat: u32, - ) -> Result, RuntimeString>; + ) -> Result, RuntimeString>; } } @@ -75,19 +86,24 @@ pub trait Benchmarking { /// The pallet benchmarking trait. pub trait Benchmarking { + /// Get the benchmarks available for this pallet. Generally there is one benchmark per + /// extrinsic, so these are sometimes just called "extrinsics". + fn benchmarks() -> Vec<&'static [u8]>; + /// Run the benchmarks for this pallet. /// /// Parameters - /// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes. + /// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as + /// bytes. /// - `steps`: The number of sample points you want to take across the range of parameters. /// - `lowest_range_values`: The lowest number for each range of parameters. /// - `highest_range_values`: The highest number for each range of parameters. /// - `repeat`: The number of times you want to repeat a benchmark. fn run_benchmark( - extrinsic: Vec, - lowest_range_values: Vec, - highest_range_values: Vec, - steps: Vec, + name: &[u8], + lowest_range_values: &[u32], + highest_range_values: &[u32], + steps: &[u32], repeat: u32, ) -> Result, &'static str>; } diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 1c02a754016e3..f9851810bf6c9 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -22,7 +22,7 @@ use sc_client_db::BenchmarkingState; use sc_service::{Configuration, ChainSpec}; use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use codec::{Encode, Decode}; -use frame_benchmarking::{BenchmarkResults, Analysis}; +use frame_benchmarking::{BenchmarkBatch, Analysis}; use sp_core::{ tasks, traits::KeystoreExt, @@ -33,11 +33,11 @@ use sp_externalities::Extensions; /// The `benchmark` command used to benchmark FRAME Pallets. #[derive(Debug, structopt::StructOpt, Clone)] pub struct BenchmarkCmd { - /// Select a FRAME Pallet to benchmark. + /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). #[structopt(short, long)] pub pallet: String, - /// Select an extrinsic to benchmark. + /// Select an extrinsic inside the pallet to benchmark, or `*` for all. #[structopt(short, long)] pub extrinsic: String, @@ -46,17 +46,29 @@ pub struct BenchmarkCmd { pub steps: Vec, /// Indicates lowest values for each of the component ranges. - #[structopt(long, use_delimiter = true)] + #[structopt(long = "low", use_delimiter = true)] pub lowest_range_values: Vec, /// Indicates highest values for each of the component ranges. - #[structopt(long, use_delimiter = true)] + #[structopt(long = "high", use_delimiter = true)] pub highest_range_values: Vec, /// Select how many repetitions of this benchmark should run. #[structopt(short, long, default_value = "1")] pub repeat: u32, + /// Print the raw results. + #[structopt(long = "raw")] + pub raw_data: bool, + + /// Don't print the median-slopes linear regression analysis. + #[structopt(long)] + pub no_median_slopes: bool, + + /// Don't print the min-squares linear regression analysis. + #[structopt(long)] + pub no_min_squares: bool, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, @@ -135,47 +147,50 @@ impl BenchmarkCmd { .execute(strategy.into()) .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; - let results = , String> as Decode>::decode(&mut &result[..]) + let results = , String> as Decode>::decode(&mut &result[..]) .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; match results { - Ok(results) => { + Ok(batches) => for batch in batches.into_iter() { // Print benchmark metadata println!( "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - self.pallet, - self.extrinsic, + String::from_utf8(batch.pallet).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), self.lowest_range_values, self.highest_range_values, self.steps, self.repeat, ); - // Print the table header - results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + if self.raw_data { + // Print the table header + batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); - print!("extrinsic_time,storage_root_time\n"); - // Print the values - results.iter().for_each(|result| { - let parameters = &result.0; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.1, result.2); - }); + print!("extrinsic_time,storage_root_time\n"); + // Print the values + batch.results.iter().for_each(|result| { + let parameters = &result.0; + parameters.iter().for_each(|param| print!("{:?},", param.1)); + // Print extrinsic time and storage root time + print!("{:?},{:?}\n", result.1, result.2); + }); - print!("\n"); + print!("\n"); + } // Conduct analysis. - if let Some(analysis) = Analysis::median_slopes(&results) { - println!("Median Slopes Analysis\n========\n{}", analysis); + if !self.no_median_slopes { + if let Some(analysis) = Analysis::median_slopes(&batch.results) { + println!("Median Slopes Analysis\n========\n{}", analysis); + } } - - if let Some(analysis) = Analysis::min_squares_iqr(&results) { - println!("Min Squares Analysis\n========\n{}", analysis); + if !self.no_min_squares { + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { + println!("Min Squares Analysis\n========\n{}", analysis); + } } - - eprintln!("Done."); - } + }, Err(error) => eprintln!("Error: {:?}", error), }