diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 900744f7e38d2..621f3a2941ff5 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -29,8 +29,15 @@ pub struct Analysis { model: Option, } +pub enum BenchmarkSelector { + ExtrinsicTime, + StorageRootTime, + Reads, + Writes, +} + impl Analysis { - pub fn median_slopes(r: &Vec) -> Option { + pub fn median_slopes(r: &Vec, selector: BenchmarkSelector) -> Option { let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { let mut counted = BTreeMap::, usize>::new(); for result in r.iter() { @@ -46,7 +53,16 @@ impl Analysis { .zip(others.iter()) .enumerate() .all(|(j, (v1, v2))| j == i || v1 == *v2) - ).map(|result| (result.components[i].1, result.extrinsic_time)) + ).map(|result| { + // Extract the data we are interested in analyzing + let data = match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }; + (result.components[i].1, data) + }) .collect::>(); (format!("{:?}", param), i, others, values) }).collect::>(); @@ -97,12 +113,18 @@ impl Analysis { }) } - pub fn min_squares_iqr(r: &Vec) -> Option { + pub fn min_squares_iqr(r: &Vec, selector: BenchmarkSelector) -> Option { let mut results = BTreeMap::, Vec>::new(); for result in r.iter() { let p = result.components.iter().map(|x| x.1).collect::>(); - results.entry(p).or_default().push(result.extrinsic_time); + results.entry(p).or_default().push(match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }) } + for (_, rs) in results.iter_mut() { rs.sort(); let ql = rs.len() / 4; @@ -217,6 +239,16 @@ impl std::fmt::Display for Analysis { } } +impl std::fmt::Debug for Analysis { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.base)?; + for (&m, n) in self.slopes.iter().zip(self.names.iter()) { + write!(f, " + ({} * {})", m, n)?; + } + write!(f,"") + } +} + #[cfg(test)] mod tests { use super::*; @@ -226,47 +258,69 @@ mod tests { components: Vec<(BenchmarkParameter, u32)>, extrinsic_time: u128, storage_root_time: u128, + reads: u32, + writes: u32, ) -> BenchmarkResults { BenchmarkResults { components, extrinsic_time, storage_root_time, - reads: 0, + reads, repeat_reads: 0, - writes: 0, + writes, repeat_writes: 0, } } #[test] fn analysis_median_slopes_should_work() { - let a = Analysis::median_slopes(&vec![ - benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } #[test] fn analysis_median_min_squares_should_work() { - let a = Analysis::min_squares_iqr(&vec![ - benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index d8fa1c98185c4..f0b31b4fbe65c 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -26,7 +26,7 @@ mod analysis; pub use utils::*; #[cfg(feature = "std")] -pub use analysis::Analysis; +pub use analysis::{Analysis, BenchmarkSelector}; #[doc(hidden)] pub use sp_io::storage::root as storage_root; pub use sp_runtime::traits::Zero; diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index c5719bcf6ae86..7f55672885d46 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -17,7 +17,7 @@ use crate::BenchmarkCmd; use codec::{Decode, Encode}; -use frame_benchmarking::{Analysis, BenchmarkBatch}; +use frame_benchmarking::{Analysis, BenchmarkBatch, BenchmarkSelector}; use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result}; use sc_client_db::BenchmarkingState; use sc_executor::NativeExecutor; @@ -130,13 +130,27 @@ impl BenchmarkCmd { // Conduct analysis. if !self.no_median_slopes { - if let Some(analysis) = Analysis::median_slopes(&batch.results) { - println!("Median Slopes Analysis\n========\n{}", analysis); + println!("Median Slopes Analysis\n========"); + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } if !self.no_min_squares { - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { - println!("Min Squares Analysis\n========\n{}", analysis); + println!("Min Squares Analysis\n========"); + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } },