Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Add regression analysis to DB Tracking #6475

Merged
merged 3 commits into from
Jun 22, 2020
Merged

Add regression analysis to DB Tracking #6475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
af3d482
Add selector
shawntabrizi Jun 22, 2020
20e57c8
add tests
shawntabrizi Jun 22, 2020
4038366
debug formatter for easy formula
shawntabrizi Jun 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 84 additions & 30 deletions frame/benchmarking/src/analysis.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,15 @@ pub struct Analysis {
model: Option<RegressionModel>,
}

pub enum BenchmarkSelector {
ExtrinsicTime,
StorageRootTime,
Reads,
Writes,
}

impl Analysis {
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
pub fn median_slopes(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
for result in r.iter() {
Expand All @@ -46,7 +53,16 @@ impl Analysis {
.zip(others.iter())
.enumerate()
.all(|(j, (v1, v2))| j == i || v1 == *v2)
).map(|result| (result.components[i].1, result.extrinsic_time))
).map(|result| {
// Extract the data we are interested in analyzing
let data = match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
};
(result.components[i].1, data)
})
.collect::<Vec<_>>();
(format!("{:?}", param), i, others, values)
}).collect::<Vec<_>>();
Expand Down Expand Up @@ -97,12 +113,18 @@ impl Analysis {
})
}

pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
pub fn min_squares_iqr(r: &Vec<BenchmarkResults>, selector: BenchmarkSelector) -> Option<Self> {
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
for result in r.iter() {
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(result.extrinsic_time);
results.entry(p).or_default().push(match selector {
BenchmarkSelector::ExtrinsicTime => result.extrinsic_time,
BenchmarkSelector::StorageRootTime => result.storage_root_time,
BenchmarkSelector::Reads => result.reads.into(),
BenchmarkSelector::Writes => result.writes.into(),
})
}

for (_, rs) in results.iter_mut() {
rs.sort();
let ql = rs.len() / 4;
Expand Down Expand Up @@ -217,6 +239,16 @@ impl std::fmt::Display for Analysis {
}
}

impl std::fmt::Debug for Analysis {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.base)?;
for (&m, n) in self.slopes.iter().zip(self.names.iter()) {
write!(f, " + ({} * {})", m, n)?;
}
write!(f,"")
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand All @@ -226,47 +258,69 @@ mod tests {
components: Vec<(BenchmarkParameter, u32)>,
extrinsic_time: u128,
storage_root_time: u128,
reads: u32,
writes: u32,
) -> BenchmarkResults {
BenchmarkResults {
components,
extrinsic_time,
storage_root_time,
reads: 0,
reads,
repeat_reads: 0,
writes: 0,
writes,
repeat_writes: 0,
}
}

#[test]
fn analysis_median_slopes_should_work() {
let a = Analysis::median_slopes(&vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];

let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);

let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);

let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}

#[test]
fn analysis_median_min_squares_should_work() {
let a = Analysis::min_squares_iqr(&vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
let data = vec![
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20),
];

let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap();
assert_eq!(extrinsic_time.base, 10_000_000);
assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]);

let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap();
assert_eq!(reads.base, 2);
assert_eq!(reads.slopes, vec![1, 0]);

let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap();
assert_eq!(writes.base, 0);
assert_eq!(writes.slopes, vec![0, 2]);
}
}
2 changes: 1 addition & 1 deletion frame/benchmarking/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ mod analysis;

pub use utils::*;
#[cfg(feature = "std")]
pub use analysis::Analysis;
pub use analysis::{Analysis, BenchmarkSelector};
#[doc(hidden)]
pub use sp_io::storage::root as storage_root;
pub use sp_runtime::traits::Zero;
Expand Down
24 changes: 19 additions & 5 deletions utils/frame/benchmarking-cli/src/command.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

use crate::BenchmarkCmd;
use codec::{Decode, Encode};
use frame_benchmarking::{Analysis, BenchmarkBatch};
use frame_benchmarking::{Analysis, BenchmarkBatch, BenchmarkSelector};
use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result};
use sc_client_db::BenchmarkingState;
use sc_executor::NativeExecutor;
Expand Down Expand Up @@ -130,13 +130,27 @@ impl BenchmarkCmd {

// Conduct analysis.
if !self.no_median_slopes {
if let Some(analysis) = Analysis::median_slopes(&batch.results) {
println!("Median Slopes Analysis\n========\n{}", analysis);
println!("Median Slopes Analysis\n========");
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
}
if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
}
}
if !self.no_min_squares {
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) {
println!("Min Squares Analysis\n========\n{}", analysis);
println!("Min Squares Analysis\n========");
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) {
println!("-- Extrinsic Time --\n{}", analysis);
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) {
println!("Reads = {:?}", analysis);
}
if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) {
println!("Writes = {:?}", analysis);
}
}
},
Expand Down