Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Commit

Permalink
Batch benchmarks together with * notation. (#5436)
Browse files Browse the repository at this point in the history 
* Batch benchmarks together with `*` notation.

* Fix short structopt conflict

* Return error if `batches` is empty

* Move fast benchmarks macro into `frame_benchmarking` (#5445)

* Move macro into `frame_benchmarking`

* Update docs

* Extra line

* Return error if `batches` is empty

Co-authored-by: Shawn Tabrizi <[email protected]>
  • Loading branch information
@gavofyork @shawntabrizi
gavofyork and shawntabrizi authored Mar 31, 2020
1 parent ea5ea42 commit 5f51dd8
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 126 deletions.
97 changes: 18 additions & 79 deletions bin/node/runtime/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,95 +858,34 @@ impl_runtime_apis! {
#[cfg(feature = "runtime-benchmarks")]
impl frame_benchmarking::Benchmark<Block> for Runtime {
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<frame_benchmarking::BenchmarkResults>, sp_runtime::RuntimeString> {
use frame_benchmarking::Benchmarking;
) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark};
// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
// To get around that, we separated the Session benchmarks into its own crate, which is why
// we need these two lines below.
use pallet_session_benchmarking::Module as SessionBench;
impl pallet_session_benchmarking::Trait for Runtime {}

let result = match module.as_slice() {
b"pallet-balances" | b"balances" => Balances::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-im-online" | b"im-online" => ImOnline::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-identity" | b"identity" => Identity::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-session" | b"session" => SessionBench::<Runtime>::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-staking" | b"staking" => Staking::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-treasury" | b"treasury" => Treasury::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-vesting" | b"vesting" => Vesting::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-democracy" | b"democracy" => Democracy::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
b"pallet-collective" | b"collective" => Council::run_benchmark(
extrinsic,
lowest_range_values,
highest_range_values,
steps,
repeat,
),
_ => Err("Benchmark not found for this pallet."),
};

result.map_err(|e| e.into())
let mut batches = Vec::<BenchmarkBatch>::new();
let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
add_benchmark!(params, batches, b"balances", Balances);
add_benchmark!(params, batches, b"im-online", ImOnline);
add_benchmark!(params, batches, b"identity", Identity);
add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
add_benchmark!(params, batches, b"staking", Staking);
add_benchmark!(params, batches, b"timestamp", Timestamp);
add_benchmark!(params, batches, b"treasury", Treasury);
add_benchmark!(params, batches, b"vesting", Vesting);
add_benchmark!(params, batches, b"democracy", Democracy);
add_benchmark!(params, batches, b"collective", Council);
if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
Ok(batches)
}
}
}
Expand Down
90 changes: 80 additions & 10 deletions frame/benchmarking/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,15 +505,19 @@ macro_rules! impl_benchmark {
NO_INSTANCE $( $name:ident ),*
) => {
impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}

fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
Expand Down Expand Up @@ -597,15 +601,19 @@ macro_rules! impl_benchmark {
INSTANCE $( $name:ident ),*
) => {
impl<T: Trait<I>, I: Instance> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T, I> {
fn benchmarks() -> Vec<&'static [u8]> {
vec![ $( stringify!($name).as_ref() ),* ]
}

fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
extrinsic: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
// Map the input to the selected benchmark.
let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
let extrinsic = sp_std::str::from_utf8(extrinsic)
.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
let selected_benchmark = match extrinsic {
$( stringify!($name) => SelectedBenchmark::$name, )*
Expand Down Expand Up @@ -686,3 +694,65 @@ macro_rules! impl_benchmark {
}
}
}


/// This macro adds pallet benchmarks to a `Vec<BenchmarkBatch>` object.
///
/// First create an object that holds in the input parameters for the benchmark:
///
/// ```ignore
/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat);
/// ```
///
/// Then define a mutable local variable to hold your `BenchmarkBatch` object:
///
/// ```ignore
/// let mut batches = Vec::<BenchmarkBatch>::new();
/// ````
///
/// Then add the pallets you want to benchmark to this object, including the string
/// you want to use target a particular pallet:
///
/// ```ignore
/// add_benchmark!(params, batches, b"balances", Balances);
/// add_benchmark!(params, batches, b"identity", Identity);
/// add_benchmark!(params, batches, b"session", SessionBench::<Runtime>);
/// ...
/// ```
///
/// At the end of `dispatch_benchmark`, you should return this batches object.
#[macro_export]
macro_rules! add_benchmark {
( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => (
let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params;
if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] {
if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] {
for benchmark in $( $location )*::benchmarks().into_iter() {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
benchmark,
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.to_vec(),
});
}
} else {
$batches.push($crate::BenchmarkBatch {
results: $( $location )*::run_benchmark(
&benchmark[..],
&lowest_range_values[..],
&highest_range_values[..],
&steps[..],
repeat,
)?,
pallet: pallet.to_vec(),
benchmark: benchmark.clone(),
});
}
}
)
}
34 changes: 25 additions & 9 deletions frame/benchmarking/src/utils.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,24 @@ use sp_io::hashing::blake2_256;
use sp_runtime::RuntimeString;

/// An alphabet of possible parameters to use for benchmarking.
#[derive(codec::Encode, codec::Decode, Clone, Copy, PartialEq, Debug)]
#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)]
#[allow(missing_docs)]
#[allow(non_camel_case_types)]
pub enum BenchmarkParameter {
a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z,
}

/// The results of a single of benchmark.
#[derive(Encode, Decode, Clone, PartialEq, Debug)]
pub struct BenchmarkBatch {
/// The pallet containing this benchmark.
pub pallet: Vec<u8>,
/// The extrinsic (or benchmark name) of this benchmark.
pub benchmark: Vec<u8>,
/// The results from this benchmark.
pub results: Vec<BenchmarkResults>,
}

/// Results from running benchmarks on a FRAME pallet.
/// Contains duration of the function call in nanoseconds along with the benchmark parameters
/// used for that benchmark result.
Expand All @@ -39,13 +50,13 @@ sp_api::decl_runtime_apis! {
pub trait Benchmark {
/// Dispatch the given benchmark.
fn dispatch_benchmark(
module: Vec<u8>,
extrinsic: Vec<u8>,
pallet: Vec<u8>,
benchmark: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
repeat: u32,
) -> Result<Vec<BenchmarkResults>, RuntimeString>;
) -> Result<Vec<BenchmarkBatch>, RuntimeString>;
}
}

Expand Down Expand Up @@ -75,19 +86,24 @@ pub trait Benchmarking {

/// The pallet benchmarking trait.
pub trait Benchmarking<T> {
/// Get the benchmarks available for this pallet. Generally there is one benchmark per
/// extrinsic, so these are sometimes just called "extrinsics".
fn benchmarks() -> Vec<&'static [u8]>;

/// Run the benchmarks for this pallet.
///
/// Parameters
/// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes.
/// - `name`: The name of extrinsic function or benchmark you want to benchmark encoded as
/// bytes.
/// - `steps`: The number of sample points you want to take across the range of parameters.
/// - `lowest_range_values`: The lowest number for each range of parameters.
/// - `highest_range_values`: The highest number for each range of parameters.
/// - `repeat`: The number of times you want to repeat a benchmark.
fn run_benchmark(
extrinsic: Vec<u8>,
lowest_range_values: Vec<u32>,
highest_range_values: Vec<u32>,
steps: Vec<u32>,
name: &[u8],
lowest_range_values: &[u32],
highest_range_values: &[u32],
steps: &[u32],
repeat: u32,
) -> Result<Vec<T>, &'static str>;
}
Expand Down
Loading

0 comments on commit 5f51dd8

Please sign in to comment.