diff --git a/jsontests/Cargo.toml b/jsontests/Cargo.toml index 2c6af1b6..75222176 100644 --- a/jsontests/Cargo.toml +++ b/jsontests/Cargo.toml @@ -4,6 +4,10 @@ version = "0.0.0" license = "Apache-2.0" authors = ["Stewart Mackenzie ", "Wei Tang "] +[[bench]] +name = "performance" +harness = false + [dependencies] sputnikvm = { path = '..' } jsontests-derive = { path = "./jsontests-derive" } @@ -14,7 +18,7 @@ lazy_static = "0.2" env_logger = "0.5.11" sha3 = "0.6" etcommon-rlp = { version = "0.2", default-features = false } - +criterion = "0.2.5" [features] default = [] diff --git a/jsontests/benches/performance.rs b/jsontests/benches/performance.rs index b519a294..b1e798b4 100644 --- a/jsontests/benches/performance.rs +++ b/jsontests/benches/performance.rs @@ -1,14 +1,27 @@ -#![cfg(feature = "bench")] -#![feature(test)] #![allow(non_snake_case)] #[macro_use] extern crate jsontests_derive; extern crate jsontests; -extern crate test; +#[macro_use] +extern crate criterion; + +use criterion::Criterion; +use std::time::Duration; #[derive(JsonTests)] -#[directory = "jsontests/res/files/vmPerformance"] +#[directory = "jsontests/res/files/eth/VMTests/vmPerformance"] #[test_with = "jsontests::util::run_test"] #[bench_with = "jsontests::util::run_bench"] +#[criterion_config = "criterion_cfg"] struct Performance; + + +pub fn criterion_cfg() -> Criterion { + // Due to poor SputnikVM performance, there's no chance to get a lot of measurements + // and higher threshold is needed + Criterion::default() + .sample_size(2) + .measurement_time(Duration::from_secs(10)) + .noise_threshold(0.07) +} diff --git a/jsontests/jsontests-derive/Cargo.toml b/jsontests/jsontests-derive/Cargo.toml index d6b2e966..c257661c 100644 --- a/jsontests/jsontests-derive/Cargo.toml +++ b/jsontests/jsontests-derive/Cargo.toml @@ -16,3 +16,5 @@ quote = "0.3" serde_json = "1.0" failure = "0.1.2" itertools = "0.7.8" +criterion = "0.2.5" + diff --git a/jsontests/jsontests-derive/src/attr.rs b/jsontests/jsontests-derive/src/attr.rs index 052a6a39..e3ccaa9e 100644 --- a/jsontests/jsontests-derive/src/attr.rs +++ b/jsontests/jsontests-derive/src/attr.rs @@ -10,6 +10,7 @@ pub struct Config { pub directory: String, pub test_with: ExternalRef, pub bench_with: Option, + pub criterion_config: Option, pub skip: bool, pub should_panic: bool, } @@ -61,6 +62,7 @@ pub fn extract_attrs(ast: &syn::DeriveInput) -> Result { "directory" => Config { directory: value.clone(), ..config }, "test_with" => Config { test_with: ExternalRef::from(value.clone()), ..config }, "bench_with" => Config { bench_with: Some(ExternalRef::from(value.clone())), ..config }, + "criterion_config" => Config { criterion_config: Some(ExternalRef::from(value.clone())), ..config }, _ => panic!("{}", ERROR_MSG), } }, diff --git a/jsontests/jsontests-derive/src/lib.rs b/jsontests/jsontests-derive/src/lib.rs index d9eb192e..acb32085 100644 --- a/jsontests/jsontests-derive/src/lib.rs +++ b/jsontests/jsontests-derive/src/lib.rs @@ -11,7 +11,7 @@ mod attr; mod tests; mod util; -use attr::extract_attrs; +use attr::{Config, extract_attrs}; use tests::read_tests_from_dir; use util::*; @@ -21,7 +21,7 @@ use itertools::Itertools; use syn::Ident; use proc_macro::TokenStream; -#[proc_macro_derive(JsonTests, attributes(directory, test_with, bench_with, skip, should_panic))] +#[proc_macro_derive(JsonTests, attributes(directory, test_with, bench_with, criterion_config, skip, should_panic))] pub fn json_tests(input: TokenStream) -> TokenStream { // Construct a string representation of the type definition let s = input.to_string(); @@ -43,60 +43,53 @@ fn impl_json_tests(ast: &syn::DeriveInput) -> Result { let config = extract_attrs(&ast)?; let tests = read_tests_from_dir(&config.directory)?; let mut tokens = quote::Tokens::new(); + let mut bench_idents = Vec::new(); // split tests into groups by filepath let tests = tests.group_by(|test| test.path.clone()); - open_directory_module(&config, &mut tokens); + let dir_mod_name = open_directory_module(&config, &mut tokens); + + // If behchmarking support is requested, import Criterion + if config.bench_with.is_some() { + tokens.append(quote! { + use criterion::Criterion; + }) + } for (filepath, tests) in &tests { // If tests count in this file is 1, we don't need submodule let tests = tests.collect::>(); let need_file_submodule = tests.len() > 1; - + let mut file_mod_name = None; if need_file_submodule { - open_file_module(&filepath, &mut tokens); + file_mod_name = Some(open_file_module(&filepath, &mut tokens)); + // If behchmarking support is requested, import Criterion + if config.bench_with.is_some() { + tokens.append(quote! { + use criterion::Criterion; + }) + } } // Generate test function for test in tests { - let test_func_path = &config.test_with.path; - let test_func_name = &config.test_with.name; let name = sanitize_ident(&test.name); let name_ident = Ident::from(name.as_ref()); let data = json::to_string(&test.data)?; - // generate test attrs - tokens.append(quote!{#[test]}); - if config.should_panic { - tokens.append(quote!{#[should_panic]}); - } - - // generate test body - tokens.append(quote! { - fn #name_ident() { - use #test_func_path; - let data = #data; - #test_func_name(#name, data); - } - }); - - // generate optional benchmark body - if let Some(ref bench) = config.bench_with { - let bench_func_path = &bench.path; - let bench_func_name = &bench.name; - let name = format!("bench_{}", name); - let name_ident = Ident::from(name.as_ref()); - - tokens.append(quote! { - #[bench] - fn #name_ident(b: &mut test::Bencher) { - use #bench_func_path; - let data = #data; - #bench_func_name(b, #name, data); + generate_test(&config, &name_ident, &data, &mut tokens); + generate_bench(&config, &name_ident, &data, &mut tokens) + .map(|mut ident| { + // prepend dir submodule + ident = Ident::from(format!("{}::{}", dir_mod_name, ident.as_ref())); + // prepend file submodule + if need_file_submodule { + ident = Ident::from(format!("{}::{}", file_mod_name.as_ref().unwrap(), ident.as_ref())); } - }) - } + bench_idents.push(ident); + }); + } if need_file_submodule { @@ -108,8 +101,68 @@ fn impl_json_tests(ast: &syn::DeriveInput) -> Result { // Close directory module close_brace(&mut tokens); + generate_criterion_macros(&config, &bench_idents, &mut tokens); + Ok(tokens) } +fn generate_test(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) { + let test_func_path = &config.test_with.path; + let test_func_name = &config.test_with.name; + let test_name_str = test_name.as_ref(); + tokens.append(quote!{#[test]}); + if config.should_panic { + tokens.append(quote!{#[should_panic]}); + } + + tokens.append(quote! { + fn #test_name() { + use #test_func_path; + let data = #data; + #test_func_name(#test_name_str, data); + } + }); +} + +fn generate_bench(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) -> Option { + if config.bench_with.is_none() { + return None + } + + let bench = config.bench_with.as_ref().unwrap(); + let bench_func_path = &bench.path; + let bench_func_name = &bench.name; + let bench_name = format!("bench_{}", test_name.as_ref()); + let bench_ident = Ident::from(bench_name.as_ref()); + + tokens.append(quote! { + pub fn #bench_ident(c: &mut Criterion) { + use #bench_func_path; + let data = #data; + #bench_func_name(c, #bench_name, data); + } + }); + + Some(bench_ident) +} + +fn generate_criterion_macros(config: &Config, benches: &[Ident], tokens: &mut quote::Tokens) { + // Generate criterion macros + if config.bench_with.is_some() { + let benches = benches.iter().map(AsRef::as_ref).join(" , "); + let config = config.criterion_config + .as_ref() + .map(|cfg| cfg.path.clone()) + .unwrap_or_else(|| Ident::from("Criterion::default")); + let template = quote! { + criterion_group! { + name = main; + config = #config(); + targets = TARGETS + }; + }; + tokens.append(template.as_ref().replace("TARGETS", &benches)); + } +} diff --git a/jsontests/jsontests-derive/src/util.rs b/jsontests/jsontests-derive/src/util.rs index 6657b1c6..e784b796 100644 --- a/jsontests/jsontests-derive/src/util.rs +++ b/jsontests/jsontests-derive/src/util.rs @@ -3,29 +3,34 @@ use quote; use attr::Config; -pub fn open_directory_module(config: &Config, tokens: &mut quote::Tokens) { +pub fn open_directory_module(config: &Config, tokens: &mut quote::Tokens) -> String { // get the leaf directory name let dirname = config.directory.rsplit('/').next().unwrap(); // create identifier let dirname = sanitize_ident(dirname); - let dirname = Ident::from(dirname); + let dirname_ident = Ident::from(dirname.as_ref()); - open_module(&dirname, tokens); + open_module(dirname_ident, tokens); + + dirname } -pub fn open_file_module(filepath: &str, tokens: &mut quote::Tokens) { +pub fn open_file_module(filepath: &str, tokens: &mut quote::Tokens) -> String { // get file name without extension let filename = filepath.rsplit('/').next().unwrap() .split('.').next().unwrap(); // create identifier let filename = sanitize_ident(filename); - let filename = Ident::from(filename); + let filename_ident = Ident::from(filename.as_ref()); + + open_module(filename_ident, tokens); - open_module(&filename, tokens); + filename } -pub fn open_module(module_name: &Ident, tokens: &mut quote::Tokens) { +pub fn open_module>(module_name: I, tokens: &mut quote::Tokens) { + let module_name = module_name.into(); // append module opening tokens tokens.append(quote! { mod #module_name diff --git a/jsontests/src/lib.rs b/jsontests/src/lib.rs index 0e25863e..a67f288c 100644 --- a/jsontests/src/lib.rs +++ b/jsontests/src/lib.rs @@ -1,4 +1,3 @@ -#![cfg_attr(feature = "bench", feature(test))] extern crate sputnikvm; extern crate serde_json; extern crate hexutil; @@ -6,9 +5,7 @@ extern crate bigint; extern crate env_logger; extern crate sha3; extern crate rlp; - -#[cfg(feature = "bench")] -extern crate test; +extern crate criterion; mod blockchain; pub mod util; @@ -299,6 +296,25 @@ pub fn test_transaction(_name: &str, v: &Value, debug: bool) -> Result>> = Arc::new(Mutex::new(Vec::new())); + let history_closure = history.clone(); + let mut machine = create_machine(&v, &block); + machine.add_context_history_hook(move |context| { + history_closure.lock().unwrap().push(context.clone()); + }); + (machine, block) + }, |(mut machine, block)| { + fire_with_block(&mut machine, &block); + }) + }); +} + /// Read U256 number exactly the way go big.Int parses strings /// except for base 2 and 8 which are not used in tests pub fn read_u256(number: &str) -> U256 { diff --git a/jsontests/src/util.rs b/jsontests/src/util.rs index f2261641..e2465b83 100644 --- a/jsontests/src/util.rs +++ b/jsontests/src/util.rs @@ -1,21 +1,16 @@ use serde_json::Value; use serde_json as json; use test_transaction; +use bench_transaction; pub fn run_test(name: &str, test: &str) { let test: Value = json::from_str(test).unwrap(); assert_eq!(test_transaction(name, &test, true), Ok(true)); } -#[cfg(feature = "bench")] -use test::Bencher; +use criterion::Criterion; -#[cfg(feature = "bench")] -pub fn run_bench(b: &mut Bencher, name: &str, test: &str) { +pub fn run_bench(c: &mut Criterion, name: &'static str, test: &str) { let test: Value = json::from_str(test).unwrap(); - b.iter(|| { - // TODO: adjust test_transaction or write another function - // TODO: in order to start benchmark as close to actual sputnik code as possible - assert_eq!(test_transaction(name, &test, true), Ok(true)); - }) + bench_transaction(name, test, c); }