Merge branch 'master' into shift-opcodes

jsontests/Cargo.toml

@@ -4,6 +4,10 @@ version = "0.0.0"
 license = "Apache-2.0"
 authors = ["Stewart Mackenzie <[email protected]>", "Wei Tang <[email protected]>"]
 
+[[bench]]
+name = "performance"
+harness = false
+
 [dependencies]
 sputnikvm = { path = '..' }
 jsontests-derive = { path = "./jsontests-derive" }
@@ -14,7 +18,7 @@ lazy_static = "0.2"
 env_logger = "0.5.11"
 sha3 = "0.6"
 etcommon-rlp = { version = "0.2", default-features = false }
-
+criterion = "0.2.5"
 
 [features]
 default = []

jsontests/benches/performance.rs

@@ -1,14 +1,27 @@
-#![cfg(feature = "bench")]
-#![feature(test)]
 #![allow(non_snake_case)]
 
 #[macro_use]
 extern crate jsontests_derive;
 extern crate jsontests;
-extern crate test;
+#[macro_use]
+extern crate criterion;
+
+use criterion::Criterion;
+use std::time::Duration;
 
 #[derive(JsonTests)]
-#[directory = "jsontests/res/files/vmPerformance"]
+#[directory = "jsontests/res/files/eth/VMTests/vmPerformance"]
 #[test_with = "jsontests::util::run_test"]
 #[bench_with = "jsontests::util::run_bench"]
+#[criterion_config = "criterion_cfg"]
 struct Performance;
+
+
+pub fn criterion_cfg() -> Criterion {
+    // Due to poor SputnikVM performance, there's no chance to get a lot of measurements
+    // and higher threshold is needed
+    Criterion::default()
+        .sample_size(2)
+        .measurement_time(Duration::from_secs(10))
+        .noise_threshold(0.07)
+}

jsontests/jsontests-derive/Cargo.toml

@@ -16,3 +16,5 @@ quote = "0.3"
 serde_json = "1.0"
 failure = "0.1.2"
 itertools = "0.7.8"
+criterion = "0.2.5"
+

jsontests/jsontests-derive/src/attr.rs

@@ -10,6 +10,7 @@ pub struct Config {
     pub directory: String,
     pub test_with: ExternalRef,
     pub bench_with: Option<ExternalRef>,
+    pub criterion_config: Option<ExternalRef>,
     pub skip: bool,
     pub should_panic: bool,
 }
@@ -61,6 +62,7 @@ pub fn extract_attrs(ast: &syn::DeriveInput) -> Result<Config, Error> {
                     "directory" => Config { directory: value.clone(), ..config },
                     "test_with" => Config { test_with: ExternalRef::from(value.clone()), ..config },
                     "bench_with" => Config { bench_with: Some(ExternalRef::from(value.clone())), ..config },
+                    "criterion_config" => Config { criterion_config: Some(ExternalRef::from(value.clone())), ..config },
                     _ => panic!("{}", ERROR_MSG),
                 }
             },

jsontests/jsontests-derive/src/lib.rs

@@ -11,7 +11,7 @@ mod attr;
 mod tests;
 mod util;
 
-use attr::extract_attrs;
+use attr::{Config, extract_attrs};
 use tests::read_tests_from_dir;
 use util::*;
 
@@ -21,7 +21,7 @@ use itertools::Itertools;
 use syn::Ident;
 use proc_macro::TokenStream;
 
-#[proc_macro_derive(JsonTests, attributes(directory, test_with, bench_with, skip, should_panic))]
+#[proc_macro_derive(JsonTests, attributes(directory, test_with, bench_with, criterion_config, skip, should_panic))]
 pub fn json_tests(input: TokenStream) -> TokenStream {
     // Construct a string representation of the type definition
     let s = input.to_string();
@@ -43,60 +43,53 @@ fn impl_json_tests(ast: &syn::DeriveInput) -> Result<quote::Tokens, Error> {
     let config = extract_attrs(&ast)?;
     let tests = read_tests_from_dir(&config.directory)?;
     let mut tokens = quote::Tokens::new();
+    let mut bench_idents = Vec::new();
 
     // split tests into groups by filepath
     let tests = tests.group_by(|test| test.path.clone());
 
-    open_directory_module(&config, &mut tokens);
+    let dir_mod_name = open_directory_module(&config, &mut tokens);
+
+    // If behchmarking support is requested, import Criterion
+    if config.bench_with.is_some() {
+        tokens.append(quote! {
+            use criterion::Criterion;
+        })
+    }
 
     for (filepath, tests) in &tests {
         // If tests count in this file is 1, we don't need submodule
         let tests = tests.collect::<Vec<_>>();
         let need_file_submodule = tests.len() > 1;
-
+        let mut file_mod_name = None;
         if need_file_submodule {
-            open_file_module(&filepath, &mut tokens);
+            file_mod_name = Some(open_file_module(&filepath, &mut tokens));
+            // If behchmarking support is requested, import Criterion
+            if config.bench_with.is_some() {
+                tokens.append(quote! {
+                    use criterion::Criterion;
+                })
+            }
         }
 
         // Generate test function
         for test in tests {
-            let test_func_path = &config.test_with.path;
-            let test_func_name = &config.test_with.name;
             let name = sanitize_ident(&test.name);
             let name_ident = Ident::from(name.as_ref());
             let data = json::to_string(&test.data)?;
 
-            // generate test attrs
-            tokens.append(quote!{#[test]});
-            if config.should_panic {
-                tokens.append(quote!{#[should_panic]});
-            }
-
-            // generate test body
-            tokens.append(quote! {
-                fn #name_ident() {
-                    use #test_func_path;
-                    let data = #data;
-                    #test_func_name(#name, data);
-                }
-            });
-
-            // generate optional benchmark body
-            if let Some(ref bench) = config.bench_with {
-                let bench_func_path = &bench.path;
-                let bench_func_name = &bench.name;
-                let name = format!("bench_{}", name);
-                let name_ident = Ident::from(name.as_ref());
-
-                tokens.append(quote! {
-                    #[bench]
-                    fn #name_ident(b: &mut test::Bencher) {
-                        use #bench_func_path;
-                        let data = #data;
-                        #bench_func_name(b, #name, data);
+            generate_test(&config, &name_ident, &data, &mut tokens);
+            generate_bench(&config, &name_ident, &data, &mut tokens)
+                .map(|mut ident| {
+                    // prepend dir submodule
+                    ident = Ident::from(format!("{}::{}", dir_mod_name, ident.as_ref()));
+                    // prepend file submodule
+                    if need_file_submodule {
+                        ident = Ident::from(format!("{}::{}", file_mod_name.as_ref().unwrap(), ident.as_ref()));
                     }
-                })
-            }
+                    bench_idents.push(ident);
+                });
+
         }
 
         if need_file_submodule {
@@ -108,8 +101,68 @@ fn impl_json_tests(ast: &syn::DeriveInput) -> Result<quote::Tokens, Error> {
     // Close directory module
     close_brace(&mut tokens);
 
+    generate_criterion_macros(&config, &bench_idents, &mut tokens);
+
     Ok(tokens)
 }
 
+fn generate_test(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) {
+    let test_func_path = &config.test_with.path;
+    let test_func_name = &config.test_with.name;
+    let test_name_str = test_name.as_ref();
 
+    tokens.append(quote!{#[test]});
+    if config.should_panic {
+        tokens.append(quote!{#[should_panic]});
+    }
+
+    tokens.append(quote! {
+        fn #test_name() {
+            use #test_func_path;
+            let data = #data;
+            #test_func_name(#test_name_str, data);
+        }
+    });
+}
+
+fn generate_bench(config: &Config, test_name: &Ident, data: &str, tokens: &mut quote::Tokens) -> Option<Ident> {
+    if config.bench_with.is_none() {
+        return None
+    }
+
+    let bench = config.bench_with.as_ref().unwrap();
+    let bench_func_path = &bench.path;
+    let bench_func_name = &bench.name;
 
+    let bench_name = format!("bench_{}", test_name.as_ref());
+    let bench_ident = Ident::from(bench_name.as_ref());
+
+    tokens.append(quote! {
+        pub fn #bench_ident(c: &mut Criterion) {
+            use #bench_func_path;
+            let data = #data;
+            #bench_func_name(c, #bench_name, data);
+        }
+    });
+
+    Some(bench_ident)
+}
+
+fn generate_criterion_macros(config: &Config, benches: &[Ident], tokens: &mut quote::Tokens) {
+    // Generate criterion macros
+    if config.bench_with.is_some() {
+        let benches = benches.iter().map(AsRef::as_ref).join(" , ");
+        let config = config.criterion_config
+            .as_ref()
+            .map(|cfg| cfg.path.clone())
+            .unwrap_or_else(|| Ident::from("Criterion::default"));
+        let template = quote! {
+            criterion_group! {
+                name = main;
+                config = #config();
+                targets = TARGETS
+            };
+        };
+        tokens.append(template.as_ref().replace("TARGETS", &benches));
+    }
+}

jsontests/jsontests-derive/src/util.rs

@@ -3,29 +3,34 @@ use quote;
 
 use attr::Config;
 
-pub fn open_directory_module(config: &Config, tokens: &mut quote::Tokens) {
+pub fn open_directory_module(config: &Config, tokens: &mut quote::Tokens) -> String {
     // get the leaf directory name
     let dirname = config.directory.rsplit('/').next().unwrap();
 
     // create identifier
     let dirname = sanitize_ident(dirname);
-    let dirname = Ident::from(dirname);
+    let dirname_ident = Ident::from(dirname.as_ref());
 
-    open_module(&dirname, tokens);
+    open_module(dirname_ident, tokens);
+
+    dirname
 }
 
-pub fn open_file_module(filepath: &str, tokens: &mut quote::Tokens) {
+pub fn open_file_module(filepath: &str, tokens: &mut quote::Tokens) -> String {
     // get file name without extension
     let filename = filepath.rsplit('/').next().unwrap()
         .split('.').next().unwrap();
     // create identifier
     let filename = sanitize_ident(filename);
-    let filename = Ident::from(filename);
+    let filename_ident = Ident::from(filename.as_ref());
+
+    open_module(filename_ident, tokens);
 
-    open_module(&filename, tokens);
+    filename
 }
 
-pub fn open_module(module_name: &Ident, tokens: &mut quote::Tokens) {
+pub fn open_module<I: Into<Ident>>(module_name: I, tokens: &mut quote::Tokens) {
+    let module_name = module_name.into();
     // append module opening tokens
     tokens.append(quote! {
         mod #module_name

jsontests/src/lib.rs

@@ -1,14 +1,11 @@
-#![cfg_attr(feature = "bench", feature(test))]
 extern crate sputnikvm;
 extern crate serde_json;
 extern crate hexutil;
 extern crate bigint;
 extern crate env_logger;
 extern crate sha3;
 extern crate rlp;
-
-#[cfg(feature = "bench")]
-extern crate test;
+extern crate criterion;
 
 mod blockchain;
 pub mod util;
@@ -299,6 +296,25 @@ pub fn test_transaction(_name: &str, v: &Value, debug: bool) -> Result<bool, VMS
     }
 }
 
+use criterion::Criterion;
+
+pub fn bench_transaction(_name: &str, v: Value, c: &mut Criterion) {
+    c.bench_function(_name, move |b| {
+        b.iter_with_large_setup(|| {
+            let block = create_block(&v);
+            let history: Arc<Mutex<Vec<Context>>> = Arc::new(Mutex::new(Vec::new()));
+            let history_closure = history.clone();
+            let mut machine = create_machine(&v, &block);
+            machine.add_context_history_hook(move |context| {
+                history_closure.lock().unwrap().push(context.clone());
+            });
+            (machine, block)
+        }, |(mut machine, block)| {
+            fire_with_block(&mut machine, &block);
+        })
+    });
+}
+
 /// Read U256 number exactly the way go big.Int parses strings
 /// except for base 2 and 8 which are not used in tests
 pub fn read_u256(number: &str) -> U256 {

jsontests/src/util.rs

@@ -1,21 +1,16 @@
 use serde_json::Value;
 use serde_json as json;
 use test_transaction;
+use bench_transaction;
 
 pub fn run_test(name: &str, test: &str) {
     let test: Value = json::from_str(test).unwrap();
     assert_eq!(test_transaction(name, &test, true), Ok(true));
 }
 
-#[cfg(feature = "bench")]
-use test::Bencher;
+use criterion::Criterion;
 
-#[cfg(feature = "bench")]
-pub fn run_bench(b: &mut Bencher, name: &str, test: &str) {
+pub fn run_bench(c: &mut Criterion, name: &'static str, test: &str) {
     let test: Value = json::from_str(test).unwrap();
-    b.iter(|| {
-        // TODO: adjust test_transaction or write another function
-        // TODO: in order to start benchmark as close to actual sputnik code as possible
-        assert_eq!(test_transaction(name, &test, true), Ok(true));
-    })
+    bench_transaction(name, test, c);
 }

network/classic/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sputnikvm-network-classic"
-version = "0.10.0"
+version = "0.10.1"
 description = "Ethereum Classic patches for SputnikVM."
 license = "Apache-2.0"
 authors = ["Wei Tang <[email protected]>"]

network/ellaism/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sputnikvm-network-ellaism"
-version = "0.10.0"
+version = "0.10.1"
 description = "Ellaism patches for SputnikVM."
 license = "Apache-2.0"
 authors = ["Wei Tang <[email protected]>"]

network/expanse/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
 name = "sputnikvm-network-expanse"
-version = "0.10.0"
+version = "0.10.1"
 description = "Expanse patches for SputnikVM."
 license = "Apache-2.0"
 authors = ["Wei Tang <[email protected]>"]
 repository = "https://github.com/ethereumproject/sputnikvm"
 
 [dependencies]
 sputnikvm = { version = "0.10", path = "../..", default-features = false }
-sputnikvm-precompiled-bn128 = { version = "0.10", path = "../../precompiled/bn128", default-features = false}
-sputnikvm-precompiled-modexp = { version = "0.10", path = "../../precompiled/modexp", default-features = false }
+sputnikvm-precompiled-bn128 = { version = "0.10.1", path = "../../precompiled/bn128", default-features = false}
+sputnikvm-precompiled-modexp = { version = "0.10.1", path = "../../precompiled/modexp", default-features = false }
 etcommon-bigint = { version = "0.2", default-features = false }
 
 [features]