From 26ec80362db8c8b983b387be9cbcb9daf62850b1 Mon Sep 17 00:00:00 2001 From: Andrey Kuprianov <59489470+andrey-kuprianov@users.noreply.github.com> Date: Thu, 10 Sep 2020 17:58:53 +0200 Subject: [PATCH] Testgen tester - Utilities to run multiple tests with logs and reports (#547) * #414: testgen tester -- utilities to run multiple tests with logs and reports * #547 add missing file updates from #529 * fix merge typo * TestEnv: change path parameters into AsRef * change TestEnv::full_path to return PathBuf * apply simplifications suggested by Romain * apply simplification from Romain * account for WOW Romain's suggestion on RefUnwindSafe * address Romain's suggestion on TestEnv::cleanup * cargo clippy * update CHANGELOG.md --- CHANGELOG.md | 2 + light-client/src/tests.rs | 46 ++++ light-client/tests/light_client.rs | 53 +--- light-client/tests/supervisor.rs | 4 +- testgen/Cargo.toml | 1 + testgen/src/tester.rs | 416 +++++++++++++++++++++-------- 6 files changed, 358 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c03492bde..28cef06e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ ## Unreleased +- Add testgen tester to factor out test execution from integration tests ([#524]) - Add spec for the light client attack evidence handling ([#526]) - Return RFC6962 hash for empty merkle tree ([#498]) - The `tendermint`, `tendermint-rpc`, and `tendermint-light-client` crates now compile to WASM on the `wasm32-unknown-unknown` and `wasm32-wasi` targets ([#463]) - Implement protobuf encoding/decoding of Tendermint Proto types ([#504]) - Separate protobuf types from Rust domain types using the DomainType trait ([#535]) +[#524]: https://github.com/informalsystems/tendermint-rs/issues/524 [#526]: https://github.com/informalsystems/tendermint-rs/issues/526 [#498]: https://github.com/informalsystems/tendermint-rs/issues/498 [#463]: https://github.com/informalsystems/tendermint-rs/issues/463 diff --git a/light-client/src/tests.rs b/light-client/src/tests.rs index dcc407bcb..6ad1a400e 100644 --- a/light-client/src/tests.rs +++ b/light-client/src/tests.rs @@ -8,9 +8,14 @@ use tendermint_rpc as rpc; use crate::components::clock::Clock; use crate::components::io::{AtHeight, Io, IoError}; +use crate::components::verifier::{ProdVerifier, Verdict, Verifier}; +use crate::errors::Error; use crate::evidence::EvidenceReporter; +use crate::light_client::{LightClient, Options}; +use crate::state::State; use contracts::contract_trait; use std::collections::HashMap; +use std::time::Duration; use tendermint::block::Height as HeightStr; use tendermint::evidence::{Duration as DurationStr, Evidence}; @@ -148,6 +153,47 @@ impl MockEvidenceReporter { } } +pub fn verify_single( + trusted_state: Trusted, + input: LightBlock, + trust_threshold: TrustThreshold, + trusting_period: Duration, + clock_drift: Duration, + now: Time, +) -> Result { + let verifier = ProdVerifier::default(); + + let trusted_state = LightBlock::new( + trusted_state.signed_header, + trusted_state.next_validators.clone(), + trusted_state.next_validators, + default_peer_id(), + ); + + let options = Options { + trust_threshold, + trusting_period, + clock_drift, + }; + + let result = verifier.verify(&input, &trusted_state, &options, now); + + match result { + Verdict::Success => Ok(input), + error => Err(error), + } +} + +pub fn verify_bisection( + untrusted_height: Height, + light_client: &mut LightClient, + state: &mut State, +) -> Result, Error> { + light_client + .verify_to_target(untrusted_height, state) + .map(|_| state.get_trace(untrusted_height)) +} + // ----------------------------------------------------------------------------- // Everything below is a temporary workaround for the lack of `provider` field // in the light blocks serialized in the JSON fixtures. diff --git a/light-client/tests/light_client.rs b/light-client/tests/light_client.rs index 8c6d1146c..d618d2350 100644 --- a/light-client/tests/light_client.rs +++ b/light-client/tests/light_client.rs @@ -5,14 +5,14 @@ use tendermint_light_client::{ components::{ io::{AtHeight, Io}, scheduler, - verifier::{ProdVerifier, Verdict, Verifier}, + verifier::ProdVerifier, }, errors::{Error, ErrorKind}, light_client::{LightClient, Options}, state::State, store::{memory::MemoryStore, LightStore}, tests::{Trusted, *}, - types::{Height, LightBlock, Status, Time, TrustThreshold}, + types::{LightBlock, Status, TrustThreshold}, }; use tendermint_testgen::Tester; @@ -21,47 +21,6 @@ use tendermint_testgen::Tester; // https://github.com/informalsystems/conformance-tests const TEST_FILES_PATH: &str = "./tests/support/"; -fn verify_single( - trusted_state: Trusted, - input: LightBlock, - trust_threshold: TrustThreshold, - trusting_period: Duration, - clock_drift: Duration, - now: Time, -) -> Result { - let verifier = ProdVerifier::default(); - - let trusted_state = LightBlock::new( - trusted_state.signed_header, - trusted_state.next_validators.clone(), - trusted_state.next_validators, - default_peer_id(), - ); - - let options = Options { - trust_threshold, - trusting_period, - clock_drift, - }; - - let result = verifier.verify(&input, &trusted_state, &options, now); - - match result { - Verdict::Success => Ok(input), - error => Err(error), - } -} - -fn verify_bisection( - untrusted_height: Height, - light_client: &mut LightClient, - state: &mut State, -) -> Result, Error> { - light_client - .verify_to_target(untrusted_height, state) - .map(|_| state.get_trace(untrusted_height)) -} - struct BisectionTestResult { untrusted_light_block: LightBlock, new_states: Result, Error>, @@ -229,17 +188,17 @@ fn bisection_lower_test(tc: TestBisection) { #[test] fn run_single_step_tests() { - let mut tester = Tester::new(TEST_FILES_PATH); + let mut tester = Tester::new("single_step", TEST_FILES_PATH); tester.add_test("single-step test", single_step_test); tester.run_foreach_in_dir("single_step"); - tester.print_results(); + tester.finalize(); } #[test] fn run_bisection_tests() { - let mut tester = Tester::new(TEST_FILES_PATH); + let mut tester = Tester::new("bisection", TEST_FILES_PATH); tester.add_test("bisection test", bisection_test); tester.add_test("bisection lower test", bisection_lower_test); tester.run_foreach_in_dir("bisection/single_peer"); - tester.print_results(); + tester.finalize(); } diff --git a/light-client/tests/supervisor.rs b/light-client/tests/supervisor.rs index ce8098d2a..a5867ff3c 100644 --- a/light-client/tests/supervisor.rs +++ b/light-client/tests/supervisor.rs @@ -121,8 +121,8 @@ fn run_multipeer_test(tc: TestBisection) { #[test] fn run_multipeer_tests() { - let mut tester = Tester::new(TEST_FILES_PATH); + let mut tester = Tester::new("bisection_multi_peer", TEST_FILES_PATH); tester.add_test("multipeer test", run_multipeer_test); tester.run_foreach_in_dir("bisection/multi_peer"); - tester.print_results(); + tester.finalize(); } diff --git a/testgen/Cargo.toml b/testgen/Cargo.toml index 42855ed5e..5e1d2b5b6 100644 --- a/testgen/Cargo.toml +++ b/testgen/Cargo.toml @@ -11,6 +11,7 @@ serde_json = "1" ed25519-dalek = "1" gumdrop = "0.8.0" simple-error = "0.2.1" +tempfile = "3.1.0" [[bin]] name = "tendermint-testgen" diff --git a/testgen/src/tester.rs b/testgen/src/tester.rs index a91a4884d..114b15236 100644 --- a/testgen/src/tester.rs +++ b/testgen/src/tester.rs @@ -1,49 +1,103 @@ use crate::helpers::*; use crate::tester::TestResult::{Failure, ParseError, ReadError, Success}; use serde::de::DeserializeOwned; -use std::panic::UnwindSafe; -use std::{fs, path::PathBuf}; use std::{ - panic, + fs::{self, DirEntry}, + io::Write, + panic::{self, RefUnwindSafe, UnwindSafe}, + path::{Path, PathBuf}, sync::{Arc, Mutex}, }; +use tempfile::TempDir; +/// A test environment, which is essentially a wrapper around some directory, +/// with some utility functions operating relative to that directory. #[derive(Debug, Clone)] pub struct TestEnv { - root_dir: String, - logs: Vec, + /// Directory where the test is being executed + current_dir: String, } impl TestEnv { - pub fn add_log(&mut self, log: &str) { - self.logs.push(log.to_string()); + pub fn new(current_dir: &str) -> Option { + fs::create_dir_all(current_dir).ok().map(|_| TestEnv { + current_dir: current_dir.to_string(), + }) } - /// Read a file from a path relative to the environment root into a string - pub fn read_file(&mut self, path: &str) -> Option { - match self.full_path(path) { - None => None, - Some(full_path) => match fs::read_to_string(&full_path) { - Ok(file) => Some(file), - Err(_) => None, - }, + pub fn push(&self, child: &str) -> Option { + let mut path = PathBuf::from(&self.current_dir); + path.push(child); + path.to_str().and_then(|path| TestEnv::new(path)) + } + + pub fn current_dir(&self) -> &str { + &self.current_dir + } + + pub fn logln(&self, msg: &str) -> Option<()> { + println!("{}", msg); + fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.full_path("log")) + .ok() + .and_then(|mut file| writeln!(file, "{}", msg).ok()) + } + + pub fn logln_to(&self, msg: &str, rel_path: impl AsRef) -> Option<()> { + println!("{}", msg); + fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.full_path(rel_path)) + .ok() + .and_then(|mut file| writeln!(file, "{}", msg).ok()) + } + + /// Read a file from a path relative to the environment current dir into a string + pub fn read_file(&self, rel_path: impl AsRef) -> Option { + fs::read_to_string(self.full_path(rel_path)).ok() + } + + /// Write a file to a path relative to the environment current dir + pub fn write_file(&self, rel_path: impl AsRef, contents: &str) -> Option<()> { + fs::write(self.full_path(rel_path), contents).ok() + } + + /// Parse a file from a path relative to the environment current dir as the given type + pub fn parse_file(&self, rel_path: impl AsRef) -> Option { + self.read_file(rel_path) + .and_then(|input| serde_json::from_str(&input).ok()) + } + + /// Copy a file from the path outside environment into the environment current dir + /// Returns None if copying was not successful + pub fn copy_file_from(&self, path: impl AsRef) -> Option<()> { + let path = path.as_ref(); + if !path.is_file() { + return None; } + let name = path.file_name()?.to_str()?; + fs::copy(path, self.full_path(name)).ok().map(|_| ()) + } + + /// Copy a file from the path relative to the other environment into the environment current dir + /// Returns None if copying was not successful + pub fn copy_file_from_env(&self, other: &TestEnv, path: impl AsRef) -> Option<()> { + self.copy_file_from(other.full_path(path)) } /// Convert a relative path to the full path from the test root /// Return None if the full path can't be formed - pub fn full_path(&self, rel_path: &str) -> Option { - let full_path = PathBuf::from(&self.root_dir).join(rel_path); - match full_path.to_str() { - None => None, - Some(full_path) => Some(full_path.to_string()), - } + pub fn full_path(&self, rel_path: impl AsRef) -> PathBuf { + PathBuf::from(&self.current_dir).join(rel_path) } /// Convert a full path to the path relative to the test root - /// Return None if the full path doesn't contain test root as prefix - pub fn rel_path(&self, full_path: &str) -> Option { - match PathBuf::from(full_path).strip_prefix(&self.root_dir) { + /// Returns None if the full path doesn't contain test root as prefix + pub fn rel_path(&self, full_path: impl AsRef) -> Option { + match PathBuf::from(full_path.as_ref()).strip_prefix(&self.current_dir) { Err(_) => None, Ok(rel_path) => match rel_path.to_str() { None => None, @@ -51,27 +105,70 @@ impl TestEnv { }, } } -} -type TestFn = Box TestResult>; - -pub struct Tester { - root_dir: String, - tests: Vec<(String, TestFn)>, - results: std::collections::BTreeMap>, + /// Convert a relative path to the full path from the test root, canonicalized + /// Returns None the full path can't be formed + pub fn full_canonical_path(&self, rel_path: impl AsRef) -> Option { + let full_path = PathBuf::from(&self.current_dir).join(rel_path); + full_path + .canonicalize() + .ok() + .and_then(|p| p.to_str().map(|x| x.to_string())) + } } #[derive(Debug, Clone)] pub enum TestResult { ReadError, ParseError, - Success(TestEnv), + Success, Failure { message: String, location: String }, } +/// A function that takes as input the test file path and its content, +/// and returns the result of running the test on it +type TestFn = Box TestResult>; + +/// A function that takes as input the batch file path and its content, +/// and returns the vector of test names/contents for tests in the batch, +/// or None if the batch could not be parsed +type BatchFn = Box Option>>; + +pub struct Test { + /// test name + pub name: String, + /// test function + pub test: TestFn, +} + +/// Tester allows you to easily run some test functions over a set of test files. +/// You create a Tester instance with the reference to some specific directory, containing your test files. +/// After a creation, you can add several types of tests there: +/// * add_test() adds a simple test function, which can run on some test, deserilizable from a file. +/// * add_test_with_env() allows your test function to receive several test environments, +/// so that it can easily perform some operations on files when necessary +/// * add_test_batch() adds a batch of test: a function that accepts a ceserializable batch description, +/// and produces a set of test from it +/// +/// After you have added all your test functions, you run Tester either on individual files +/// using run_for_file(), or for whole directories, using run_foreach_in_dir(); +/// the directories will be traversed recursively top-down. +/// +/// The last step involves calling the finalize() function, which will produce the test report +/// and panic in case there was at least one failing test. +/// When there are files in the directories you run Tester on, that could not be read/parsed, +/// it is also considered an error, and leads to panic. +pub struct Tester { + name: String, + root_dir: String, + tests: Vec, + batches: Vec, + results: std::collections::BTreeMap>, +} + impl TestResult { pub fn is_success(&self) -> bool { - matches!(self, TestResult::Success(_)) + matches!(self, TestResult::Success) } pub fn is_failure(&self) -> bool { matches!(self, TestResult::Failure { @@ -88,24 +185,30 @@ impl TestResult { } impl Tester { - pub fn new(root_dir: &str) -> Tester { + pub fn new(name: &str, root_dir: &str) -> Tester { Tester { + name: name.to_string(), root_dir: root_dir.to_string(), tests: vec![], + batches: vec![], results: Default::default(), } } - pub fn env(&self) -> TestEnv { - TestEnv { - root_dir: self.root_dir.clone(), - logs: vec![], - } + pub fn env(&self) -> Option { + TestEnv::new(&self.root_dir) + } + + pub fn output_env(&self) -> Option { + let output_dir = self.root_dir.clone() + "/_" + &self.name; + fs::create_dir_all(&output_dir) + .ok() + .and(TestEnv::new(&output_dir)) } - fn capture_test(env: TestEnv, test: F) -> TestResult + fn capture_test(test: F) -> TestResult where - F: FnOnce(TestEnv) -> TestEnv + UnwindSafe, + F: FnOnce() + UnwindSafe, { let test_result = Arc::new(Mutex::new(ParseError)); let old_hook = panic::take_hook(); @@ -127,73 +230,94 @@ impl Tester { *result = Failure { message, location }; }) }); - let test_fun = || test(env.clone()); - let result = panic::catch_unwind(test_fun); + let result = panic::catch_unwind(|| test()); panic::set_hook(old_hook); match result { - Ok(res) => Success(res), + Ok(_) => Success, Err(_) => (*test_result.lock().unwrap()).clone(), } } - pub fn add_test(&mut self, name: &str, test: fn(T)) + pub fn add_test(&mut self, name: &str, test: F) where T: 'static + DeserializeOwned + UnwindSafe, + F: Fn(T) + UnwindSafe + RefUnwindSafe + 'static, { - let test_env = self.env(); - let test_fn = move |input: &str| match parse_as::(&input) { - Ok(test_case) => Tester::capture_test(test_env.clone(), |env| { + let test_fn = move |_path: &str, input: &str| match parse_as::(&input) { + Ok(test_case) => Tester::capture_test(|| { test(test_case); - env }), Err(_) => ParseError, }; - self.tests.push((name.to_string(), Box::new(test_fn))); + self.tests.push(Test { + name: name.to_string(), + test: Box::new(test_fn), + }); } - pub fn add_test_with_env(&mut self, name: &str, test: fn(T, &mut TestEnv)) + pub fn add_test_with_env(&mut self, name: &str, test: F) where T: 'static + DeserializeOwned + UnwindSafe, + F: Fn(T, &TestEnv, &TestEnv, &TestEnv) + UnwindSafe + RefUnwindSafe + 'static, { - let test_env = self.env(); - let test_fn = move |input: &str| match parse_as::(&input) { - Ok(test_case) => Tester::capture_test(test_env.clone(), |env| { - let mut env = env; - test(test_case, &mut env); - env + let test_env = self.env().unwrap(); + let output_env = self.output_env().unwrap(); + let test_fn = move |path: &str, input: &str| match parse_as::(&input) { + Ok(test_case) => Tester::capture_test(|| { + // It is OK to unwrap() here: in case of unwrapping failure, the test will fail. + let dir = TempDir::new().unwrap(); + let env = TestEnv::new(dir.path().to_str().unwrap()).unwrap(); + let output_dir = output_env.full_path(path); + let output_env = TestEnv::new(output_dir.to_str().unwrap()).unwrap(); + fs::remove_dir_all(&output_dir).unwrap(); + test(test_case, &env, &test_env, &output_env); }), Err(_) => ParseError, }; - self.tests.push((name.to_string(), Box::new(test_fn))); + self.tests.push(Test { + name: name.to_string(), + test: Box::new(test_fn), + }); } - fn add_result(&mut self, name: &str, path: &str, result: TestResult) { + pub fn add_test_batch(&mut self, batch: F) + where + T: 'static + DeserializeOwned, + F: Fn(T) -> Vec<(String, String)> + 'static, + { + let batch_fn = move |_path: &str, input: &str| match parse_as::(&input) { + Ok(test_batch) => Some(batch(test_batch)), + Err(_) => None, + }; + self.batches.push(Box::new(batch_fn)); + } + + fn results_for(&mut self, name: &str) -> &mut Vec<(String, TestResult)> { self.results .entry(name.to_string()) .or_insert_with(Vec::new) - .push((path.to_string(), result)) + } + + fn add_result(&mut self, name: &str, path: &str, result: TestResult) { + self.results_for(name).push((path.to_string(), result)); } fn read_error(&mut self, path: &str) { - self.results - .entry("".to_string()) - .or_insert_with(Vec::new) + self.results_for("") .push((path.to_string(), TestResult::ReadError)) } fn parse_error(&mut self, path: &str) { - self.results - .entry("".to_string()) - .or_insert_with(Vec::new) + self.results_for("") .push((path.to_string(), TestResult::ParseError)) } - pub fn successful_tests(&self, test: &str) -> Vec<(String, TestEnv)> { + pub fn successful_tests(&self, test: &str) -> Vec { let mut tests = Vec::new(); if let Some(results) = self.results.get(test) { for (path, res) in results { - if let Success(env) = res { - tests.push((path.clone(), env.clone())) + if let Success = res { + tests.push(path.clone()) } } } @@ -236,68 +360,62 @@ impl Tester { tests } - pub fn print_results(&mut self) { - let tests = self.unreadable_tests(); - if !tests.is_empty() { - println!("Unreadable tests: "); - for path in tests { - println!(" > {}", path) + fn run_for_input(&mut self, path: &str, input: &str) { + let mut results = Vec::new(); + for Test { name, test } in &self.tests { + match test(path, input) { + TestResult::ParseError => continue, + res => results.push((name.to_string(), path, res)), } - panic!("Some tests could not be read"); } - let tests = self.unparseable_tests(); - if !tests.is_empty() { - println!("Unparseable tests: "); - for path in tests { - println!(" > {}", path) + if !results.is_empty() { + for (name, path, res) in results { + self.add_result(&name, path, res) } - panic!("Some tests could not be parsed"); - } - - for name in self.results.keys() { - println!("Results for '{}'", name); - let tests = self.successful_tests(name); - if !tests.is_empty() { - println!(" Successful tests: "); - for (path, _) in tests { - println!(" > {}", path) + } else { + // parsing as a test failed; try parse as a batch + let mut res_tests = Vec::new(); + for batch in &self.batches { + match batch(path, input) { + None => continue, + Some(tests) => { + for (name, input) in tests { + let test_path = path.to_string() + "/" + &name; + res_tests.push((test_path, input)); + } + } } } - let tests = self.failed_tests(name); - if !tests.is_empty() { - println!(" Failed tests: "); - for (path, message, location) in tests { - println!(" > {}, '{}', {}", path, message, location) + if !res_tests.is_empty() { + for (path, input) in res_tests { + self.run_for_input(&path, &input); } - panic!("Some tests failed"); + } else { + // parsing both as a test and as a batch failed + self.parse_error(path); } } } pub fn run_for_file(&mut self, path: &str) { - match self.env().read_file(path) { + match self.env().unwrap().read_file(path) { None => self.read_error(path), - Some(input) => { - let mut results = Vec::new(); - for (name, test) in &self.tests { - match test(&input) { - TestResult::ParseError => continue, - res => results.push((name.to_string(), path, res)), - } - } - if results.is_empty() { - self.parse_error(path); - } else { - for (name, path, res) in results { - self.add_result(&name, path, res) - } - } - } + Some(input) => self.run_for_input(path, &input), } } pub fn run_foreach_in_dir(&mut self, dir: &str) { let full_dir = PathBuf::from(&self.root_dir).join(dir); + let starts_with_underscore = |entry: &DirEntry| { + if let Some(last) = entry.path().iter().rev().next() { + if let Some(last) = last.to_str() { + if last.starts_with('_') { + return true; + } + } + } + false + }; match full_dir.to_str() { None => self.read_error(dir), Some(full_dir) => match fs::read_dir(full_dir) { @@ -305,9 +423,13 @@ impl Tester { Ok(paths) => { for path in paths { if let Ok(entry) = path { + // ignore path components starting with '_' + if starts_with_underscore(&entry) { + continue; + } if let Ok(kind) = entry.file_type() { let path = format!("{}", entry.path().display()); - let rel_path = self.env().rel_path(&path).unwrap(); + let rel_path = self.env().unwrap().rel_path(&path).unwrap(); if kind.is_file() || kind.is_symlink() { if !rel_path.ends_with(".json") { continue; @@ -324,4 +446,68 @@ impl Tester { }, } } + + pub fn finalize(&mut self) { + let env = self.output_env().unwrap(); + env.write_file("report", ""); + let print = |msg: &str| { + env.logln_to(msg, "report"); + }; + let mut do_panic = false; + + print(&format!( + "\n====== Report for '{}' tester run ======", + &self.name + )); + for name in self.results.keys() { + if name.is_empty() { + continue; + } + print(&format!("\nResults for '{}'", name)); + let tests = self.successful_tests(name); + if !tests.is_empty() { + print(" Successful tests: "); + for path in tests { + print(&format!(" {}", path)); + if let Some(logs) = env.read_file(&(path + "/log")) { + print(&logs) + } + } + } + let tests = self.failed_tests(name); + if !tests.is_empty() { + do_panic = true; + print(" Failed tests: "); + for (path, message, location) in tests { + print(&format!(" {}, '{}', {}", path, message, location)); + if let Some(logs) = env.read_file(&(path + "/log")) { + print(&logs) + } + } + } + } + let tests = self.unreadable_tests(); + if !tests.is_empty() { + do_panic = true; + print("\nUnreadable tests: "); + for path in tests { + print(&format!(" {}", path)) + } + } + let tests = self.unparseable_tests(); + if !tests.is_empty() { + do_panic = true; + print("\nUnparseable tests: "); + for path in tests { + print(&format!(" {}", path)) + } + } + print(&format!( + "\n====== End of report for '{}' tester run ======\n", + &self.name + )); + if do_panic { + panic!("Some tests failed or could not be read/parsed"); + } + } }