Testgen tester - Utilities to run multiple tests with logs and reports (

#547)

* #414: testgen tester -- utilities to run multiple tests with logs and reports

* #547 add missing file updates from #529

* fix merge typo

* TestEnv: change path parameters into AsRef<Path>

* change TestEnv::full_path to return PathBuf

* apply simplifications suggested by Romain

* apply simplification from Romain

* account for WOW Romain's suggestion on RefUnwindSafe

* address Romain's suggestion on TestEnv::cleanup

* cargo clippy

* update CHANGELOG.md

CHANGELOG.md

@@ -1,11 +1,13 @@
 ## Unreleased
 
+- Add testgen tester to factor out test execution from integration tests ([#524])
 - Add spec for the light client attack evidence handling ([#526])
 - Return RFC6962 hash for empty merkle tree ([#498])
 - The `tendermint`, `tendermint-rpc`, and `tendermint-light-client` crates now compile to WASM on the `wasm32-unknown-unknown` and `wasm32-wasi` targets ([#463])
 - Implement protobuf encoding/decoding of Tendermint Proto types ([#504])
 - Separate protobuf types from Rust domain types using the DomainType trait ([#535])
 
+[#524]: https://github.com/informalsystems/tendermint-rs/issues/524
 [#526]: https://github.com/informalsystems/tendermint-rs/issues/526
 [#498]: https://github.com/informalsystems/tendermint-rs/issues/498
 [#463]: https://github.com/informalsystems/tendermint-rs/issues/463

light-client/src/tests.rs

@@ -8,9 +8,14 @@ use tendermint_rpc as rpc;
 
 use crate::components::clock::Clock;
 use crate::components::io::{AtHeight, Io, IoError};
+use crate::components::verifier::{ProdVerifier, Verdict, Verifier};
+use crate::errors::Error;
 use crate::evidence::EvidenceReporter;
+use crate::light_client::{LightClient, Options};
+use crate::state::State;
 use contracts::contract_trait;
 use std::collections::HashMap;
+use std::time::Duration;
 use tendermint::block::Height as HeightStr;
 use tendermint::evidence::{Duration as DurationStr, Evidence};
 
@@ -148,6 +153,47 @@ impl MockEvidenceReporter {
     }
 }
 
+pub fn verify_single(
+    trusted_state: Trusted,
+    input: LightBlock,
+    trust_threshold: TrustThreshold,
+    trusting_period: Duration,
+    clock_drift: Duration,
+    now: Time,
+) -> Result<LightBlock, Verdict> {
+    let verifier = ProdVerifier::default();
+
+    let trusted_state = LightBlock::new(
+        trusted_state.signed_header,
+        trusted_state.next_validators.clone(),
+        trusted_state.next_validators,
+        default_peer_id(),
+    );
+
+    let options = Options {
+        trust_threshold,
+        trusting_period,
+        clock_drift,
+    };
+
+    let result = verifier.verify(&input, &trusted_state, &options, now);
+
+    match result {
+        Verdict::Success => Ok(input),
+        error => Err(error),
+    }
+}
+
+pub fn verify_bisection(
+    untrusted_height: Height,
+    light_client: &mut LightClient,
+    state: &mut State,
+) -> Result<Vec<LightBlock>, Error> {
+    light_client
+        .verify_to_target(untrusted_height, state)
+        .map(|_| state.get_trace(untrusted_height))
+}
+
 // -----------------------------------------------------------------------------
 // Everything below is a temporary workaround for the lack of `provider` field
 // in the light blocks serialized in the JSON fixtures.

light-client/tests/light_client.rs

@@ -5,14 +5,14 @@ use tendermint_light_client::{
     components::{
         io::{AtHeight, Io},
         scheduler,
-        verifier::{ProdVerifier, Verdict, Verifier},
+        verifier::ProdVerifier,
     },
     errors::{Error, ErrorKind},
     light_client::{LightClient, Options},
     state::State,
     store::{memory::MemoryStore, LightStore},
     tests::{Trusted, *},
-    types::{Height, LightBlock, Status, Time, TrustThreshold},
+    types::{LightBlock, Status, TrustThreshold},
 };
 
 use tendermint_testgen::Tester;
@@ -21,47 +21,6 @@ use tendermint_testgen::Tester;
 // https://github.com/informalsystems/conformance-tests
 const TEST_FILES_PATH: &str = "./tests/support/";
 
-fn verify_single(
-    trusted_state: Trusted,
-    input: LightBlock,
-    trust_threshold: TrustThreshold,
-    trusting_period: Duration,
-    clock_drift: Duration,
-    now: Time,
-) -> Result<LightBlock, Verdict> {
-    let verifier = ProdVerifier::default();
-
-    let trusted_state = LightBlock::new(
-        trusted_state.signed_header,
-        trusted_state.next_validators.clone(),
-        trusted_state.next_validators,
-        default_peer_id(),
-    );
-
-    let options = Options {
-        trust_threshold,
-        trusting_period,
-        clock_drift,
-    };
-
-    let result = verifier.verify(&input, &trusted_state, &options, now);
-
-    match result {
-        Verdict::Success => Ok(input),
-        error => Err(error),
-    }
-}
-
-fn verify_bisection(
-    untrusted_height: Height,
-    light_client: &mut LightClient,
-    state: &mut State,
-) -> Result<Vec<LightBlock>, Error> {
-    light_client
-        .verify_to_target(untrusted_height, state)
-        .map(|_| state.get_trace(untrusted_height))
-}
-
 struct BisectionTestResult {
     untrusted_light_block: LightBlock,
     new_states: Result<Vec<LightBlock>, Error>,
@@ -229,17 +188,17 @@ fn bisection_lower_test(tc: TestBisection<AnonLightBlock>) {
 
 #[test]
 fn run_single_step_tests() {
-    let mut tester = Tester::new(TEST_FILES_PATH);
+    let mut tester = Tester::new("single_step", TEST_FILES_PATH);
     tester.add_test("single-step test", single_step_test);
     tester.run_foreach_in_dir("single_step");
-    tester.print_results();
+    tester.finalize();
 }
 
 #[test]
 fn run_bisection_tests() {
-    let mut tester = Tester::new(TEST_FILES_PATH);
+    let mut tester = Tester::new("bisection", TEST_FILES_PATH);
     tester.add_test("bisection test", bisection_test);
     tester.add_test("bisection lower test", bisection_lower_test);
     tester.run_foreach_in_dir("bisection/single_peer");
-    tester.print_results();
+    tester.finalize();
 }

light-client/tests/supervisor.rs

@@ -121,8 +121,8 @@ fn run_multipeer_test(tc: TestBisection<AnonLightBlock>) {
 
 #[test]
 fn run_multipeer_tests() {
-    let mut tester = Tester::new(TEST_FILES_PATH);
+    let mut tester = Tester::new("bisection_multi_peer", TEST_FILES_PATH);
     tester.add_test("multipeer test", run_multipeer_test);
     tester.run_foreach_in_dir("bisection/multi_peer");
-    tester.print_results();
+    tester.finalize();
 }

testgen/Cargo.toml

@@ -11,6 +11,7 @@ serde_json = "1"
 ed25519-dalek = "1"
 gumdrop = "0.8.0"
 simple-error = "0.2.1"
+tempfile = "3.1.0"
 
 [[bin]]
 name = "tendermint-testgen"