[1.0.2] Test regression of syncing and replay with actual committed reference blockchain data

libraries/testing/include/eosio/testing/tester.hpp

@@ -521,6 +521,13 @@ namespace eosio::testing {
             return {cfg, gen};
          }
 
+         static bool arguments_contains(std::string arg) {
+            auto argc = boost::unit_test::framework::master_test_suite().argc;
+            auto argv = boost::unit_test::framework::master_test_suite().argv;
+
+            return std::find(argv, argv + argc, arg) != (argv + argc);
+         }
+
          // ideally, users of `tester` should not access the controller directly,
          // so we provide APIs to access the chain head and fork_db head, and some
          // other commonly used APIs.
@@ -529,6 +536,7 @@ namespace eosio::testing {
          block_handle     fork_db_head() const { return control->fork_db_head(); }
 
          chain_id_type    get_chain_id() const { return control->get_chain_id(); }
+         block_id_type    last_irreversible_block_id() const { return control->last_irreversible_block_id(); }
          uint32_t         last_irreversible_block_num() const { return control->last_irreversible_block_num(); }
          bool             block_exists(const block_id_type& id) const { return  control->block_exists(id); }
 

unittests/savanna_misc_tests.cpp

@@ -1,4 +1,5 @@
 #include "savanna_cluster.hpp"
+#include <test-data.hpp>
 
 using namespace eosio::chain;
 using namespace eosio::testing;
@@ -591,18 +592,141 @@ BOOST_FIXTURE_TEST_CASE(validate_qc_requiring_finalizer_policies, savanna_cluste
 
 } FC_LOG_AND_RETHROW()
 
+static void save_blockchain_data(const std::filesystem::path& blocks_dir,
+                                 const block_id_type&         id,
+                                 const std::string&           snapshot) {
+   auto source_log_file   = blocks_dir / "blocks.log";
+   auto source_index_file = blocks_dir / "blocks.index";
+
+   std::filesystem::path test_data_path { UNITTEST_TEST_DATA_DIR };
+   auto consensus_blockchain_dir = test_data_path / "consensus_blockchain";
+   auto ref_log_file             = consensus_blockchain_dir / "blocks.log";
+   auto ref_index_file           = consensus_blockchain_dir / "blocks.index";
+   auto ref_id_file_name         = consensus_blockchain_dir / "id";
+   auto ref_snapshot_file_name   = consensus_blockchain_dir / "snapshot";
+
+   // save reference blocks log
+   std::filesystem::copy_file(source_log_file, ref_log_file, std::filesystem::        copy_options::overwrite_existing);
+   std::filesystem::copy_file(source_index_file, ref_index_file, std::filesystem::        copy_options::overwrite_existing);
+
+   // save reference block_id
+   fc::cfile ref_id_file;
+   ref_id_file.set_file_path(ref_id_file_name);
+   ref_id_file.open("wb");
+   ref_id_file.write(id.data(), id.data_size());
+   ref_id_file.close();
+
+   // save reference snapshot
+   fc::cfile snapshot_file;
+   snapshot_file.set_file_path(ref_snapshot_file_name);
+   snapshot_file.open("w");
+   snapshot_file.write(snapshot.data(), snapshot.size());
+   snapshot_file.close();
+}
+
+static std::vector<char> read_reference_file(std::string filename, const char* mode) {
+   std::filesystem::path test_data_path { UNITTEST_TEST_DATA_DIR };
+   auto consensus_blockchain_dir = test_data_path / "consensus_blockchain";
+   auto ref_file_name = consensus_blockchain_dir / filename;
+
+   fc::cfile ref_file;
+   ref_file.set_file_path(ref_file_name);
+   ref_file.open(mode);
+
+   ref_file.seek_end(0);
+   auto size = ref_file.tellp();
+
+   std::vector<char> buf;
+   buf.resize(size);
+
+   ref_file.seek(0);
+   ref_file.read(buf.data(), size);
+
+   ref_file.close();
+
+   return buf;
+}
+
+static block_id_type read_reference_id() {
+   std::vector<char> buf = read_reference_file("id", "rb");
+   return block_id_type(buf.data(), buf.size());
+}
+
+static std::string read_reference_snapshot() {
+   std::vector<char> buf = read_reference_file("snapshot", "r");
+   return std::string(buf.data(), buf.size());
+}
+
+static std::shared_ptr<tester> replay_reference_blockchain(const block_log& blog) {
+   // replay the reference blockchain and make sure LIB id in the replayed
+   // chain matches reference LIB id
+   // --------------------------------------------
+   fc::temp_directory temp_dir;
+   auto config = tester::default_config(temp_dir).first;
+
+   std::filesystem::path test_data_path { UNITTEST_TEST_DATA_DIR };
+   auto ref_blockchain_dir = test_data_path / "consensus_blockchain";
+
+   auto genesis = eosio::chain::block_log::extract_genesis_state(ref_blockchain_dir);
+   BOOST_REQUIRE(genesis);
+
+   std::filesystem::create_directories(config.blocks_dir);
+
+   std::filesystem::copy(ref_blockchain_dir / "blocks.log", config.blocks_dir / "blocks.log");
+   std::filesystem::copy(ref_blockchain_dir / "blocks.index", config.blocks_dir / "blocks.index");
+
+   // do a full block invariants check
+   config.force_all_checks = true;
+
+   // replay the reference blockchain
+   std::shared_ptr<tester> replay_chain = std::make_shared<tester>(config, *genesis);
+
+   auto ref_lib_id = blog.head_id();
+   BOOST_REQUIRE_EQUAL(*ref_lib_id, replay_chain->last_irreversible_block_id());
+
+   return replay_chain;
+}
+
+static void sync_replayed_blockchain(const std::shared_ptr<tester>& replay_chain, const block_log& blog) {
+   tester sync_chain;
+   std::filesystem::remove_all(sync_chain.get_config().state_dir);
+   std::filesystem::remove_all(sync_chain.get_config().blocks_dir);
+   sync_chain.close();
+
+   // start from reference snapshot
+   sync_chain.open(buffered_snapshot_suite::get_reader(read_reference_snapshot()));
+
+   // Sync with the replayed blockchain
+   while( sync_chain.fork_db_head().block_num() < replay_chain->fork_db_head().block_num() ) {
+      auto fb = replay_chain->fetch_block_by_number( sync_chain.fork_db_head().block_num()+1 );
+      sync_chain.push_block( fb );
+   }
+
+   // In syncing, use the head for checking as it advances further than LIB
+   auto head_block_num = sync_chain.head().block_num();
+   signed_block_ptr ref_block = blog.read_block_by_num(head_block_num);
+
+   BOOST_REQUIRE_EQUAL(ref_block->calculate_id(), sync_chain.head().id());
+}
 
 // ----------------------------------------------------------------------------------------------------
 // For issue #694, we need to change the finality core of the `block_header_state`, but we want to
 // ensure that this doesn't create a consensus incompatibility with Spring 1.0.0, so the blocks created
 // with newer versions remain compatible (and linkable) by Spring 1.0.0.
 // ----------------------------------------------------------------------------------------------------
-BOOST_FIXTURE_TEST_CASE(verify_spring_1_0_block_compatibitity, savanna_cluster::cluster_t) try {
+BOOST_FIXTURE_TEST_CASE(verify_block_compatibitity, savanna_cluster::cluster_t) try {
    using namespace savanna_cluster;
    auto& A=_nodes[0];
    const auto& tester_account = "tester"_n;
    //_debug_mode = true;
 
+   bool save_blockchain = tester::arguments_contains("--save-blockchain");
+
+   std::string snapshot;
+   if (save_blockchain) { // take a snapshot at the beginning
+      snapshot = A.snapshot();
+   }
+
    // update finalizer_policy with a new key for B
    // --------------------------------------------
    base_tester::finalizer_policy_input input;
@@ -681,10 +805,28 @@ BOOST_FIXTURE_TEST_CASE(verify_spring_1_0_block_compatibitity, savanna_cluster::
    BOOST_REQUIRE_EQUAL(qc_s(qc(b9)), strong_qc(b8));    // b9 claims a strong QC on b8
    BOOST_REQUIRE_EQUAL(A.lib_number, b6->block_num());  // b9 makes B6 final
 
-   // check that the block id of b9 match what we got with Spring 1.0.0
+   // check that the block id of b9 match what we got before.
    auto b9_id = b9->calculate_id();
-   BOOST_REQUIRE_EQUAL(b9_id, block_id_type{"00000013725f3d79bd4dd4091d0853d010a320f95240981711a942673ad87918"});
+   BOOST_REQUIRE_EQUAL(b9_id, read_reference_id());
+
+   if (save_blockchain) {
+      save_blockchain_data(A.get_config().blocks_dir, b9_id, snapshot);
+      return;
+   }
+
+   std::filesystem::path test_data_path { UNITTEST_TEST_DATA_DIR };
+   auto ref_blockchain_dir = test_data_path / "consensus_blockchain";
+   block_log blog(ref_blockchain_dir);
+
+   // replay the reference blockchain and make sure LIB id in the replayed
+   // chain matches reference LIB id
+   // --------------------------------------------
+   std::shared_ptr<tester> replay_chain = replay_reference_blockchain(blog);
 
+   // start another blockchain using snapshot, and sync with the blocks
+   // in the replayed blockchain as source
+   // -----------------------------------------------
+   sync_replayed_blockchain(replay_chain, blog);
 } FC_LOG_AND_RETHROW()
 
 /* -----------------------------------------------------------------------------------------------------