Blocks log replay fix, enhance libtester to support customizing startup options, and add blocks log replay tests

libraries/chain/controller.cpp

@@ -1606,7 +1606,10 @@ struct controller_impl {
                      // note if is_proper_svnn_block is not reached then transistion will happen live
                   }
                });
-               if( check_shutdown() ) break;
+               if( check_shutdown() ) {
+                  ilog( "quitting from replay_block_log because of shutdown" );
+                  break;
+               }
                if( next->block_num() % 500 == 0 ) {
                   ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) );
                }
@@ -1644,6 +1647,11 @@ struct controller_impl {
          ilog( "no block log found" );
       }
 
+      if( check_shutdown() ) {
+         ilog( "quitting from replay because of shutdown" );
+         return;
+      }
+
       try {
          if (startup != startup_t::existing_state)
            open_fork_db();
@@ -1721,7 +1729,6 @@ struct controller_impl {
             auto branch = fork_db.fetch_branch_from_head();
             int rev = 0;
             for( auto i = branch.rbegin(); i != branch.rend(); ++i ) {
-               if( check_shutdown() ) break;
                if( (*i)->block_num() <= head_block_num ) continue;
                ++rev;
                replay_push_block<BSP>( *i, controller::block_status::validated );

libraries/testing/include/eosio/testing/tester.hpp

@@ -170,7 +170,7 @@ namespace eosio::testing {
 
          void              init(const setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::HEAD, std::optional<uint32_t> genesis_max_inline_action_size = std::optional<uint32_t>{});
          void              init(controller::config config, const snapshot_reader_ptr& snapshot);
-         void              init(controller::config config, const genesis_state& genesis);
+         void              init(controller::config config, const genesis_state& genesis, bool do_startup = true);
          void              init(controller::config config);
          void              init(controller::config config, protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot);
          void              init(controller::config config, protocol_feature_set&& pfs, const genesis_state& genesis);
@@ -180,10 +180,10 @@ namespace eosio::testing {
          void              close();
          void              open( protocol_feature_set&& pfs, std::optional<chain_id_type> expected_chain_id, const std::function<void()>& lambda );
          void              open( protocol_feature_set&& pfs, const snapshot_reader_ptr& snapshot );
-         void              open( protocol_feature_set&& pfs, const genesis_state& genesis );
+         void              open( protocol_feature_set&& pfs, const genesis_state& genesis, bool do_startup = true );
          void              open( protocol_feature_set&& pfs, std::optional<chain_id_type> expected_chain_id = {} );
          void              open( const snapshot_reader_ptr& snapshot );
-         void              open( const genesis_state& genesis );
+         void              open( const genesis_state& genesis, bool do_startup = true );
          void              open( std::optional<chain_id_type> expected_chain_id = {} );
          bool              is_same_chain( base_tester& other );
 
@@ -553,8 +553,15 @@ namespace eosio::testing {
          init(policy, read_mode, genesis_max_inline_action_size);
       }
 
-      tester(controller::config config, const genesis_state& genesis) {
-         init(std::move(config), genesis);
+      // If `do_startup` is true, tester starts the chain during initialization.
+      //
+      // If `do_startup` is true, tester does NOT start the chain during initialization;
+      // the user must call `startup()` explicitly.
+      // Before calling `startup()`, the user can do additional setups like connecting
+      // to a particular signal, and customizing shutdown conditions.
+      // See blocks_log_replay_tests.cpp in unit_test for an example.
+      tester(controller::config config, const genesis_state& genesis, bool do_startup = true) {
+         init(std::move(config), genesis, do_startup);
       }
 
       tester(controller::config config) {

libraries/testing/tester.cpp

@@ -199,9 +199,9 @@ namespace eosio::testing {
       open(snapshot);
    }
 
-   void base_tester::init(controller::config config, const genesis_state& genesis) {
+   void base_tester::init(controller::config config, const genesis_state& genesis, bool do_startup) {
       cfg = std::move(config);
-      open(genesis);
+      open(genesis, do_startup);
    }
 
    void base_tester::init(controller::config config) {
@@ -302,8 +302,8 @@ namespace eosio::testing {
       open( make_protocol_feature_set(), snapshot );
    }
 
-   void base_tester::open( const genesis_state& genesis ) {
-      open( make_protocol_feature_set(), genesis );
+   void base_tester::open( const genesis_state& genesis, bool do_startup ) {
+      open( make_protocol_feature_set(), genesis, do_startup );
    }
 
    void base_tester::open( std::optional<chain_id_type> expected_chain_id ) {
@@ -358,10 +358,14 @@ namespace eosio::testing {
       });
    }
 
-   void base_tester::open( protocol_feature_set&& pfs, const genesis_state& genesis ) {
-      open(std::move(pfs), genesis.compute_chain_id(), [&genesis,&control=this->control]() {
-         control->startup( [](){}, []() { return false; }, genesis );
-      });
+   void base_tester::open( protocol_feature_set&& pfs, const genesis_state& genesis, bool do_startup ) {
+      if (do_startup) {
+         open(std::move(pfs), genesis.compute_chain_id(), [&genesis,&control=this->control]() {
+            control->startup( [](){}, []() { return false; }, genesis );
+         });
+      } else {
+         open(std::move(pfs), genesis.compute_chain_id(), nullptr);
+      }
    }
 
    void base_tester::open( protocol_feature_set&& pfs, std::optional<chain_id_type> expected_chain_id ) {

unittests/blocks_log_replay_tests.cpp

@@ -0,0 +1,152 @@
+#include <eosio/testing/tester.hpp>
+#include <boost/test/unit_test.hpp>
+
+// Test scenarios
+//    * replay through blocks log and irreverible blocks
+//    * replay stopping in the middle of blocks log and resuming
+//    * replay stopping in the middle of irreverible blocks and resuming
+//
+
+BOOST_AUTO_TEST_SUITE(blocks_log_replay_tests)
+
+using namespace eosio::testing;
+using namespace eosio::chain;
+
+struct blog_replay_fixture {
+   eosio::testing::tester chain;
+   uint32_t               last_head_block_num {0}; // head_block_num at stopping
+   uint32_t               last_irreversible_block_num {0}; // LIB at stopping
+
+   // Activate Savanna and create blocks log
+   blog_replay_fixture() {
+      // Activate Savanna
+      size_t num_keys    = 21u;
+      size_t finset_size = 21u;
+      finalizer_keys fin_keys(chain, num_keys, finset_size); // Create finalizer keys
+      fin_keys.set_node_finalizers(0u, num_keys); // set finalizers on current node
+      fin_keys.set_finalizer_policy(0u);
+      fin_keys.transition_to_savanna();
+
+      // Create a few accounts and produce a few blocks to fill in blocks log
+      chain.create_account("replay1"_n);
+      chain.produce_blocks(1);
+      chain.create_account("replay2"_n);
+      chain.produce_blocks(1);
+      chain.create_account("replay3"_n);
+
+      chain.produce_blocks(10);
+
+      // Make sure the account were created
+      BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay1"_n));
+      BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay2"_n));
+      BOOST_REQUIRE_NO_THROW(chain.control->get_account("replay3"_n));
+
+      // Store head_block_num and irreversible_block_num when the node is stopped
+      last_head_block_num = chain.control->head_block_num();
+      last_irreversible_block_num = chain.control->last_irreversible_block_num();
+
+      // Stop the node and save blocks_log
+      chain.close();
+   }
+
+   // Stop replay at block number `stop_at` and resume the replay afterwards
+   void stop_and_resume_replay(uint32_t stop_at) try {
+      controller::config copied_config = chain.get_config();
+
+      auto genesis = block_log::extract_genesis_state(copied_config.blocks_dir); 
+      BOOST_REQUIRE(genesis);
+
+      // remove the state files to make sure starting from blocks log
+      remove_existing_states(copied_config);
+
+      // Create a replay chain without starting it
+      eosio::testing::tester replay_chain(copied_config, *genesis, false); // false for not starting the chain
+
+      // Simulate shutdown by CTRL-C
+      bool is_quiting = false;
+      auto check_shutdown = [&is_quiting](){ return is_quiting; };
+
+      // Set up shutdown at a particular block number
+      replay_chain.control->irreversible_block().connect([&](const block_signal_params& t) {
+         const auto& [ block, id ] = t;
+         // Stop replay at block `stop_at`
+         if (block->block_num() == stop_at) {
+            is_quiting = true;
+         }
+      });
+
+      // Make sure irreversible fork_db exists
+      BOOST_CHECK(std::filesystem::exists(copied_config.blocks_dir / config::reversible_blocks_dir_name / "fork_db.dat"));
+
+      // Start replay and stop at block `stop_at`
+      replay_chain.control->startup( [](){}, check_shutdown, *genesis );
+      replay_chain.close();
+
+      // Make sure irreversible fork_db still exists
+      BOOST_CHECK(std::filesystem::exists(copied_config.blocks_dir / config::reversible_blocks_dir_name / "fork_db.dat"));
+
+      // Prepare resuming replay
+      controller::config copied_config_1 = replay_chain.get_config();
+      auto genesis_1 = block_log::extract_genesis_state(copied_config_1.blocks_dir); 
+      BOOST_REQUIRE(genesis_1);
+
+      // Remove the state files to make sure starting from block log
+      remove_existing_states(copied_config_1);
+
+      // Resume replay
+      eosio::testing::tester replay_chain_1(copied_config_1, *genesis);
+
+      // Make sure new chain contain the account created by original chain
+      BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay1"_n));
+      BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay2"_n));
+      BOOST_REQUIRE_NO_THROW(replay_chain_1.control->get_account("replay3"_n));
+
+      // Make sure replayed irreversible_block_num and head_block_num match
+      // with last_irreversible_block_num and last_head_block_num
+      BOOST_CHECK(replay_chain_1.control->last_irreversible_block_num() == last_irreversible_block_num);
+      BOOST_CHECK(replay_chain_1.control->head_block_num() == last_head_block_num);
+   } FC_LOG_AND_RETHROW()
+
+   void remove_existing_states(eosio::chain::controller::config& config) {
+      auto state_path = config.state_dir;
+      remove_all(state_path);
+      std::filesystem::create_directories(state_path);
+   }
+};
+
+// Test replay through blocks log and irreverible blocks
+BOOST_FIXTURE_TEST_CASE(replay_through, blog_replay_fixture) try {
+   eosio::chain::controller::config copied_config = chain.get_config();
+
+   auto genesis = eosio::chain::block_log::extract_genesis_state(copied_config.blocks_dir); 
+   BOOST_REQUIRE(genesis);
+
+   // remove the state files to make sure we are starting from block log
+   remove_existing_states(copied_config);
+   eosio::testing::tester replay_chain(copied_config, *genesis);
+
+   // Make sure new chain contain the account created by original chain
+   BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay1"_n));
+   BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay2"_n));
+   BOOST_REQUIRE_NO_THROW(replay_chain.control->get_account("replay3"_n));
+
+   // Make sure replayed irreversible_block_num and head_block_num match
+   // with last_irreversible_block_num and last_head_block_num
+   BOOST_CHECK(replay_chain.control->last_irreversible_block_num() == last_irreversible_block_num);
+   BOOST_CHECK(replay_chain.control->head_block_num() == last_head_block_num);
+} FC_LOG_AND_RETHROW()
+
+// Test replay stopping in the middle of blocks log and resuming
+BOOST_FIXTURE_TEST_CASE(replay_stop_in_middle, blog_replay_fixture) try {
+   // block `last_irreversible_block_num - 1` is within blocks log
+   stop_and_resume_replay(last_irreversible_block_num - 1);
+} FC_LOG_AND_RETHROW()
+
+// Test  replay stopping in the middle of irreverible blocks and resuming
+BOOST_FIXTURE_TEST_CASE(replay_stop_in_reversible_blocks, blog_replay_fixture) try {
+   // block `last_head_block_num - 1` is within reversible_blocks, where in Savanna
+   // we have at least 3 reversible blocks
+   stop_and_resume_replay(last_head_block_num - 1);
+} FC_LOG_AND_RETHROW()
+
+BOOST_AUTO_TEST_SUITE_END()