Merge pull request #595 from AntelopeIO/gh_380_1.0

[main -> 1.0] Implement last three Savanna unittests modeled after the fast testnet wave tests.

unittests/savanna_cluster.cpp

@@ -27,10 +27,11 @@ node_t::node_t(size_t node_idx, cluster_t& cluster, setup_policy policy /* = set
       }
    };
 
-   auto node_initialization_fn = [&]() {
+   auto node_initialization_fn = [&, node_idx]() {
       [[maybe_unused]] auto _a = control->voted_block().connect(voted_block_cb);
       [[maybe_unused]] auto _b = control->accepted_block().connect(accepted_block_cb);
       tester::set_node_finalizers(node_finalizers);
+      cluster.get_new_blocks_from_peers(node_idx);
    };
 
    node_initialization_fn();                  // initialize the node when it is first created

unittests/savanna_cluster.hpp

@@ -59,6 +59,7 @@ namespace savanna_cluster {
 
       std::function<void(const block_signal_params&)> accepted_block_cb;
       std::function<void(const vote_signal_params&)>  voted_block_cb;
+
    public:
       std::vector<account_name> node_finalizers;
 
@@ -141,10 +142,12 @@ namespace savanna_cluster {
       }
 
       void push_block(const signed_block_ptr& b) {
-         assert(!pushing_a_block);
-         pushing_a_block = true;
-         auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; });
-         tester::push_block(b);
+         if (is_open() && !fetch_block_by_id(b->calculate_id())) {
+            assert(!pushing_a_block);
+            pushing_a_block = true;
+            auto reset_pending_on_exit = fc::make_scoped_exit([this]{ pushing_a_block = false; });
+            tester::push_block(b);
+         }
       }
 
       template <class Node>
@@ -419,7 +422,7 @@ namespace savanna_cluster {
       // -----------------------------------------------------------------------------
       void push_blocks(size_t src_idx, size_t dst_idx, uint32_t start_block_num) {
          auto& src = _nodes[src_idx];
-         assert(src.is_open() &&  _nodes[dst_idx].is_open());
+         assert(src.is_open() && _nodes[dst_idx].is_open());
 
          auto end_block_num   = src.fork_db_head().block_num();
 
@@ -464,26 +467,35 @@ namespace savanna_cluster {
       void dispatch_vote_to_peers(size_t node_idx, skip_self_t skip_self, const vote_message_ptr& msg) {
          static uint32_t connection_id = 0;
          for_each_peer(node_idx, skip_self, [&](node_t& n) {
-            n.control->process_vote_message(++connection_id, msg);
+            if (n.is_open())
+               n.control->process_vote_message(++connection_id, msg);
          });
       }
 
       void push_block_to_peers(size_t node_idx, skip_self_t skip_self, const signed_block_ptr& b) {
          for_each_peer(node_idx, skip_self, [&](node_t& n) {
-            if (!n.fetch_block_by_id(b->calculate_id()))
-               n.push_block(b);
+            n.push_block(b);
+         });
+      }
+
+      // when a node restarts, simulate receiving newly produced blocks from peers
+      void get_new_blocks_from_peers(size_t node_idx) {
+         assert(_nodes[node_idx].is_open());
+         for_each_peer(node_idx, skip_self_t::yes, [&](node_t& n) {
+            if (n.is_open())
+               n.push_blocks_to(_nodes[node_idx]);
          });
       }
 
       template<class CB>
       void for_each_peer(size_t node_idx, skip_self_t skip_self, const CB& cb) {
-         if (_shutting_down)
+         if (_shutting_down || _peers.empty())
             return;
          assert(_peers.find(node_idx) != _peers.end());
          const auto& peers = _peers[node_idx];
          for (auto i : peers) {
             bool dont_skip = skip_self == skip_self_t::no || i != node_idx;
-            if (dont_skip && _nodes[i].is_open())
+            if (dont_skip)
                cb(_nodes[i]);
          }
       }

unittests/savanna_cluster_tests.cpp

@@ -54,7 +54,7 @@ BOOST_FIXTURE_TEST_CASE(simple_test, savanna_cluster::cluster_t) { try {
       // all 4 blocks produced by _nodes[0] will have the same `latest_qc_claim_block_num`, which is node0_lib+2
 
       set_partition({});                                    // Reunite the two partitions
-      push_blocks(0, partition);                            // Push the blocks that _nodes[0] produced to the other
+      propagate_heads();                                    // Push the blocks that _nodes[0] produced to the other
                                                             // nodes which will vote
       _nodes[0].produce_block();                            // Produce one block so the newly produced QC propagates.
                                                             // this is needed because we don't advance lib when

unittests/savanna_disaster_recovery_tests.cpp

@@ -15,7 +15,6 @@ BOOST_FIXTURE_TEST_CASE(node_goes_down, savanna_cluster::cluster_t) try {
    C.close();                                                                  // shutdown node C
    BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // lib still advances with 3 finalizers
    C.open();                                                                   // restart node C
-   A.push_blocks_to(C);                                                        // propagate blocks A -> C
    BOOST_REQUIRE_EQUAL(4u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -37,7 +36,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_old_fsi, savanna_cluster::clust
    C.remove_state();
    C.overwrite_fsi(fsi);
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -57,7 +55,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_with_deleted_fsi, savanna_cluster::c
    C.remove_state();
    C.remove_fsi();
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -76,7 +73,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_node_while_retaining_fsi, savanna_cluster
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // lib still advances with 3 finalizers
    C.remove_state();
    C.open_from_snapshot(snapshot);
-   A.push_blocks_to(C);
    BOOST_REQUIRE_EQUAL(2u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
    BOOST_REQUIRE(!C.is_head_missing_finalizer_votes());                        // let's make sure of that
 } FC_LOG_AND_RETHROW()
@@ -97,7 +93,6 @@ BOOST_FIXTURE_TEST_CASE(nodes_go_down, savanna_cluster::cluster_t) try {
    for (auto& N : failing_nodes) N->close();
    BOOST_REQUIRE_EQUAL(1u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // lib stalls with 3 finalizers down, 1 QC in flight
    for (auto& N : failing_nodes) N->open();
-   for (auto& N : failing_nodes) A.push_blocks_to(*N);
    BOOST_REQUIRE_EQUAL(7u, A.lib_advances_by([&]() { A.produce_blocks(4);  })); // all 4 finalizers should be back voting
    for (auto& N : failing_nodes) BOOST_REQUIRE(!N->is_head_missing_finalizer_votes());
 } FC_LOG_AND_RETHROW()
@@ -125,7 +120,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_old_fsi, savanna_cluster::clus
       N->remove_state();
       N->overwrite_fsi(fsis[i]);
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -152,7 +146,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_with_deleted_fsi, savanna_cluster::
       N->remove_state();
       N->remove_fsi();
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -178,7 +171,6 @@ BOOST_FIXTURE_TEST_CASE(recover_killed_nodes_while_retaining_fsi, savanna_cluste
    for (auto& N : failing_nodes) {
       N->remove_state();
       N->open_from_snapshot(snapshots[i]);
-      A.push_blocks_to(*N);
       ++i;
    }
    BOOST_REQUIRE_EQUAL(3u, A.lib_advances_by([&]() { A.produce_blocks(2);  })); // all 4 finalizers should be back voting
@@ -238,9 +230,13 @@ BOOST_FIXTURE_TEST_CASE(all_nodes_shutdown_with_reversible_blocks_lost, savanna_
       N->close();
       N->remove_state();
       remove_blocks_log ? N->remove_reversible_data_and_blocks_log() : N->remove_reversible_data();
-      N->open_from_snapshot(snapshot);
    }
 
+   // reopen after all nodes closed
+   // -----------------------------
+   for (auto& N : failing_nodes)
+      N->open_from_snapshot(snapshot);
+
    propagate_heads(); // needed only if we don't remove the blocks log, otherwise lib advanced by 1 block
                       // which was stored in the blocks log, and when replayed after loading A and B's
                       // snapshots advanced head() by one