diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 44300e20d9..c7cb850ff0 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -222,8 +222,8 @@ void block_state::verify_qc(const valid_quorum_certificate& qc) const { }; // compute strong and weak accumulated weights - auto strong_weights = qc._strong_votes ? weights( *qc._strong_votes ) : 0; - auto weak_weights = qc._weak_votes ? weights( *qc._weak_votes ) : 0; + auto strong_weights = qc.strong_votes ? weights( *qc.strong_votes ) : 0; + auto weak_weights = qc.weak_votes ? weights( *qc.weak_votes ) : 0; // verfify quorum is met if( qc.is_strong() ) { @@ -259,18 +259,18 @@ void block_state::verify_qc(const valid_quorum_certificate& qc) const { }; // aggregate public keys and digests for strong and weak votes - if( qc._strong_votes ) { - pubkeys.emplace_back(aggregate_pubkeys(*qc._strong_votes)); + if( qc.strong_votes ) { + pubkeys.emplace_back(aggregate_pubkeys(*qc.strong_votes)); digests.emplace_back(std::vector{strong_digest.data(), strong_digest.data() + strong_digest.data_size()}); } - if( qc._weak_votes ) { - pubkeys.emplace_back(aggregate_pubkeys(*qc._weak_votes)); + if( qc.weak_votes ) { + pubkeys.emplace_back(aggregate_pubkeys(*qc.weak_votes)); digests.emplace_back(std::vector{weak_digest.begin(), weak_digest.end()}); } // validate aggregated signature - EOS_ASSERT( bls12_381::aggregate_verify(pubkeys, digests, qc._sig.jacobian_montgomery_le()), + EOS_ASSERT( bls12_381::aggregate_verify(pubkeys, digests, qc.sig.jacobian_montgomery_le()), invalid_qc_claim, "signature validation failed" ); } diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 657cf2eb7a..14d78eee12 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3721,10 +3721,10 @@ struct controller_impl { ("n1", qc_proof.block_num)("n2", new_qc_claim.block_num)("b", block_num) ); // Verify claimed strictness is the same as in proof - EOS_ASSERT( qc_proof.qc.is_strong() == new_qc_claim.is_strong_qc, + EOS_ASSERT( qc_proof.data.is_strong() == new_qc_claim.is_strong_qc, invalid_qc_claim, "QC is_strong (${s1}) in block extension does not match is_strong_qc (${s2}) in header extension. Block number: ${b}", - ("s1", qc_proof.qc.is_strong())("s2", new_qc_claim.is_strong_qc)("b", block_num) ); + ("s1", qc_proof.data.is_strong())("s2", new_qc_claim.is_strong_qc)("b", block_num) ); // find the claimed block's block state on branch of id auto bsp = fetch_bsp_on_branch_by_num( prev.id(), new_qc_claim.block_num ); @@ -3734,7 +3734,7 @@ struct controller_impl { ("q", new_qc_claim.block_num)("b", block_num) ); // verify the QC proof against the claimed block - bsp->verify_qc(qc_proof.qc); + bsp->verify_qc(qc_proof.data); } // thread safe, expected to be called from thread other than the main thread @@ -3843,7 +3843,7 @@ struct controller_impl { return; } const auto& qc_ext = std::get(block_exts.lower_bound(qc_ext_id)->second); - const auto& received_qc = qc_ext.qc.qc; + const auto& received_qc = qc_ext.qc.data; const auto claimed = fetch_bsp_on_branch_by_num( bsp_in->previous(), qc_ext.qc.block_num ); if( !claimed ) { diff --git a/libraries/chain/finality/quorum_certificate.cpp b/libraries/chain/finality/quorum_certificate.cpp index ab1576d66d..5059d18d29 100644 --- a/libraries/chain/finality/quorum_certificate.cpp +++ b/libraries/chain/finality/quorum_certificate.cpp @@ -22,38 +22,38 @@ inline std::vector bitset_to_vector(const vote_bitset& bs) { } bool pending_quorum_certificate::has_voted(size_t index) const { - return _strong_votes.has_voted(index) || _weak_votes.has_voted(index); + return strong_votes.has_voted(index) || weak_votes.has_voted(index); } bool pending_quorum_certificate::has_voted_no_lock(bool strong, size_t index) const { if (strong) { - return _strong_votes.has_voted(index); + return strong_votes.has_voted(index); } - return _weak_votes.has_voted(index); + return weak_votes.has_voted(index); } void pending_quorum_certificate::votes_t::reflector_init() { - _processed = std::vector>(_bitset.size()); - for (size_t i = 0; i < _bitset.size(); ++i) { - if (_bitset[i]) { - _processed[i].store(true, std::memory_order_relaxed); + processed = std::vector>(bitset.size()); + for (size_t i = 0; i < bitset.size(); ++i) { + if (bitset[i]) { + processed[i].store(true, std::memory_order_relaxed); } } } bool pending_quorum_certificate::votes_t::has_voted(size_t index) const { - assert(index < _processed.size()); - return _processed[index].load(std::memory_order_relaxed); + assert(index < processed.size()); + return processed[index].load(std::memory_order_relaxed); } -vote_status pending_quorum_certificate::votes_t::add_vote(size_t index, const bls_signature& sig) { - if (_bitset[index]) { // check here as could have come in while unlocked +vote_status pending_quorum_certificate::votes_t::add_vote(size_t index, const bls_signature& signature) { + if (bitset[index]) { // check here as could have come in while unlocked return vote_status::duplicate; // shouldn't be already present } - _processed[index].store(true, std::memory_order_relaxed); - _bitset.set(index); - _sig.aggregate(sig); // works even if _sig is default initialized (fp2::zero()) + processed[index].store(true, std::memory_order_relaxed); + bitset.set(index); + sig.aggregate(signature); // works even if _sig is default initialized (fp2::zero()) return vote_status::success; } @@ -63,10 +63,10 @@ pending_quorum_certificate::pending_quorum_certificate() pending_quorum_certificate::pending_quorum_certificate(size_t num_finalizers, uint64_t quorum, uint64_t max_weak_sum_before_weak_final) : _mtx(std::make_unique()) - , _quorum(quorum) - , _max_weak_sum_before_weak_final(max_weak_sum_before_weak_final) - , _weak_votes(num_finalizers) - , _strong_votes(num_finalizers) { + , quorum(quorum) + , max_weak_sum_before_weak_final(max_weak_sum_before_weak_final) + , weak_votes(num_finalizers) + , strong_votes(num_finalizers) { } bool pending_quorum_certificate::is_quorum_met() const { @@ -76,24 +76,24 @@ bool pending_quorum_certificate::is_quorum_met() const { // called by add_vote, already protected by mutex vote_status pending_quorum_certificate::add_strong_vote(size_t index, const bls_signature& sig, uint64_t weight) { - if (auto s = _strong_votes.add_vote(index, sig); s != vote_status::success) { + if (auto s = strong_votes.add_vote(index, sig); s != vote_status::success) { return s; } - _strong_sum += weight; + strong_sum += weight; - switch (_state) { + switch (pending_state) { case state_t::unrestricted: case state_t::restricted: - if (_strong_sum >= _quorum) { - assert(_state != state_t::restricted); - _state = state_t::strong; - } else if (_weak_sum + _strong_sum >= _quorum) - _state = (_state == state_t::restricted) ? state_t::weak_final : state_t::weak_achieved; + if (strong_sum >= quorum) { + assert(pending_state != state_t::restricted); + pending_state = state_t::strong; + } else if (weak_sum + strong_sum >= quorum) + pending_state = (pending_state == state_t::restricted) ? state_t::weak_final : state_t::weak_achieved; break; case state_t::weak_achieved: - if (_strong_sum >= _quorum) - _state = state_t::strong; + if (strong_sum >= quorum) + pending_state = state_t::strong; break; case state_t::weak_final: @@ -106,27 +106,27 @@ vote_status pending_quorum_certificate::add_strong_vote(size_t index, const bls_ // called by add_vote, already protected by mutex vote_status pending_quorum_certificate::add_weak_vote(size_t index, const bls_signature& sig, uint64_t weight) { - if (auto s = _weak_votes.add_vote(index, sig); s != vote_status::success) + if (auto s = weak_votes.add_vote(index, sig); s != vote_status::success) return s; - _weak_sum += weight; + weak_sum += weight; - switch (_state) { + switch (pending_state) { case state_t::unrestricted: case state_t::restricted: - if (_weak_sum + _strong_sum >= _quorum) - _state = state_t::weak_achieved; - - if (_weak_sum > _max_weak_sum_before_weak_final) { - if (_state == state_t::weak_achieved) - _state = state_t::weak_final; - else if (_state == state_t::unrestricted) - _state = state_t::restricted; + if (weak_sum + strong_sum >= quorum) + pending_state = state_t::weak_achieved; + + if (weak_sum > max_weak_sum_before_weak_final) { + if (pending_state == state_t::weak_achieved) + pending_state = state_t::weak_final; + else if (pending_state == state_t::unrestricted) + pending_state = state_t::restricted; } break; case state_t::weak_achieved: - if (_weak_sum >= _max_weak_sum_before_weak_final) - _state = state_t::weak_final; + if (weak_sum >= max_weak_sum_before_weak_final) + pending_state = state_t::weak_final; break; case state_t::weak_final: @@ -152,10 +152,10 @@ vote_status pending_quorum_certificate::add_vote(uint32_t connection_id, block_n } std::unique_lock g(*_mtx); - state_t pre_state = _state; + state_t pre_state = pending_state; vote_status s = strong ? add_strong_vote(index, sig, weight) : add_weak_vote(index, sig, weight); - state_t post_state = _state; + state_t post_state = pending_state; g.unlock(); fc_dlog(vote_logger, "connection - ${c} block_num: ${bn}, vote strong: ${sv}, status: ${s}, pre-state: ${pre}, post-state: ${state}, quorum_met: ${q}", @@ -167,14 +167,14 @@ vote_status pending_quorum_certificate::add_vote(uint32_t connection_id, block_n valid_quorum_certificate pending_quorum_certificate::to_valid_quorum_certificate() const { valid_quorum_certificate valid_qc; - if( _state == state_t::strong ) { - valid_qc._strong_votes = _strong_votes._bitset; - valid_qc._sig = _strong_votes._sig; + if( pending_state == state_t::strong ) { + valid_qc.strong_votes = strong_votes.bitset; + valid_qc.sig = strong_votes.sig; } else if (is_quorum_met_no_lock()) { - valid_qc._strong_votes = _strong_votes._bitset; - valid_qc._weak_votes = _weak_votes._bitset; - valid_qc._sig = _strong_votes._sig; - valid_qc._sig.aggregate(_weak_votes._sig); + valid_qc.strong_votes = strong_votes.bitset; + valid_qc.weak_votes = weak_votes.bitset; + valid_qc.sig = strong_votes.sig; + valid_qc.sig.aggregate(weak_votes.sig); } else assert(0); // this should be called only when we have a valid qc. @@ -185,8 +185,8 @@ std::optional pending_quorum_certificate::get_best_qc(block_ std::lock_guard g(*_mtx); // if pending_qc does not have a valid QC, consider valid_qc only if( !is_quorum_met_no_lock() ) { - if( _valid_qc ) { - return std::optional{quorum_certificate{ block_num, *_valid_qc }}; + if( valid_qc ) { + return std::optional{quorum_certificate{ block_num, *valid_qc }}; } else { return std::nullopt; } @@ -196,31 +196,31 @@ std::optional pending_quorum_certificate::get_best_qc(block_ valid_quorum_certificate valid_qc_from_pending = to_valid_quorum_certificate(); // if valid_qc does not have value, consider valid_qc_from_pending only - if( !_valid_qc ) { + if( !valid_qc ) { return std::optional{quorum_certificate{ block_num, valid_qc_from_pending }}; } // Both valid_qc and valid_qc_from_pending have value. Compare them and select a better one. // Strong beats weak. Tie break by valid_qc. const auto& best_qc = - _valid_qc->is_strong() == valid_qc_from_pending.is_strong() ? - *_valid_qc : // tie broken by valid_qc - _valid_qc->is_strong() ? *_valid_qc : valid_qc_from_pending; // strong beats weak + valid_qc->is_strong() == valid_qc_from_pending.is_strong() ? + *valid_qc : // tie broken by valid_qc + valid_qc->is_strong() ? *valid_qc : valid_qc_from_pending; // strong beats weak return std::optional{quorum_certificate{ block_num, best_qc }}; } void pending_quorum_certificate::set_valid_qc(const valid_quorum_certificate& qc) { std::lock_guard g(*_mtx); - _valid_qc = qc; + valid_qc = qc; } bool pending_quorum_certificate::valid_qc_is_strong() const { std::lock_guard g(*_mtx); - return _valid_qc && _valid_qc->is_strong(); + return valid_qc && valid_qc->is_strong(); } bool pending_quorum_certificate::is_quorum_met_no_lock() const { - return is_quorum_met(_state); + return is_quorum_met(pending_state); } } // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/finality/quorum_certificate.hpp b/libraries/chain/include/eosio/chain/finality/quorum_certificate.hpp index 0f8fdf0b92..2803718d6f 100644 --- a/libraries/chain/include/eosio/chain/finality/quorum_certificate.hpp +++ b/libraries/chain/include/eosio/chain/finality/quorum_certificate.hpp @@ -5,8 +5,7 @@ #include #include #include - -#include +#include #include @@ -31,21 +30,21 @@ namespace eosio::chain { // valid_quorum_certificate struct valid_quorum_certificate { - bool is_weak() const { return !!_weak_votes; } - bool is_strong() const { return !_weak_votes; } + bool is_weak() const { return !!weak_votes; } + bool is_strong() const { return !weak_votes; } - std::optional _strong_votes; - std::optional _weak_votes; - bls_aggregate_signature _sig; + std::optional strong_votes; + std::optional weak_votes; + bls_aggregate_signature sig; }; // quorum_certificate struct quorum_certificate { uint32_t block_num; - valid_quorum_certificate qc; + valid_quorum_certificate data; qc_claim_t to_qc_claim() const { - return {.block_num = block_num, .is_strong_qc = qc.is_strong()}; + return {.block_num = block_num, .is_strong_qc = data.is_strong()}; } }; @@ -75,15 +74,15 @@ namespace eosio::chain { friend struct fc::has_reflector_init; friend class pending_quorum_certificate; - vote_bitset _bitset; - bls_aggregate_signature _sig; - std::vector> _processed; // avoid locking mutex for _bitset duplicate check + vote_bitset bitset; + bls_aggregate_signature sig; + std::vector> processed; // avoid locking mutex for _bitset duplicate check void reflector_init(); public: explicit votes_t(size_t num_finalizers) - : _bitset(num_finalizers) - , _processed(num_finalizers) {} + : bitset(num_finalizers) + , processed(num_finalizers) {} // thread safe bool has_voted(size_t index) const; @@ -114,7 +113,7 @@ namespace eosio::chain { // thread safe bool has_voted(size_t index) const; - state_t state() const { std::lock_guard g(*_mtx); return _state; }; + state_t state() const { std::lock_guard g(*_mtx); return pending_state; }; std::optional get_best_qc(block_num_type block_num) const; void set_valid_qc(const valid_quorum_certificate& qc); @@ -123,14 +122,14 @@ namespace eosio::chain { friend struct fc::reflector; friend class qc_chain; std::unique_ptr _mtx; - std::optional _valid_qc; // best qc received from the network inside block extension - uint64_t _quorum {0}; - uint64_t _max_weak_sum_before_weak_final {0}; // max weak sum before becoming weak_final - state_t _state { state_t::unrestricted }; - uint64_t _strong_sum {0}; // accumulated sum of strong votes so far - uint64_t _weak_sum {0}; // accumulated sum of weak votes so far - votes_t _weak_votes {0}; - votes_t _strong_votes {0}; + std::optional valid_qc; // best qc received from the network inside block extension + uint64_t quorum {0}; + uint64_t max_weak_sum_before_weak_final {0}; // max weak sum before becoming weak_final + state_t pending_state { state_t::unrestricted }; + uint64_t strong_sum {0}; // accumulated sum of strong votes so far + uint64_t weak_sum {0}; // accumulated sum of weak votes so far + votes_t weak_votes {0}; + votes_t strong_votes {0}; // called by add_vote, already protected by mutex vote_status add_strong_vote(size_t index, @@ -150,8 +149,8 @@ namespace eosio::chain { FC_REFLECT_ENUM(eosio::chain::vote_status, (success)(duplicate)(unknown_public_key)(invalid_signature)(unknown_block)(max_exceeded)) -FC_REFLECT(eosio::chain::valid_quorum_certificate, (_strong_votes)(_weak_votes)(_sig)); -FC_REFLECT(eosio::chain::pending_quorum_certificate, (_valid_qc)(_quorum)(_max_weak_sum_before_weak_final)(_state)(_strong_sum)(_weak_sum)(_weak_votes)(_strong_votes)); +FC_REFLECT(eosio::chain::valid_quorum_certificate, (strong_votes)(weak_votes)(sig)); +FC_REFLECT(eosio::chain::pending_quorum_certificate, (valid_qc)(quorum)(max_weak_sum_before_weak_final)(pending_state)(strong_sum)(weak_sum)(weak_votes)(strong_votes)); FC_REFLECT_ENUM(eosio::chain::pending_quorum_certificate::state_t, (unrestricted)(restricted)(weak_achieved)(weak_final)(strong)); -FC_REFLECT(eosio::chain::pending_quorum_certificate::votes_t, (_bitset)(_sig)); -FC_REFLECT(eosio::chain::quorum_certificate, (block_num)(qc)); +FC_REFLECT(eosio::chain::pending_quorum_certificate::votes_t, (bitset)(sig)); +FC_REFLECT(eosio::chain::quorum_certificate, (block_num)(data)); diff --git a/libraries/libfc/include/fc/variant_dynamic_bitset.hpp b/libraries/libfc/include/fc/variant_dynamic_bitset.hpp index ce2acb68a3..3862553b88 100644 --- a/libraries/libfc/include/fc/variant_dynamic_bitset.hpp +++ b/libraries/libfc/include/fc/variant_dynamic_bitset.hpp @@ -3,8 +3,6 @@ #include #include -#include "variant_object.hpp" - namespace fc { inline void to_variant( const fc::dynamic_bitset& bs, fc::variant& v ) { @@ -12,19 +10,16 @@ namespace fc if ( num_blocks > MAX_NUM_ARRAY_ELEMENTS ) throw std::range_error( "number of blocks of dynamic_bitset cannot be greather than MAX_NUM_ARRAY_ELEMENTS" ); - std::vector blocks(num_blocks); - boost::to_block_range(bs, blocks.begin()); - v = fc::mutable_variant_object("size", bs.size()) - ("bits", blocks); + std::string s; + to_string(bs, s); + // From boost::dynamic_bitset docs: + // A character in the string is '1' if the corresponding bit is set, and '0' if it is not. Character + // position i in the string corresponds to bit position b.size() - 1 - i. + v = std::move(s); } inline void from_variant( const fc::variant& v, fc::dynamic_bitset& bs ) { - fc::dynamic_bitset::size_type size; - std::vector blocks; - - from_variant(v["size"], size); - from_variant(v["bits"], blocks); - bs = { blocks.cbegin(), blocks.cend() }; - bs.resize(size); + std::string s = v.get_string(); + bs = fc::dynamic_bitset(s); } } // namespace fc diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 1075c15144..6120703399 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -639,12 +639,12 @@ class producer_plugin_impl : public std::enable_shared_from_thisfinalizers; assert(votes.size() == finalizers.size()); for (size_t i = 0; i < votes.size(); ++i) { - if (!votes[i] && !check_weak(qc._weak_votes, i)) { + if (!votes[i] && !check_weak(qc.weak_votes, i)) { not_voted.push_back(finalizers[i].description); if (_finalizers.contains(finalizers[i].public_key)) { fc_wlog(vote_logger, "Local finalizer ${f} did not vote on block ${n}:${id}", @@ -675,20 +675,20 @@ class producer_plugin_impl : public std::enable_shared_from_thisfinalizers.size()) { fc::dynamic_bitset not_voted(active_finalizer_policy->finalizers.size()); - if (qc._strong_votes) { - not_voted = *qc._strong_votes; + if (qc.strong_votes) { + not_voted = *qc.strong_votes; } - if (qc._weak_votes) { - assert(not_voted.size() == qc._weak_votes->size()); - not_voted |= *qc._weak_votes; + if (qc.weak_votes) { + assert(not_voted.size() == qc.weak_votes->size()); + not_voted |= *qc.weak_votes; } not_voted.flip(); add_votes(not_voted, m.no_votes); @@ -709,7 +709,7 @@ class producer_plugin_impl : public std::enable_shared_from_thiscontains_extension(quorum_certificate_extension::extension_id())) { if (const auto& active_finalizers = chain.head_active_finalizer_policy()) { const auto& qc_ext = block->extract_extension(); - const auto& qc = qc_ext.qc.qc; + const auto& qc = qc_ext.qc.data; log_missing_votes(block, id, active_finalizers, qc); update_vote_block_metrics(block->block_num(), active_finalizers, qc); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 083eb53e84..fec5b43a94 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -65,6 +65,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/get_account_test.py ${CMAKE_CURRENT_B configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_high_transaction_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_high_transaction_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_retry_transaction_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_retry_transaction_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/transition_to_if.py ${CMAKE_CURRENT_BINARY_DIR}/transition_to_if.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/disaster_recovery.py ${CMAKE_CURRENT_BINARY_DIR}/disaster_recovery.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/trx_finality_status_forked_test.py ${CMAKE_CURRENT_BINARY_DIR}/trx_finality_status_forked_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugin_http_api_test.py ${CMAKE_CURRENT_BINARY_DIR}/plugin_http_api_test.py COPYONLY) @@ -144,6 +145,9 @@ set_property(TEST transition_to_if PROPERTY LABELS nonparallelizable_tests) add_test(NAME transition_to_if_lr COMMAND tests/transition_to_if.py -v -p 20 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST transition_to_if_lr PROPERTY LABELS long_running_tests) +add_test(NAME disaster_recovery COMMAND tests/disaster_recovery.py -v ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST disaster_recovery PROPERTY LABELS nonparallelizable_tests) + add_test(NAME ship_test COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 ${UNSHARE} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST ship_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME ship_test_unix COMMAND tests/ship_test.py -v --num-clients 10 --num-requests 5000 ${UNSHARE} --unix-socket WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/tests/TestHarness/Node.py b/tests/TestHarness/Node.py index b9c5c9b8e8..385f5a8f0a 100644 --- a/tests/TestHarness/Node.py +++ b/tests/TestHarness/Node.py @@ -553,6 +553,11 @@ def removeState(self): state = os.path.join(dataDir, "state") shutil.rmtree(state, ignore_errors=True) + def removeReversibleBlks(self): + dataDir = Utils.getNodeDataDir(self.nodeId) + reversibleBlks = os.path.join(dataDir, "blocks", "reversible") + shutil.rmtree(reversibleBlks, ignore_errors=True) + @staticmethod def findStderrFiles(path): files=[] diff --git a/tests/disaster_recovery.py b/tests/disaster_recovery.py new file mode 100755 index 0000000000..9ced9b756f --- /dev/null +++ b/tests/disaster_recovery.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import os +import shutil +import signal + +from TestHarness import Cluster, TestHelper, Utils, WalletMgr +from TestHarness.Node import BlockType + +############################################################### +# disaster_recovery +# +# Integration test with 4 finalizers (A, B, C, and D). +# +# The 4 nodes are cleanly shutdown in the following state: +# - A has LIB N. A has a finalizer safety information file that locks on a block after N. +# - B, C, and D have LIB less than N. They have finalizer safety information files that lock on N. +# +# All nodes lose their reversible blocks and restart from an earlier snapshot. +# +# A is restarted and replays up to block N after restarting from snapshot. Block N is sent to the other +# nodes B, C, and D after they are also started up again. +# +# Verify that LIB advances and that A, B, C, and D are eventually voting strong on new blocks. +# +############################################################### + +Print=Utils.Print +errorExit=Utils.errorExit + +args=TestHelper.parse_args({"-d","--keep-logs","--dump-error-details","-v","--leave-running","--unshared"}) +pnodes=4 +delay=args.d +debug=args.v +prod_count = 1 # per node prod count +total_nodes=pnodes +dumpErrorDetails=args.dump_error_details + +Utils.Debug=debug +testSuccessful=False + +cluster=Cluster(unshared=args.unshared, keepRunning=args.leave_running, keepLogs=args.keep_logs) +walletMgr=WalletMgr(True, keepRunning=args.leave_running, keepLogs=args.keep_logs) + +try: + TestHelper.printSystemInfo("BEGIN") + + cluster.setWalletMgr(walletMgr) + + Print(f'producing nodes: {pnodes}, delay between nodes launch: {delay} second{"s" if delay != 1 else ""}') + + Print("Stand up cluster") + # For now do not load system contract as it does not support setfinalizer + if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, totalProducers=pnodes, delay=delay, loadSystemContract=False, + activateIF=True) is False: + errorExit("Failed to stand up eos cluster.") + + assert cluster.biosNode.getInfo(exitOnError=True)["head_block_producer"] != "eosio", "launch should have waited for production to change" + cluster.biosNode.kill(signal.SIGTERM) + cluster.waitOnClusterSync(blockAdvancing=5) + + node0 = cluster.getNode(0) # A + node1 = cluster.getNode(1) # B + node2 = cluster.getNode(2) # C + node3 = cluster.getNode(3) # D + + Print("Create snapshot (node 0)") + ret = node0.createSnapshot() + assert ret is not None, "Snapshot creation failed" + ret_head_block_num = ret["payload"]["head_block_num"] + Print(f"Snapshot head block number {ret_head_block_num}") + + Print("Wait for snapshot node lib to advance") + node0.waitForBlock(ret_head_block_num+1, blockType=BlockType.lib) + assert node1.waitForLibToAdvance(), "Ndoe1 did not advance LIB after snapshot of Node0" + + assert node0.waitForLibToAdvance(), "Node0 did not advance LIB after snapshot" + currentLIB = node0.getIrreversibleBlockNum() + + for node in [node1, node2, node3]: + node.kill(signal.SIGTERM) + + for node in [node1, node2, node3]: + assert not node.verifyAlive(), "Node did not shutdown" + + # node0 will have higher lib than 1,2,3 since it can incorporate QCs in blocks + Print("Wait for node 0 LIB to advance") + assert node0.waitForBlock(currentLIB, blockType=BlockType.lib), "Node0 did not advance LIB" # uses getBlockNum(blockType=blockType) > blockNum + node0.kill(signal.SIGTERM) + assert not node0.verifyAlive(), "Node0 did not shutdown" + + node0.removeState() + for node in [node1, node2, node3]: + node.removeReversibleBlks() + node.removeState() + + for i in range(4): + isRelaunchSuccess = cluster.getNode(i).relaunch(chainArg=" -e --snapshot {}".format(node0.getLatestSnapshot())) + assert isRelaunchSuccess, f"node {i} relaunch from snapshot failed" + + for node in [node0, node1, node2, node3]: + assert node.waitForLibToAdvance(), "Node did not advance LIB after relaunch" + + testSuccessful=True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, dumpErrorDetails=dumpErrorDetails) + +exitCode = 0 if testSuccessful else 1 +exit(exitCode)