Merge pull request #719 from AntelopeIO/gh_694

[1.0.1] Add missing data to block_header_state and change verify_qc_claim to use block_header_state data rather than only looking up a block_state in the fork database

libraries/chain/block_handle.cpp

@@ -4,6 +4,16 @@
 
 namespace eosio::chain {
 
+// -------------------------------------------------------------------------------------------
+// prior to writing magic and version numbers, we simply serialized the class (*this) to
+// file. Let's call this the implicit version 0, which is not supported anymore in
+// Spring 1.0.1 and above.
+// However, we need to make sure that `chain_head_magic` can't match the tag of a std::variant
+// -------------------------------------------------------------------------------------------
+constexpr uint64_t chain_head_magic   = 0xf1f2f3f4f4f3f2f1;
+constexpr uint64_t chain_head_version = 1;
+
+
 void block_handle::write(const std::filesystem::path& state_file) {
    if (!is_valid())
       return;
@@ -13,24 +23,37 @@ void block_handle::write(const std::filesystem::path& state_file) {
    fc::datastream<fc::cfile> f;
    f.set_file_path(state_file);
    f.open("wb");
-
+   fc::raw::pack(f, chain_head_magic);
+   fc::raw::pack(f, chain_head_version);
    fc::raw::pack(f, *this);
 }
 
 bool block_handle::read(const std::filesystem::path& state_file) {
    if (!std::filesystem::exists(state_file))
       return false;
 
-   fc::datastream<fc::cfile> f;
-   f.set_file_path(state_file);
-   f.open("rb");
+   EOS_ASSERT(std::filesystem::file_size(state_file) >= 2 * sizeof(chain_head_magic), chain_exception,
+              "File `chain_head.dat` seems to be corrupted. The best course of action might be to restart from a snapshot" );
 
-   fc::raw::unpack(f, *this);
+   try {
+      fc::datastream<fc::cfile> f;
+      f.set_file_path(state_file);
+      f.open("rb");
 
-   ilog("Loading chain_head block ${bn} ${id}", ("bn", block_num())("id", id()));
+      uint64_t magic, version;
+      fc::raw::unpack(f, magic);
+      fc::raw::unpack(f, version);
 
-   std::filesystem::remove(state_file);
+      EOS_ASSERT(magic == chain_head_magic && version == chain_head_version, chain_exception,
+                 "Error reading `chain_head.dat` file. It is likely a Spring 1.0.0 version which is not supported by Spring 1.0.1 and above"
+                 "The best course of action might be to restart from a snapshot" );
 
+      fc::raw::unpack(f, *this);
+      ilog("Loading chain_head block ${bn} ${id}", ("bn", block_num())("id", id()));
+   } FC_CAPTURE_AND_RETHROW( (state_file) );
+
+   // remove the `chain_head.dat` file only if we were able to successfully load it.
+   std::filesystem::remove(state_file);
    return true;
 }
 

libraries/chain/block_header_state.cpp

@@ -16,7 +16,7 @@ digest_type block_header_state::compute_base_digest() const {
    digest_type::encoder enc;
 
    fc::raw::pack( enc, header );
-   fc::raw::pack( enc, core );
+   core.pack_for_digest( enc );
 
    fc::raw::pack( enc, proposed_finalizer_policies );
    fc::raw::pack( enc, pending_finalizer_policy );
@@ -141,6 +141,67 @@ const finalizer_policy& block_header_state::get_last_pending_finalizer_policy()
    return *active_finalizer_policy;
 }
 
+// Only defined for core.latest_qc_claim().block_num <= ref.block_num() <= core.current_block_num()
+// Retrieves the finalizer policies applicable for the block referenced by `ref`.
+// See full explanation in issue #694.
+// ------------------------------------------------------------------------------------------------
+finalizer_policies_t block_header_state::get_finalizer_policies(const block_ref& ref) const {
+   assert(core.links.empty() ||   // called from a bogus block_state constructed in a test
+          (core.latest_qc_claim().block_num <= ref.block_num() && ref.block_num() <= core.current_block_num()));
+   finalizer_policies_t res;
+
+   res.finality_digest = ref.finality_digest;
+
+   auto active_gen = ref.active_policy_generation;
+   assert(active_gen != 0);                                       // we should always have an active policy
+
+   if (active_finalizer_policy->generation == active_gen)
+      res.active_finalizer_policy = active_finalizer_policy;      // the one active at block_num is still active
+   else {
+      // cannot be the pending one as it never was active
+      assert(!pending_finalizer_policy || pending_finalizer_policy->second->generation > active_gen);
+
+      // has to be the one in latest_qc_claim_block_active_finalizer_policy
+      assert(latest_qc_claim_block_active_finalizer_policy != nullptr);
+      assert(latest_qc_claim_block_active_finalizer_policy->generation == active_gen);
+      EOS_ASSERT(latest_qc_claim_block_active_finalizer_policy != nullptr &&
+                    latest_qc_claim_block_active_finalizer_policy->generation == active_gen,
+                 chain_exception,
+                 "Logic error in finalizer policy retrieval"); // just in case
+      res.active_finalizer_policy = latest_qc_claim_block_active_finalizer_policy;
+   }
+
+   auto pending_gen = ref.pending_policy_generation;
+   if (pending_gen == 0)
+      res.pending_finalizer_policy = nullptr;                    // no pending policy at block_num.
+   else if (pending_gen == active_finalizer_policy->generation)
+      res.pending_finalizer_policy = active_finalizer_policy;    // policy pending at block_num became active
+   else {
+      // cannot be the one in latest_qc_claim_block_active_finalizer_policy since it was active at
+      // core.latest_qc_claim().block_num. So it must be the one still pending.
+      assert(pending_finalizer_policy && pending_finalizer_policy->second->generation == pending_gen);
+      EOS_ASSERT(pending_finalizer_policy && pending_finalizer_policy->second->generation == pending_gen, chain_exception,
+                 "Logic error in finalizer policy retrieval");  // just in case
+      res.pending_finalizer_policy = pending_finalizer_policy->second;
+   }
+
+   return res;
+}
+
+// Only defined for core.latest_qc_claim().block_num <= num <= core.current_block_num()
+// Retrieves the active finalizer policy generation applicatble for the block `num`, which
+// can be the current block or one of its ancestors up to core.latest_qc_claim().block_num (incl).
+// -----------------------------------------------------------------------------------------------
+uint32_t block_header_state::get_active_finalizer_policy_generation(block_num_type num) const {
+   assert(core.links.empty() ||   // called from a bogus block_state constructed in a test
+          (core.last_final_block_num() <= num && num <= core.current_block_num()));
+   if (num == block_num())
+      return active_finalizer_policy->generation;
+   const block_ref& ref = core.get_block_reference(num);
+   return ref.active_policy_generation;
+}
+
+
 // The last proposed proposer policy, if none proposed then the active proposer policy
 const proposer_policy& block_header_state::get_last_proposed_proposer_policy() const {
    if (latest_proposed_proposer_policy) {
@@ -325,6 +386,22 @@ void finish_next(const block_header_state& prev,
       next_header_state.finalizer_policy_generation = prev.finalizer_policy_generation;
    }
 
+   // now populate next_header_state.latest_qc_claim_block_active_finalizer_policy
+   // this keeps track of the finalizer policy which was active @ latest_qc_claim().block_num, but which
+   // can be overwritten by a previously pending policy (member `active_finalizer_policy`)
+   // See full explanation in issue #694.
+   // --------------------------------------------------------------------------------------------------
+   const auto& next_core                 = next_header_state.core;
+   auto        latest_qc_claim_block_num = next_core.latest_qc_claim().block_num;
+   const auto  active_generation_num     = next_header_state.active_finalizer_policy->generation;
+   if (prev.get_active_finalizer_policy_generation(latest_qc_claim_block_num) != active_generation_num) {
+      const auto& latest_qc_claim_block_ref = next_header_state.core.get_block_reference(latest_qc_claim_block_num);
+      next_header_state.latest_qc_claim_block_active_finalizer_policy =
+         prev.get_finalizer_policies(latest_qc_claim_block_ref).active_finalizer_policy;
+   } else {
+      next_header_state.latest_qc_claim_block_active_finalizer_policy = nullptr;
+   }
+
    // Finally update block id from header
    // -----------------------------------
    next_header_state.block_id = next_header_state.header.calculate_id();

libraries/chain/block_state.cpp

@@ -184,25 +184,30 @@ block_state_ptr block_state::create_transition_block(
    return result_ptr;
 }
 
-block_state::block_state(snapshot_detail::snapshot_block_state_v7&& sbs)
+// Spring 1.0.1 to ? snapshot v8 format. Updated `finality_core` to include finalizer policies
+// generation numbers. Also new member `block_state::latest_qc_claim_block_active_finalizer_policy`
+// ------------------------------------------------------------------------------------------------
+block_state::block_state(snapshot_detail::snapshot_block_state_v8&& sbs)
    : block_header_state {
-         .block_id                    = sbs.block_id,
-         .header                      = std::move(sbs.header),
-         .activated_protocol_features = std::move(sbs.activated_protocol_features),
-         .core                        = std::move(sbs.core),
-         .active_finalizer_policy     = std::move(sbs.active_finalizer_policy),
-         .active_proposer_policy      = std::move(sbs.active_proposer_policy),
+         .block_id                        = sbs.block_id,
+         .header                          = std::move(sbs.header),
+         .activated_protocol_features     = std::move(sbs.activated_protocol_features),
+         .core                            = std::move(sbs.core),
+         .active_finalizer_policy         = std::move(sbs.active_finalizer_policy),
+         .active_proposer_policy          = std::move(sbs.active_proposer_policy),
          .latest_proposed_proposer_policy = std::move(sbs.latest_proposed_proposer_policy),
          .latest_pending_proposer_policy  = std::move(sbs.latest_pending_proposer_policy),
-         .proposed_finalizer_policies = std::move(sbs.proposed_finalizer_policies),
-         .pending_finalizer_policy    = std::move(sbs.pending_finalizer_policy),
-         .finalizer_policy_generation = sbs.finalizer_policy_generation,
+         .proposed_finalizer_policies     = std::move(sbs.proposed_finalizer_policies),
+         .pending_finalizer_policy        = std::move(sbs.pending_finalizer_policy),
+         .latest_qc_claim_block_active_finalizer_policy = std::move(sbs.latest_qc_claim_block_active_finalizer_policy),
+         .finalizer_policy_generation     = sbs.finalizer_policy_generation,
          .last_pending_finalizer_policy_digest = sbs.last_pending_finalizer_policy_digest,
          .last_pending_finalizer_policy_start_timestamp = sbs.last_pending_finalizer_policy_start_timestamp
       }
    , strong_digest(compute_finality_digest())
    , weak_digest(create_weak_digest(strong_digest))
-   , aggregating_qc(active_finalizer_policy, pending_finalizer_policy ? pending_finalizer_policy->second : finalizer_policy_ptr{}) // just in case we receive votes
+   , aggregating_qc(active_finalizer_policy,
+                    pending_finalizer_policy ? pending_finalizer_policy->second : finalizer_policy_ptr{}) // just in case we receive votes
    , valid(std::move(sbs.valid))
 {
    header_exts = header.validate_and_extract_header_extensions();
@@ -233,7 +238,14 @@ vote_status_t block_state::has_voted(const bls_public_key& key) const {
 
 // Called from net threads
 void block_state::verify_qc(const qc_t& qc) const {
-   aggregating_qc.verify_qc(qc, strong_digest, weak_digest);
+   // Do not use `block_state::aggregating_qc` which applies only for `this` block.
+   // `verify_qc()` can be called on a descendant `block_state` of `qc.block_num`, so we need
+   // to create a new `aggregating_qc_t` with the finalizer policies of the claimed block.
+   // ---------------------------------------------------------------------------------------
+   finalizer_policies_t policies = get_finalizer_policies(qc.block_num);
+   aggregating_qc_t aggregating_qc(policies.active_finalizer_policy, policies.pending_finalizer_policy);
+
+   aggregating_qc.verify_qc(qc, policies.finality_digest, create_weak_digest(policies.finality_digest));
 }
 
 qc_claim_t block_state::extract_qc_claim() const {

libraries/chain/controller.cpp

@@ -2212,7 +2212,7 @@ struct controller_impl {
       });
 
       snapshot->write_section("eosio::chain::block_state", [&]( auto& section ) {
-         section.add_row(snapshot_detail::snapshot_block_state_data_v7(get_block_state_to_snapshot()), db);
+         section.add_row(snapshot_detail::snapshot_block_state_data_v8(get_block_state_to_snapshot()), db);
       });
 
       controller_index_set::walk_indices([this, &snapshot, &row_counter]( auto utils ){
@@ -2261,15 +2261,15 @@ struct controller_impl {
       });
 
       using namespace snapshot_detail;
-      using v7 = snapshot_block_state_data_v7;
+      using v8 = snapshot_block_state_data_v8;
 
       block_state_pair result;
-      if (header.version >= v7::minimum_version) {
-         // loading a snapshot saved by Spring 1.0 and above.
-         // -----------------------------------------------
-         if (std::clamp(header.version, v7::minimum_version, v7::maximum_version) == header.version ) {
+      if (header.version >= v8::minimum_version) {
+         // loading a snapshot saved by Spring 1.0.1 and above.
+         // ---------------------------------------------------
+         if (std::clamp(header.version, v8::minimum_version, v8::maximum_version) == header.version ) {
             snapshot->read_section("eosio::chain::block_state", [this, &result]( auto &section ){
-               v7 block_state_data;
+               v8 block_state_data;
                section.read_row(block_state_data, db);
                assert(block_state_data.bs_l || block_state_data.bs);
                if (block_state_data.bs_l) {
@@ -2291,8 +2291,13 @@ struct controller_impl {
          } else {
             EOS_THROW(snapshot_exception, "Unsupported block_state version");
          }
+      } else if (header.version == 7) {
+         // snapshot created with Spring 1.0.0, which was very soon superseded by Spring 1.0.1
+         // and a new snapshot format.
+         // ----------------------------------------------------------------------------------
+         EOS_THROW(snapshot_exception, "v7 snapshots are not supported anymore in Spring 1.0.1 and above");
       } else {
-         // loading a snapshot saved by Leap up to version 5.
+         // loading a snapshot saved by Leap up to version 6.
          // -------------------------------------------------
          auto head_header_state = std::make_shared<block_state_legacy>();
          using v2 = snapshot_block_header_state_legacy_v2;
@@ -4076,14 +4081,8 @@ struct controller_impl {
    }
 
    // This verifies BLS signatures and is expensive.
-   void verify_qc( const signed_block_ptr& b, const block_header_state& prev, const qc_t& qc ) {
-      // find the claimed block's block state on branch of id
-      auto bsp = fork_db_fetch_bsp_on_branch_by_num( prev.id(), qc.block_num );
-      EOS_ASSERT( bsp, invalid_qc_claim,
-                  "Block state was not found in forkdb for claimed block ${bn}. Current block number: ${b}",
-                  ("bn", qc.block_num)("b", b->block_num()) );
-
-      bsp->verify_qc(qc);
+   void verify_qc( const block_state& prev, const qc_t& qc ) {
+      prev.verify_qc(qc);
    }
 
    // thread safe, expected to be called from thread other than the main thread
@@ -4096,7 +4095,7 @@ struct controller_impl {
 
       if constexpr (is_proper_savanna_block) {
          if (qc) {
-            verify_qc(b, prev, *qc);
+            verify_qc(prev, *qc);
 
             dlog("received block: #${bn} ${t} ${prod} ${id}, qc claim: ${qc_claim}, previous: ${p}",
                  ("bn", b->block_num())("t", b->timestamp)("prod", b->producer)("id", id)
@@ -4341,7 +4340,7 @@ struct controller_impl {
 
                if constexpr (std::is_same_v<typename std::decay_t<BSP>, block_state_ptr>) {
                   if (conf.force_all_checks && qc) {
-                     verify_qc(b, *head, *qc);
+                     verify_qc(*head, *qc);
                   }
                }