Merge remote-tracking branch 'spring/main' into GH-376-joint-qcs

libraries/chain/authorization_manager.cpp

@@ -110,21 +110,22 @@ namespace eosio { namespace chain {
       });
    }
 
-   void authorization_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot ) {
-      authorization_index_set::walk_indices([this, &snapshot]( auto utils ){
+   void authorization_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot, snapshot_loaded_row_counter& row_counter ) {
+      authorization_index_set::walk_indices([this, &snapshot, &row_counter]( auto utils ){
          using section_t = typename decltype(utils)::index_t::value_type;
 
          // skip the permission_usage_index as its inlined with permission_index
          if (std::is_same<section_t, permission_usage_object>::value) {
             return;
          }
 
-         snapshot->read_section<section_t>([this]( auto& section ) {
+         snapshot->read_section<section_t>([this, &row_counter]( auto& section ) {
             bool more = !section.empty();
             while(more) {
                decltype(utils)::create(_db, [this, &section, &more]( auto &row ) {
                   more = section.read_row(row, _db);
                });
+               row_counter.progress();
             }
          });
       });

libraries/chain/controller.cpp

@@ -2053,8 +2053,8 @@ struct controller_impl {
       });
    }
 
-   void read_contract_tables_from_snapshot( const snapshot_reader_ptr& snapshot ) {
-      snapshot->read_section("contract_tables", [this]( auto& section ) {
+   void read_contract_tables_from_snapshot( const snapshot_reader_ptr& snapshot, snapshot_loaded_row_counter& row_counter ) {
+      snapshot->read_section("contract_tables", [this, &row_counter]( auto& section ) {
          bool more = !section.empty();
          while (more) {
             // read the row for the table
@@ -2063,19 +2063,22 @@ struct controller_impl {
                section.read_row(row, db);
                t_id = row.id;
             });
+            row_counter.progress();
 
             // read the size and data rows for each type of table
-            contract_database_index_set::walk_indices([this, &section, &t_id, &more](auto utils) {
+            contract_database_index_set::walk_indices([this, &section, &t_id, &more, &row_counter](auto utils) {
                using utils_t = decltype(utils);
 
                unsigned_int size;
                more = section.read_row(size, db);
+               row_counter.progress();
 
                for (size_t idx = 0; idx < size.value; idx++) {
                   utils_t::create(db, [this, &section, &more, &t_id](auto& row) {
                      row.t_id = t_id;
                      more = section.read_row(row, db);
                   });
+                  row_counter.progress();
                }
             });
          }
@@ -2150,6 +2153,8 @@ struct controller_impl {
    }
 
    block_state_pair read_from_snapshot( const snapshot_reader_ptr& snapshot, uint32_t blog_start, uint32_t blog_end ) {
+      snapshot_loaded_row_counter row_counter(snapshot->total_row_count());
+
       chain_snapshot_header header;
       snapshot->read_section<chain_snapshot_header>([this, &header]( auto &section ){
          section.read_row(header, db);
@@ -2222,7 +2227,7 @@ struct controller_impl {
                   ("block_log_last_num", blog_end)
       );
 
-      controller_index_set::walk_indices([this, &snapshot, &header]( auto utils ){
+      controller_index_set::walk_indices([this, &snapshot, &header, &row_counter]( auto utils ){
          using value_t = typename decltype(utils)::index_t::value_type;
 
          // skip the table_id_object as its inlined with contract tables section
@@ -2295,20 +2300,21 @@ struct controller_impl {
             }
          }
 
-         snapshot->read_section<value_t>([this]( auto& section ) {
+         snapshot->read_section<value_t>([this,&row_counter]( auto& section ) {
             bool more = !section.empty();
             while(more) {
                decltype(utils)::create(db, [this, &section, &more]( auto &row ) {
                   more = section.read_row(row, db);
                });
+               row_counter.progress();
             }
          });
       });
 
-      read_contract_tables_from_snapshot(snapshot);
+      read_contract_tables_from_snapshot(snapshot, row_counter);
 
-      authorization.read_from_snapshot(snapshot);
-      resource_limits.read_from_snapshot(snapshot);
+      authorization.read_from_snapshot(snapshot, row_counter);
+      resource_limits.read_from_snapshot(snapshot, row_counter);
 
       db.set_revision( chain_head.block_num() );
       db.create<database_header_object>([](const auto& header){

libraries/chain/finality/finality_core.cpp

@@ -212,6 +212,7 @@ const qc_link& finality_core::get_qc_link_from(block_num_type block_num) const
 std::tuple<block_num_type, block_num_type, block_num_type> get_new_block_numbers(const finality_core& c, const qc_claim_t& most_recent_ancestor_with_qc)
 {
    assert(most_recent_ancestor_with_qc.block_num <= c.current_block_num()); // Satisfied by the precondition.
+   assert(c.latest_qc_claim().block_num <= most_recent_ancestor_with_qc.block_num); // Satisfied by the precondition.
 
    // Invariant 2 of core guarantees that:
    // c.last_final_block_num() <= c.links.front().source_block_num <= c.final_on_strong_qc_block_num  <= c.latest_qc_claim().block_num
@@ -223,53 +224,34 @@ std::tuple<block_num_type, block_num_type, block_num_type> get_new_block_numbers
       return {c.last_final_block_num(), c.links.front().source_block_num, c.final_on_strong_qc_block_num};
    }
 
-   const auto& link1 = c.get_qc_link_from(most_recent_ancestor_with_qc.block_num);
-
-   // By the post-condition of get_qc_link_from, link1.source_block_num == most_recent_ancestor_with_qc.block_num.
-   // By the invariant on qc_link, link1.target_block_num <= link1.source_block_num.
-   // Therefore, link1.target_block_num <= most_recent_ancestor_with_qc.block_num.
-   // And also by the precondition, link1.target_block_num <= c.current_block_num().
-
-   // If c.refs.empty() == true, then by invariant 3 of core, link1 == c.links.front() == c.links.back() and so
-   // link1.target_block_num == c.current_block_num().
-
-   // Otherwise, if c.refs.empty() == false, consider two cases.
-   // Case 1: link1 != c.links.back()
-   //   In this case, link1.target_block_num <= link1.source_block_num < c.links.back().source_block_num.
-   //   The strict inequality is justified by invariant 7 of core.
-   //   Therefore, link1.target_block_num < c.current_block_num().
-   // Case 2: link1 == c.links.back()
-   //   In this case, link1.target_block_num < link1.source_block_num == c.links.back().source_block_num.
-   //   The strict inequality is justified because target_block_num and source_block_num of a qc_link can only be equal for a
-   //   genesis block. And a link mapping genesis block number to genesis block number can only possibly exist for c.links.front().
-   //   Therefore, link1.target_block_num < c.current_block_num().
-
-   // There must exist some link, call it link0, within c.links where 
-   // link0.target_block_num == c.final_on_strong_qc_block_num and link0.source_block_num <= c.latest_qc_claim().block_num.
-   // By the precondition, link0.source_block_num <= most_recent_ancestor_with_qc.block_num.
-   // If c.links.size() > 1, then by invariant 7 of core, link0.target_block_num <= link1.target_block_num.
-   // Otherwise if c.links.size() == 1, then link0 == link1 and so link0.target_block_num == link1.target_block_num.
-   // Therefore, c.final_on_strong_qc_block_num <= link1.target_block_num.
-
-   assert(c.final_on_strong_qc_block_num <= link1.target_block_num); // Satisfied by justification above.
-
-   // Finality does not advance if a better 3-chain is not found.
-   if (!link1.is_link_strong || (link1.target_block_num < c.links.front().source_block_num)) {
-      return {c.last_final_block_num(), c.links.front().source_block_num, link1.target_block_num};
-   }
-
-   const auto& link2 = c.get_qc_link_from(link1.target_block_num);
-
-   // By the post-condition of get_qc_link_from, link2.source_block_num == link1.target_block_num.
-   // By the invariant on qc_link, link2.target_block_num <= link2.source_block_num.
-   // Therefore, link2.target_block_num <= link1.target_block_num.
-
-   // Wherever link2 is found within c.links, it must be the case that c.links.front().target_block_num <= link2.target_block_num.
-   // This is obvious if c.links.size() == 1 (even though the code would even not get to this point if c.links.size() == 1), and 
-   // for the case where c.links.size() > 1, it is justified by invariant 7 of core.
-   // Therefore, c.last_final_block_num() <= link2.target_block_num.
-
-   return {link2.target_block_num, link2.source_block_num, link1.target_block_num}; 
+   const auto& link = c.get_qc_link_from(most_recent_ancestor_with_qc.block_num);
+
+   // Wherever link is found within c.links, it must be either c.links.front() or
+   // a link after c.links.front() in c.links.
+   // In both cases, by invariant 7 of core,
+   // c.links.front().target_block_num <= link.target_block_num
+   // c.links.front().source_block_num <= link.source_block_num
+   assert(c.last_final_block_num() <= link.target_block_num); // by justification above and c.last_final_block_num() == c.links.front().target_block_num
+   assert(c.links.front().source_block_num <= link.source_block_num); // by justification above
+
+   // 1. By the post-condition of get_qc_link_from, link.source_block_num == most_recent_ancestor_with_qc.block_num.
+   //    By the invariant on qc_link, link.target_block_num <= link.source_block_num.
+   //    Therefore, link.target_block_num <= most_recent_ancestor_with_qc.block_num.
+   //
+   // 2. There must exist some link, call it link0, within c.links where
+   //    link0.target_block_num == c.final_on_strong_qc_block_num and link0.source_block_num <= c.latest_qc_claim().block_num.
+   //    By the precondition, link0.source_block_num <= most_recent_ancestor_with_qc.block_num.
+   //    Since most_recent_ancestor_with_qc.block_num == link.source_block_num,
+   //    we have link0.source_block_num <= link.source_block_num.
+   //    If c.links.size() > 1, then by invariant 7 of core, link0.target_block_num <= link.target_block_num.
+   //    Otherwise if c.links.size() == 1, then link0 == link and so link0.target_block_num == link.target_block_num.
+   //    Therefore, c.final_on_strong_qc_block_num <= link.target_block_num.
+   //
+   //    From 1 and 2, we have c.final_on_strong_qc_block_num <= most_recent_ancestor_with_qc.block_num
+   assert(c.final_on_strong_qc_block_num <= most_recent_ancestor_with_qc.block_num);
+
+   // Use two-chain for finality advance
+   return {link.target_block_num, link.source_block_num, most_recent_ancestor_with_qc.block_num};
 }
 
 core_metadata finality_core::next_metadata(const qc_claim_t& most_recent_ancestor_with_qc) const

libraries/chain/include/eosio/chain/authorization_manager.hpp

@@ -25,7 +25,7 @@ namespace eosio { namespace chain {
          void add_indices();
          void initialize_database();
          void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const;
-         void read_from_snapshot( const snapshot_reader_ptr& snapshot );
+         void read_from_snapshot( const snapshot_reader_ptr& snapshot, snapshot_loaded_row_counter& row_counter );
 
          const permission_object& create_permission( account_name account,
                                                      permission_name name,

libraries/chain/include/eosio/chain/resource_limits.hpp

@@ -72,7 +72,7 @@ namespace eosio { namespace chain {
          void add_indices();
          void initialize_database();
          void add_to_snapshot( const snapshot_writer_ptr& snapshot ) const;
-         void read_from_snapshot( const snapshot_reader_ptr& snapshot );
+         void read_from_snapshot( const snapshot_reader_ptr& snapshot, snapshot_loaded_row_counter& row_counter );
 
          void initialize_account( const account_name& account, bool is_trx_transient );
          void set_block_parameters( const elastic_limit_parameters& cpu_limit_parameters, const elastic_limit_parameters& net_limit_parameters );

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -277,6 +277,8 @@ namespace eosio { namespace chain {
 
       virtual void return_to_header() = 0;
 
+      virtual size_t total_row_count() = 0;
+
       virtual ~snapshot_reader(){};
 
       protected:
@@ -313,6 +315,7 @@ namespace eosio { namespace chain {
          bool empty ( ) override;
          void clear_section() override;
          void return_to_header() override;
+         size_t total_row_count() override;
 
       private:
          const fc::variant& snapshot;
@@ -364,6 +367,7 @@ namespace eosio { namespace chain {
          bool empty ( ) override;
          void clear_section() override;
          void return_to_header() override;
+         size_t total_row_count() override;
 
       private:
          bool validate_section() const;
@@ -385,6 +389,7 @@ namespace eosio { namespace chain {
          bool empty ( ) override;
          void clear_section() override;
          void return_to_header() override;
+         size_t total_row_count() override;
 
       private:
          bool validate_section() const;
@@ -406,4 +411,16 @@ namespace eosio { namespace chain {
 
    };
 
+   struct snapshot_loaded_row_counter {
+      snapshot_loaded_row_counter(const size_t total) : total(total) {}
+      void progress() {
+         if(++count % 50000 == 0 && time(NULL) - last_print >= 5) {
+            ilog("Snapshot initialization ${pct}% complete", ("pct",(unsigned)(((double)count/total)*100)));
+            last_print = time(NULL);
+         }
+      }
+      size_t count = 0;
+      const size_t total = 0;
+      time_t last_print = time(NULL);
+   };
 }}

libraries/chain/resource_limits.cpp

@@ -81,14 +81,15 @@ void resource_limits_manager::add_to_snapshot( const snapshot_writer_ptr& snapsh
    });
 }
 
-void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot ) {
-   resource_index_set::walk_indices([this, &snapshot]( auto utils ){
-      snapshot->read_section<typename decltype(utils)::index_t::value_type>([this]( auto& section ) {
+void resource_limits_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot, snapshot_loaded_row_counter& row_counter ) {
+   resource_index_set::walk_indices([this, &snapshot, &row_counter]( auto utils ){
+      snapshot->read_section<typename decltype(utils)::index_t::value_type>([this, &row_counter]( auto& section ) {
          bool more = !section.empty();
          while(more) {
             decltype(utils)::create(_db, [this, &section, &more]( auto &row ) {
                more = section.read_row(row, _db);
             });
+            row_counter.progress();
          }
       });
    });

libraries/chain/snapshot.cpp

@@ -118,6 +118,16 @@ void variant_snapshot_reader::return_to_header() {
    clear_section();
 }
 
+size_t variant_snapshot_reader::total_row_count() {
+   size_t total = 0;
+
+   const fc::variants& sections = snapshot["sections"].get_array();
+   for(const fc::variant& section : sections)
+      total += section["rows"].get_array().size();
+
+   return total;
+}
+
 ostream_snapshot_writer::ostream_snapshot_writer(std::ostream& snapshot)
 :snapshot(snapshot)
 ,header_pos(snapshot.tellp())
@@ -337,6 +347,34 @@ void istream_snapshot_reader::return_to_header() {
    clear_section();
 }
 
+size_t istream_snapshot_reader::total_row_count() {
+   size_t total = 0;
+
+   auto restore_pos = fc::make_scoped_exit([this,pos=snapshot.tellg()](){
+      snapshot.seekg(pos);
+   });
+
+   const std::streamoff header_size = sizeof(ostream_snapshot_writer::magic_number) + sizeof(current_snapshot_version);
+
+   std::streamoff next_section_pos = header_pos + header_size;
+
+   while(true) {
+      snapshot.seekg(next_section_pos);
+      uint64_t section_size = 0;
+      snapshot.read((char*)&section_size, sizeof(section_size));
+      if(section_size == std::numeric_limits<uint64_t>::max())
+         break;
+      next_section_pos = snapshot.tellg() + std::streamoff(section_size);
+
+      uint64_t row_count = 0;
+      snapshot.read((char*)&row_count, sizeof(row_count));
+
+      total += row_count;
+   }
+
+   return total;
+}
+
 struct istream_json_snapshot_reader_impl {
    uint64_t num_rows;
    uint64_t cur_row;
@@ -422,6 +460,16 @@ void istream_json_snapshot_reader::return_to_header() {
    clear_section();
 }
 
+size_t istream_json_snapshot_reader::total_row_count() {
+   size_t total = 0;
+
+   for(const auto& section : impl->doc.GetObject())
+      if(section.value.IsObject() && section.value.HasMember("num_rows"))
+        total += section.value["num_rows"].GetUint64();
+
+   return total;
+}
+
 integrity_hash_snapshot_writer::integrity_hash_snapshot_writer(fc::sha256::encoder& enc)
 :enc(enc)
 {

libraries/testing/include/eosio/testing/tester.hpp

@@ -77,6 +77,10 @@ namespace eosio::testing {
       yes // tester calls startup() during initialization.
    };
 
+   // Number of chains required for a block to become final.
+   // Current protocol is 2: strong-strong or weak-strong.
+   constexpr size_t num_chains_to_final = 2;
+
    std::ostream& operator<<(std::ostream& os, setup_policy p);
 
    std::vector<uint8_t> read_wasm( const char* fn );
@@ -956,9 +960,9 @@ namespace eosio::testing {
             BOOST_REQUIRE(pt_block->is_proper_svnn_block());
          }
 
-         // lib must advance after 3 blocks
+         // lib must advance after num_chains_to_final blocks
          // -------------------------------
-         for (size_t i=0; i<3; ++i)
+         for (size_t i=0; i<num_chains_to_final; ++i)
             auto b = produce_block();
 
          BOOST_REQUIRE_EQUAL(t.lib_block->block_num(), pt_block->block_num());