GH-1039 If a BP then do not allow apply_blocks to run past block dead…

…line

plugins/producer_plugin/producer_plugin.cpp

@@ -2112,6 +2112,32 @@ producer_plugin_impl::determine_pending_block_mode(const fc::time_point& now,
       }
    }
 
+   if (in_producing_mode()) {
+      uint32_t production_round_index = block_timestamp_type(block_time).slot % chain::config::producer_repetitions;
+      if (production_round_index == 0) {
+         // first block of our round, wait for block production window
+         const auto start_block_time = block_time.to_time_point() - fc::microseconds(config::block_interval_us);
+         if (now < start_block_time) {
+            fc_dlog(_log, "Not starting block until ${bt}", ("bt", start_block_time));
+            schedule_delayed_production_loop(weak_from_this(), start_block_time);
+            return start_block_result::waiting_for_production;
+         }
+      }
+   }
+
+   // Calculate block deadline for both produced blocks and speculative blocks. Even though speculative blocks are
+   // ephemeral, re-start them at block intervals so that speculative transactions execute with current block times.
+   _pending_block_deadline = block_timing_util::calculate_producing_block_deadline(_produce_block_cpu_effort, block_time);
+   if (in_speculating_mode()) { // if we are producing, then produce block even if deadline has passed
+      // For a speculative block there is no reason to start a block that will immediately be re-started.
+      // Normally a block should come in during this time; if not, create a speculative block every block_interval_ms.
+      // Ideally, we would abort a transaction as soon as a block is received. For now, this block deadline allows for a
+      // full block interval to attempt to fit in transactions.
+      if (now + fc::milliseconds(config::block_interval_ms) > _pending_block_deadline) {
+         _pending_block_deadline = now + fc::milliseconds(config::block_interval_ms);
+      }
+   }
+
    return start_block_result::succeeded;
 }
 
@@ -2123,10 +2149,17 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
 
    abort_block();
 
+   auto apply_blocks = [&]() -> controller::apply_blocks_result {
+      try {
+         return chain.apply_blocks([this](const transaction_metadata_ptr& trx) { _unapplied_transactions.add_forked(trx); },
+                                   [this](const transaction_id_type& id) { return _unapplied_transactions.get_trx(id); });
+      } catch (...) {} // errors logged in apply_blocks
+      return controller::apply_blocks_result::incomplete;
+   };
+
    // producers need to be able to start producing on schedule, do not apply blocks as it might take a long time to apply
    if (!is_configured_producer()) {
-      auto r = chain.apply_blocks([this](const transaction_metadata_ptr& trx) { _unapplied_transactions.add_forked(trx); },
-                                  [this](const transaction_id_type& id) { return _unapplied_transactions.get_trx(id); });
+      auto r = apply_blocks();
       if (r != controller::apply_blocks_result::complete)
          return start_block_result::waiting_for_block;
    }
@@ -2150,8 +2183,8 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
    if (is_configured_producer() && in_speculating_mode()) {
       // if not producing right now, see if any blocks have come in that need to be applied
       const block_id_type head_id = head.id();
-      chain.apply_blocks([this](const transaction_metadata_ptr& trx) { _unapplied_transactions.add_forked(trx); },
-                         [this](const transaction_id_type& id) { return _unapplied_transactions.get_trx(id); });
+      schedule_delayed_production_loop(weak_from_this(), _pending_block_deadline); // interrupt apply_blocks at deadline
+      apply_blocks();
       head = chain.head();
       if (head_id != head.id()) { // blocks were applied
          now                = fc::time_point::now();
@@ -2164,31 +2197,6 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() {
       }
    }
 
-   if (in_producing_mode()) {
-      uint32_t production_round_index = block_timestamp_type(block_time).slot % chain::config::producer_repetitions;
-      if (production_round_index == 0) {
-         // first block of our round, wait for block production window
-         const auto start_block_time = block_time.to_time_point() - fc::microseconds(config::block_interval_us);
-         if (now < start_block_time) {
-            fc_dlog(_log, "Not starting block until ${bt}", ("bt", start_block_time));
-            schedule_delayed_production_loop(weak_from_this(), start_block_time);
-            return start_block_result::waiting_for_production;
-         }
-      }
-   }
-
-   // Calculate block deadline for both produced blocks and speculative blocks. Even though speculative blocks are
-   // ephemeral, re-start them at block intervals so that speculative transactions execute with current block times.
-   _pending_block_deadline = block_timing_util::calculate_producing_block_deadline(_produce_block_cpu_effort, block_time);
-   if (in_speculating_mode()) { // if we are producing, then produce block even if deadline has passed
-      // For a speculative block there is no reason to start a block that will immediately be re-started.
-      // Normally a block should come in during this time; if not, create a speculative block every block_interval_ms.
-      // Ideally, we would abort a transaction as soon as a block is received. For now, this block deadline allows for a
-      // full block interval to attempt to fit in transactions.
-      if (now + fc::milliseconds(config::block_interval_ms) > _pending_block_deadline) {
-         _pending_block_deadline = now + fc::milliseconds(config::block_interval_ms);
-      }
-   }
    const auto& preprocess_deadline = _pending_block_deadline;
 
    const block_num_type head_block_num    = head.block_num();