From ed4a98e5a67735bcea1153810d60d1e4b28fff5e Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Mon, 15 Jan 2024 15:13:39 +0800
Subject: [PATCH 01/14] Allow start from lib

---
 external/silkworm               |  2 +-
 src/block_conversion_plugin.cpp | 21 +++++++++++++++++
 src/engine_plugin.cpp           | 42 +++++++++++++++++++++++++++++----
 src/engine_plugin.hpp           |  2 ++
 4 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/external/silkworm b/external/silkworm
index 595d358..2a17dce 160000
--- a/external/silkworm
+++ b/external/silkworm
@@ -1 +1 @@
-Subproject commit 595d358780228bdd4c9f53332d55d1e0c5fec13d
+Subproject commit 2a17dceaa80b1633f936678ff5fb245b4908d34b
diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 18e94d4..d3aea0c 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -293,9 +293,30 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   }
 
                   // Remove irreversible evm blocks
+                  // The block at height evm_lib is actually irreversible as well.
+                  // We want to keep at least one irreversible block in the array to deal with forks.
                   while(evm_blocks.front().header.number < evm_lib) {
                      evm_blocks.pop_front();
                   }
+                  // The block at evm_lib should have already been irreversible and inserted.
+                  // So we should be able to recover from it.
+                  // 
+                  // There's one extreme case that we cannot recover: 
+                  // The block is irreversible but the chain db still have the wrong canonical branch. 
+                  // One possible case for this to happen is:
+                  // 1 We are in the wrong branch
+                  // 2 We then restart from an earlier irreverisble block
+                  // 3 Stop in the middle of the catchup process so that lib is written but the canonical branch is not updated yet. 
+                  // 4 Restart, the lib recored may not in the canonial chain.
+                  // It should be possible for this to happen since we process 5000 blocks together for irreversible blocks.
+                  // 
+                  // In this case, the "lib" block we start from will be a block in the "wrong" canonical chain. 
+                  // The first fetched block will then be rejected as it cannot link.
+                  // Checking the head block may help detecting such case. 
+                  // If the head block is not available for some reason, we cannot recover.
+                  // But the situation is rare and at least we will not recover into a wrong state.
+                  // We can always use the "ship-start-from-canonical-height" to manually recover.
+                  appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
                }
             }
          );
diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index 7cb2e1f..0121805 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -139,13 +139,26 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
       std::optional<silkworm::Block> get_canonical_block_at_height(std::optional<uint64_t> height) {
          uint64_t target = 0;
          if (!height) {
-            auto header = get_head_canonical_header();
-            if(!header) return {};
-            target = header->number;
+            auto lib = get_evm_lib();
+           
+            if (lib) {
+               target = *lib;
+            }
+            else {
+               // no lib, might be the first run from an old db.
+               // Use the old logic.
+               auto header = get_head_canonical_header();
+               if (!header) {
+                  return {};
+               }
+               else {
+                  target = header->number;
+               }
+            }
          }
          else {
-            // Do not check canonical header.
-            // If there's anything wrong in that table, overriding here has some chance to fix it.
+            // Do not check canonical header or lib.
+            // If there's anything wrong, overriding here has some chance to fix it.
             target = *height;
          }
 
@@ -156,6 +169,17 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
          return block;
       }
 
+      void record_evm_lib(uint64_t height) {
+         silkworm::db::RWTxn txn(db_env);
+         write_runtime_states_u64(txn, height, silkworm::db::RuntimeState::kLibProcessed);
+         txn.commit();
+      }
+
+      std::optional<uint64_t> get_evm_lib() {
+         silkworm::db::ROTxn txn(db_env);
+         return read_runtime_states_u64(txn, silkworm::db::RuntimeState::kLibProcessed);
+      }
+
       std::optional<silkworm::BlockHeader> get_genesis_header() {
          silkworm::db::ROTxn txn(db_env);
          return silkworm::db::read_canonical_header(txn, 0);
@@ -224,6 +248,14 @@ std::optional<silkworm::Block> engine_plugin::get_canonical_block_at_height(std:
    return my->get_canonical_block_at_height(height);
 }
 
+void engine_plugin::record_evm_lib(uint64_t height) {
+   return my->record_evm_lib(height);
+}
+
+std::optional<uint64_t> engine_plugin::get_evm_lib() {
+   return my->get_evm_lib();
+}
+
 std::optional<silkworm::BlockHeader> engine_plugin::get_genesis_header() {
    return my->get_genesis_header();
 }
diff --git a/src/engine_plugin.hpp b/src/engine_plugin.hpp
index 09fa2c5..cb55423 100644
--- a/src/engine_plugin.hpp
+++ b/src/engine_plugin.hpp
@@ -24,6 +24,8 @@ class engine_plugin : public appbase::plugin<engine_plugin> {
       std::optional<silkworm::BlockHeader> get_head_canonical_header();
       std::optional<silkworm::Block> get_canonical_block_at_height(std::optional<uint64_t> height);
       std::optional<silkworm::BlockHeader> get_genesis_header();
+      void record_evm_lib(uint64_t height);
+      std::optional<uint64_t> get_evm_lib();
 
    private:
       std::unique_ptr<class engine_plugin_impl> my;

From 3665b48598a372176a2b3109ac839100a8237160 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Mon, 15 Jan 2024 15:23:32 +0800
Subject: [PATCH 02/14] Missing parts in last commit

---
 src/block_conversion_plugin.cpp | 7 ++++---
 src/engine_plugin.cpp           | 8 ++++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index d3aea0c..3e35f0a 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -312,10 +312,11 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   // 
                   // In this case, the "lib" block we start from will be a block in the "wrong" canonical chain. 
                   // The first fetched block will then be rejected as it cannot link.
-                  // Checking the head block may help detecting such case. 
-                  // If the head block is not available for some reason, we cannot recover.
+                  // The only relatively easy way to protect against such case is checking whether the evm_lib is canonical here.
                   // But the situation is rare and at least we will not recover into a wrong state.
-                  // We can always use the "ship-start-from-canonical-height" to manually recover.
+                  // And we can always use the "ship-start-from-canonical-height" to manually recover.
+                  // So we decide we will not do the check for now.
+                  
                   appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
                }
             }
diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index 0121805..dd66e02 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -140,14 +140,18 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
          uint64_t target = 0;
          if (!height) {
             auto lib = get_evm_lib();
-           
+            auto header = get_head_canonical_header();
             if (lib) {
                target = *lib;
+               // Make sure we start from number smaller than or equal to head if possible.
+               // But ignore the case where head is not avaiable
+               if (header && target > header->number) {
+                  target = header->number;
+               }
             }
             else {
                // no lib, might be the first run from an old db.
                // Use the old logic.
-               auto header = get_head_canonical_header();
                if (!header) {
                   return {};
                }

From aaac9b5f607800bb076b6c35180a13319150d2e1 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Mon, 15 Jan 2024 15:40:47 +0800
Subject: [PATCH 03/14] Add some debug message.

---
 src/engine_plugin.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index dd66e02..054c15c 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -138,25 +138,31 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
 
       std::optional<silkworm::Block> get_canonical_block_at_height(std::optional<uint64_t> height) {
          uint64_t target = 0;
+         SILK_INFO << "Determining effective canonical header.";
          if (!height) {
             auto lib = get_evm_lib();
             auto header = get_head_canonical_header();
             if (lib) {
+               SILK_INFO << "Stored LIB at: " << "#" << *lib;
                target = *lib;
                // Make sure we start from number smaller than or equal to head if possible.
                // But ignore the case where head is not avaiable
                if (header && target > header->number) {
                   target = header->number;
+                  SILK_INFO << "Canonical header is at lower height, set the target height to: " << "#" << target;
                }
             }
             else {
+               SILK_INFO << "Stored LIB not available.";
                // no lib, might be the first run from an old db.
                // Use the old logic.
                if (!header) {
+                  SILK_INFO << "Failed to read canonical header";
                   return {};
                }
                else {
                   target = header->number;
+                  SILK_INFO << "Canonical header at: " << "#" << target;
                }
             }
          }
@@ -164,6 +170,7 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
             // Do not check canonical header or lib.
             // If there's anything wrong, overriding here has some chance to fix it.
             target = *height;
+            SILK_INFO << "Command line options set the canonical height as " << "#" << target;
          }
 
          silkworm::db::ROTxn txn(db_env);

From d471bfdef8bc824f0ef86a671471bf54d816efc5 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Wed, 17 Jan 2024 15:16:39 +0800
Subject: [PATCH 04/14] Add more debug outputs.

---
 src/block_conversion_plugin.cpp | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 3e35f0a..5c05529 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -277,6 +277,14 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                set_upper_bound(curr, *new_block);
 
                // Calculate last irreversible EVM block
+               SILK_INFO << "Pruning native block queue according to lib in new block:"
+                        << "#" << new_block->lib;
+               SILK_INFO << "Size BEFORE pruning: "
+                         << native_blocks.size();
+               if (native_blocks.size() > 0 ) {
+                  SILK_INFO << "Begin block: "
+                         << "#" << native_blocks.begin()->block_num;
+               }
                std::optional<uint64_t> lib_timestamp;
                auto it = std::upper_bound(native_blocks.begin(), native_blocks.end(), new_block->lib, [](uint32_t lib, const auto& nb) { return lib < nb.block_num; });
                if(it != native_blocks.begin()) {
@@ -286,7 +294,8 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
 
                if( lib_timestamp ) {
                   auto evm_lib = timestamp_to_evm_block_num(*lib_timestamp) - 1;
-
+                  SILK_INFO << "Pruning according to EVM LIB: "
+                         << "#" << evm_lib;
                   // Remove irreversible native blocks
                   while(timestamp_to_evm_block_num(native_blocks.front().timestamp) < evm_lib) {
                      native_blocks.pop_front();
@@ -316,9 +325,15 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   // But the situation is rare and at least we will not recover into a wrong state.
                   // And we can always use the "ship-start-from-canonical-height" to manually recover.
                   // So we decide we will not do the check for now.
-                  
+
                   appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
+
+                  SILK_INFO << "Stored EVM LIB: "
+                         << "#" << evm_lib;
+
                }
+               SILK_INFO << "Size AFTER pruning: "
+                         << native_blocks.size();
             }
          );
       }

From 185305282ab8d1675c4f2d70dd9b453b306eb100 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Wed, 17 Jan 2024 15:32:43 +0800
Subject: [PATCH 05/14] More debugging texts.

---
 src/block_conversion_plugin.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 5c05529..98af79e 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -304,6 +304,13 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   // Remove irreversible evm blocks
                   // The block at height evm_lib is actually irreversible as well.
                   // We want to keep at least one irreversible block in the array to deal with forks.
+                  SILK_INFO << "EVM Block Queue Size BEFORE pruning: "
+                           << evm_blocks.size();
+                  if (evm_blocks.size() > 0 ) {
+                     SILK_INFO << "EVM Block Queue Begin block: "
+                           << "#" << evm_blocks.begin()->header.number;
+                  }
+
                   while(evm_blocks.front().header.number < evm_lib) {
                      evm_blocks.pop_front();
                   }

From cefc6630b8ac09cc94d9bc1b6d4f7586a7db2bba Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Wed, 17 Jan 2024 15:36:32 +0800
Subject: [PATCH 06/14] More debugging text

---
 src/block_conversion_plugin.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 98af79e..ff5c654 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -314,6 +314,8 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   while(evm_blocks.front().header.number < evm_lib) {
                      evm_blocks.pop_front();
                   }
+                  SILK_INFO << "EVM Block Queue Size AFTER pruning: "
+                           << evm_blocks.size();
                   // The block at evm_lib should have already been irreversible and inserted.
                   // So we should be able to recover from it.
                   // 

From 03c89f56f0947482d956c4e17047e776a81369de Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Wed, 17 Jan 2024 15:42:27 +0800
Subject: [PATCH 07/14] More debugging string

---
 src/engine_plugin.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index 054c15c..46e485c 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -181,6 +181,7 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
       }
 
       void record_evm_lib(uint64_t height) {
+         SILK_INFO << "Saving EVM LIB " << "#" << height;
          silkworm::db::RWTxn txn(db_env);
          write_runtime_states_u64(txn, height, silkworm::db::RuntimeState::kLibProcessed);
          txn.commit();

From 773b3ce6887b44993ca2855cec6aa8884df87c90 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Wed, 17 Jan 2024 15:55:55 +0800
Subject: [PATCH 08/14] More debuggin.

---
 src/engine_plugin.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index 46e485c..2668ada 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -182,9 +182,18 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
 
       void record_evm_lib(uint64_t height) {
          SILK_INFO << "Saving EVM LIB " << "#" << height;
+         try {
          silkworm::db::RWTxn txn(db_env);
          write_runtime_states_u64(txn, height, silkworm::db::RuntimeState::kLibProcessed);
          txn.commit();
+         }
+         catch (const std::exception& e) {
+            SILK_ERROR << "exception: " << e.what();
+         }
+         catch(...) {
+            SILK_INFO << "Unknown exception";
+         }
+         SILK_INFO << "Finished EVM LIB " << "#" << height;
       }
 
       std::optional<uint64_t> get_evm_lib() {

From eefbabdac56a507b465767ec91ec687f2578d4fe Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Thu, 18 Jan 2024 11:37:10 +0800
Subject: [PATCH 09/14] Try get tx from exec_engine.

---
 src/blockchain_plugin.cpp | 25 +++++++++++++++++++++++--
 src/blockchain_plugin.hpp |  1 +
 src/engine_plugin.cpp     |  4 ++--
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/blockchain_plugin.cpp b/src/blockchain_plugin.cpp
index 983687a..6428e72 100644
--- a/src/blockchain_plugin.cpp
+++ b/src/blockchain_plugin.cpp
@@ -5,14 +5,31 @@
 #include <map>
 #include <string>
 
+#include <boost/asio/io_context.hpp>
+
 #include <silkworm/node/stagedsync/types.hpp>
 #include <silkworm/node/stagedsync/execution_engine.hpp>
 
+class ExecutionEngineEx : public silkworm::stagedsync::ExecutionEngine {
+   public :
+   
+   ExecutionEngineEx(boost::asio::io_context& io, silkworm::NodeSettings& settings, silkworm::db::RWAccess dba) : ExecutionEngine(io, settings, dba) {
+
+   }
+   silkworm::db::RWTxn& get_tx() {
+      return main_chain_.tx();
+   }
+};
+
 using sys = sys_plugin;
 class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_impl> {
    public:
       blockchain_plugin_impl() = default;
 
+      silkworm::db::RWTxn& get_tx() {
+         return exec_engine->get_tx();
+      }
+
       inline void init() {
          SILK_DEBUG << "blockchain_plugin_impl INIT";
          db_env = appbase::app().get_plugin<engine_plugin>().get_db();
@@ -26,7 +43,7 @@ class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_im
 
                   SILK_DEBUG << "EVM Block " << new_block->header.number;
                   if(!exec_engine) {
-                     exec_engine = std::make_unique<silkworm::stagedsync::ExecutionEngine>(appbase::app().get_io_context(), *node_settings, silkworm::db::RWAccess{*db_env});
+                     exec_engine = std::make_unique<ExecutionEngineEx>(appbase::app().get_io_context(), *node_settings, silkworm::db::RWAccess{*db_env});
                      exec_engine->open();
                   }
 
@@ -56,7 +73,7 @@ class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_im
       silkworm::NodeSettings*                                 node_settings;
       mdbx::env*                                              db_env;
       channels::evm_blocks::channel_type::handle              evm_blocks_subscription;
-      std::unique_ptr<silkworm::stagedsync::ExecutionEngine>  exec_engine;
+      std::unique_ptr<ExecutionEngineEx>  exec_engine;
 };
 
 blockchain_plugin::blockchain_plugin() : my(new blockchain_plugin_impl()) {}
@@ -78,3 +95,7 @@ void blockchain_plugin::plugin_shutdown() {
    my->shutdown();
    SILK_INFO << "Shutdown Blockchain plugin";
 }
+
+silkworm::db::RWTxn& blockchain_plugin::get_tx() {
+   return my->get_tx();
+}
\ No newline at end of file
diff --git a/src/blockchain_plugin.hpp b/src/blockchain_plugin.hpp
index 2f778ac..c05713c 100644
--- a/src/blockchain_plugin.hpp
+++ b/src/blockchain_plugin.hpp
@@ -15,6 +15,7 @@ class blockchain_plugin : public appbase::plugin<blockchain_plugin> {
       void plugin_initialize(const appbase::variables_map& options);
       void plugin_startup();
       void plugin_shutdown();
+      silkworm::db::RWTxn& get_tx();
 
    private:
       std::unique_ptr<class blockchain_plugin_impl> my;
diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index 2668ada..fc594eb 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -1,4 +1,5 @@
 #include "engine_plugin.hpp"
+#include "blockchain_plugin.hpp"
 #include "channels.hpp"
 
 #include <filesystem>
@@ -183,9 +184,8 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
       void record_evm_lib(uint64_t height) {
          SILK_INFO << "Saving EVM LIB " << "#" << height;
          try {
-         silkworm::db::RWTxn txn(db_env);
+         auto& txn = appbase::app().get_plugin<blockchain_plugin>().get_tx();
          write_runtime_states_u64(txn, height, silkworm::db::RuntimeState::kLibProcessed);
-         txn.commit();
          }
          catch (const std::exception& e) {
             SILK_ERROR << "exception: " << e.what();

From 2df4fb161b4f6120975dd8235c953c041a174725 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Thu, 18 Jan 2024 11:52:05 +0800
Subject: [PATCH 10/14] Try call the tx in the correct thread.

---
 src/block_conversion_plugin.cpp | 13 +++++++++++--
 src/block_conversion_plugin.hpp |  1 +
 src/blockchain_plugin.cpp       |  1 +
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index ff5c654..68f8247 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -335,8 +335,8 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   // And we can always use the "ship-start-from-canonical-height" to manually recover.
                   // So we decide we will not do the check for now.
 
-                  appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
-
+                  //appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
+                  evm_lib_ = evm_lib;
                   SILK_INFO << "Stored EVM LIB: "
                          << "#" << evm_lib;
 
@@ -353,6 +353,10 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
          return tx;
       }
 
+      uint64_t get_evm_lib() {
+         return evm_lib_;
+      }
+
       void shutdown() {}
 
       std::list<channels::native_block>             native_blocks;
@@ -361,6 +365,7 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
       channels::native_blocks::channel_type::handle native_blocks_subscription;
       std::optional<eosevm::block_mapping>          bm;
       uint64_t                                      evm_contract_name = 0;
+      uint64_t                                      evm_lib_;
 };
 
 block_conversion_plugin::block_conversion_plugin() : my(new block_conversion_plugin_impl()) {}
@@ -382,3 +387,7 @@ void block_conversion_plugin::plugin_shutdown() {
    my->shutdown();
    SILK_INFO << "Shutdown block_conversion plugin";
 }
+
+uint64_t block_conversion_plugin::get_evm_lib() {
+   return my->get_evm_lib();
+}
\ No newline at end of file
diff --git a/src/block_conversion_plugin.hpp b/src/block_conversion_plugin.hpp
index e6cadf7..9a24516 100644
--- a/src/block_conversion_plugin.hpp
+++ b/src/block_conversion_plugin.hpp
@@ -23,6 +23,7 @@ class block_conversion_plugin : public appbase::plugin<block_conversion_plugin>
       void plugin_shutdown();
 
       uint32_t get_block_stride() const;
+      uint64_t get_evm_lib();
 
    private:
       std::unique_ptr<class block_conversion_plugin_impl> my;
diff --git a/src/blockchain_plugin.cpp b/src/blockchain_plugin.cpp
index 6428e72..0ec6027 100644
--- a/src/blockchain_plugin.cpp
+++ b/src/blockchain_plugin.cpp
@@ -49,6 +49,7 @@ class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_im
 
                   exec_engine->insert_block(new_block);
                   if(!(++block_count % 5000) || !new_block->irreversible) {
+                     write_runtime_states_u64(exec_engine->get_tx(), appbase::app().get_plugin<block_conversion_plugin>().get_evm_lib(), silkworm::db::RuntimeState::kLibProcessed);
                      exec_engine->verify_chain(new_block->header.hash());
                      block_count=0;
                   }

From 84d9231c951b11d9bd587da439d1f9a32874a8e7 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Thu, 18 Jan 2024 12:09:02 +0800
Subject: [PATCH 11/14] Cleanup.

---
 src/block_conversion_plugin.cpp | 23 ++---------------------
 src/blockchain_plugin.cpp       |  5 ++++-
 src/engine_plugin.cpp           | 27 ++-------------------------
 src/engine_plugin.hpp           |  2 --
 4 files changed, 8 insertions(+), 49 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 68f8247..70ce529 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -277,14 +277,6 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                set_upper_bound(curr, *new_block);
 
                // Calculate last irreversible EVM block
-               SILK_INFO << "Pruning native block queue according to lib in new block:"
-                        << "#" << new_block->lib;
-               SILK_INFO << "Size BEFORE pruning: "
-                         << native_blocks.size();
-               if (native_blocks.size() > 0 ) {
-                  SILK_INFO << "Begin block: "
-                         << "#" << native_blocks.begin()->block_num;
-               }
                std::optional<uint64_t> lib_timestamp;
                auto it = std::upper_bound(native_blocks.begin(), native_blocks.end(), new_block->lib, [](uint32_t lib, const auto& nb) { return lib < nb.block_num; });
                if(it != native_blocks.begin()) {
@@ -294,8 +286,7 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
 
                if( lib_timestamp ) {
                   auto evm_lib = timestamp_to_evm_block_num(*lib_timestamp) - 1;
-                  SILK_INFO << "Pruning according to EVM LIB: "
-                         << "#" << evm_lib;
+
                   // Remove irreversible native blocks
                   while(timestamp_to_evm_block_num(native_blocks.front().timestamp) < evm_lib) {
                      native_blocks.pop_front();
@@ -304,18 +295,11 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   // Remove irreversible evm blocks
                   // The block at height evm_lib is actually irreversible as well.
                   // We want to keep at least one irreversible block in the array to deal with forks.
-                  SILK_INFO << "EVM Block Queue Size BEFORE pruning: "
-                           << evm_blocks.size();
-                  if (evm_blocks.size() > 0 ) {
-                     SILK_INFO << "EVM Block Queue Begin block: "
-                           << "#" << evm_blocks.begin()->header.number;
-                  }
 
                   while(evm_blocks.front().header.number < evm_lib) {
                      evm_blocks.pop_front();
                   }
-                  SILK_INFO << "EVM Block Queue Size AFTER pruning: "
-                           << evm_blocks.size();
+
                   // The block at evm_lib should have already been irreversible and inserted.
                   // So we should be able to recover from it.
                   // 
@@ -337,9 +321,6 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
 
                   //appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
                   evm_lib_ = evm_lib;
-                  SILK_INFO << "Stored EVM LIB: "
-                         << "#" << evm_lib;
-
                }
                SILK_INFO << "Size AFTER pruning: "
                          << native_blocks.size();
diff --git a/src/blockchain_plugin.cpp b/src/blockchain_plugin.cpp
index 0ec6027..e570105 100644
--- a/src/blockchain_plugin.cpp
+++ b/src/blockchain_plugin.cpp
@@ -49,7 +49,10 @@ class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_im
 
                   exec_engine->insert_block(new_block);
                   if(!(++block_count % 5000) || !new_block->irreversible) {
-                     write_runtime_states_u64(exec_engine->get_tx(), appbase::app().get_plugin<block_conversion_plugin>().get_evm_lib(), silkworm::db::RuntimeState::kLibProcessed);
+                     uint64_t evm_lib = appbase::app().get_plugin<block_conversion_plugin>().get_evm_lib();
+                     SILK_INFO << "Storing EVM Lib : " << "#" << evm_lib;
+                     write_runtime_states_u64(exec_engine->get_tx(), evm_lib, silkworm::db::RuntimeState::kLibProcessed);
+                     
                      exec_engine->verify_chain(new_block->header.hash());
                      block_count=0;
                   }
diff --git a/src/engine_plugin.cpp b/src/engine_plugin.cpp
index fc594eb..880c223 100644
--- a/src/engine_plugin.cpp
+++ b/src/engine_plugin.cpp
@@ -141,7 +141,7 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
          uint64_t target = 0;
          SILK_INFO << "Determining effective canonical header.";
          if (!height) {
-            auto lib = get_evm_lib();
+            auto lib = get_stored_evm_lib();
             auto header = get_head_canonical_header();
             if (lib) {
                SILK_INFO << "Stored LIB at: " << "#" << *lib;
@@ -181,22 +181,7 @@ class engine_plugin_impl : std::enable_shared_from_this<engine_plugin_impl> {
          return block;
       }
 
-      void record_evm_lib(uint64_t height) {
-         SILK_INFO << "Saving EVM LIB " << "#" << height;
-         try {
-         auto& txn = appbase::app().get_plugin<blockchain_plugin>().get_tx();
-         write_runtime_states_u64(txn, height, silkworm::db::RuntimeState::kLibProcessed);
-         }
-         catch (const std::exception& e) {
-            SILK_ERROR << "exception: " << e.what();
-         }
-         catch(...) {
-            SILK_INFO << "Unknown exception";
-         }
-         SILK_INFO << "Finished EVM LIB " << "#" << height;
-      }
-
-      std::optional<uint64_t> get_evm_lib() {
+      std::optional<uint64_t> get_stored_evm_lib() {
          silkworm::db::ROTxn txn(db_env);
          return read_runtime_states_u64(txn, silkworm::db::RuntimeState::kLibProcessed);
       }
@@ -269,14 +254,6 @@ std::optional<silkworm::Block> engine_plugin::get_canonical_block_at_height(std:
    return my->get_canonical_block_at_height(height);
 }
 
-void engine_plugin::record_evm_lib(uint64_t height) {
-   return my->record_evm_lib(height);
-}
-
-std::optional<uint64_t> engine_plugin::get_evm_lib() {
-   return my->get_evm_lib();
-}
-
 std::optional<silkworm::BlockHeader> engine_plugin::get_genesis_header() {
    return my->get_genesis_header();
 }
diff --git a/src/engine_plugin.hpp b/src/engine_plugin.hpp
index cb55423..09fa2c5 100644
--- a/src/engine_plugin.hpp
+++ b/src/engine_plugin.hpp
@@ -24,8 +24,6 @@ class engine_plugin : public appbase::plugin<engine_plugin> {
       std::optional<silkworm::BlockHeader> get_head_canonical_header();
       std::optional<silkworm::Block> get_canonical_block_at_height(std::optional<uint64_t> height);
       std::optional<silkworm::BlockHeader> get_genesis_header();
-      void record_evm_lib(uint64_t height);
-      std::optional<uint64_t> get_evm_lib();
 
    private:
       std::unique_ptr<class engine_plugin_impl> my;

From 240ae4ab01c60e276eefb63a01168810d3879b51 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Thu, 18 Jan 2024 12:14:11 +0800
Subject: [PATCH 12/14] Fix text.

---
 src/block_conversion_plugin.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 70ce529..83c4cb7 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -322,8 +322,6 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                   //appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
                   evm_lib_ = evm_lib;
                }
-               SILK_INFO << "Size AFTER pruning: "
-                         << native_blocks.size();
             }
          );
       }

From fb3d70df510bba3e9ea2faaef48034ce929f80fa Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Thu, 18 Jan 2024 15:30:15 +0800
Subject: [PATCH 13/14] Update comments and debug texts.

---
 src/block_conversion_plugin.cpp | 22 ++--------------------
 src/blockchain_plugin.cpp       | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 22 deletions(-)

diff --git a/src/block_conversion_plugin.cpp b/src/block_conversion_plugin.cpp
index 83c4cb7..4e280d5 100644
--- a/src/block_conversion_plugin.cpp
+++ b/src/block_conversion_plugin.cpp
@@ -285,6 +285,7 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                }
 
                if( lib_timestamp ) {
+                  // Minus 1 to make sure the resulting block is a completed EVM block built from irreversible EOS blocks.
                   auto evm_lib = timestamp_to_evm_block_num(*lib_timestamp) - 1;
 
                   // Remove irreversible native blocks
@@ -300,26 +301,7 @@ class block_conversion_plugin_impl : std::enable_shared_from_this<block_conversi
                      evm_blocks.pop_front();
                   }
 
-                  // The block at evm_lib should have already been irreversible and inserted.
-                  // So we should be able to recover from it.
-                  // 
-                  // There's one extreme case that we cannot recover: 
-                  // The block is irreversible but the chain db still have the wrong canonical branch. 
-                  // One possible case for this to happen is:
-                  // 1 We are in the wrong branch
-                  // 2 We then restart from an earlier irreverisble block
-                  // 3 Stop in the middle of the catchup process so that lib is written but the canonical branch is not updated yet. 
-                  // 4 Restart, the lib recored may not in the canonial chain.
-                  // It should be possible for this to happen since we process 5000 blocks together for irreversible blocks.
-                  // 
-                  // In this case, the "lib" block we start from will be a block in the "wrong" canonical chain. 
-                  // The first fetched block will then be rejected as it cannot link.
-                  // The only relatively easy way to protect against such case is checking whether the evm_lib is canonical here.
-                  // But the situation is rare and at least we will not recover into a wrong state.
-                  // And we can always use the "ship-start-from-canonical-height" to manually recover.
-                  // So we decide we will not do the check for now.
-
-                  //appbase::app().get_plugin<engine_plugin>().record_evm_lib(evm_lib);
+                  // Record the height of this complete EVM block from irreversible EOS blocks.
                   evm_lib_ = evm_lib;
                }
             }
diff --git a/src/blockchain_plugin.cpp b/src/blockchain_plugin.cpp
index e570105..68f6a6e 100644
--- a/src/blockchain_plugin.cpp
+++ b/src/blockchain_plugin.cpp
@@ -49,10 +49,20 @@ class blockchain_plugin_impl : std::enable_shared_from_this<blockchain_plugin_im
 
                   exec_engine->insert_block(new_block);
                   if(!(++block_count % 5000) || !new_block->irreversible) {
+                     // Get the last complete EVM block from irreversible EOS blocks.
+                     // The height is uint64_t so we can get it as a whole without worrying about atomicity.
+                     // Even some data races happen, it's fine in our scenario to read old data as starting from earlier blocks is always safer.
                      uint64_t evm_lib = appbase::app().get_plugin<block_conversion_plugin>().get_evm_lib();
-                     SILK_INFO << "Storing EVM Lib : " << "#" << evm_lib;
+                     SILK_INFO << "Storing EVM Lib: " << "#" << evm_lib;
+
+                     // Storing the EVM block height of the last complete block from irreversible EOS blocks.
+                     // We have to do this in this thread with the tx instance stored in exec_engine due to the lmitation of MDBX.
+                     // Note there's no need to commit here as the tx is borrowed. ExecutionEngine will manange the commits.
+                     // There's some other advantage to save this height in this way: 
+                     // If the system is shut down during catching up irreversible blocks, i.e. in the middle of the 5000 block run,
+                     // saving the height in this way can minimize the possibility having a stored height that is higher than the canonical header.
                      write_runtime_states_u64(exec_engine->get_tx(), evm_lib, silkworm::db::RuntimeState::kLibProcessed);
-                     
+
                      exec_engine->verify_chain(new_block->header.hash());
                      block_count=0;
                   }

From 90bcb5431a6ba7780f8f9945067a0729dceed9b0 Mon Sep 17 00:00:00 2001
From: yarkin <yarkin.who@gmail.com>
Date: Tue, 23 Jan 2024 22:18:40 +0800
Subject: [PATCH 14/14] Update silkworm

---
 external/silkworm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/external/silkworm b/external/silkworm
index 2a17dce..dd05bab 160000
--- a/external/silkworm
+++ b/external/silkworm
@@ -1 +1 @@
-Subproject commit 2a17dceaa80b1633f936678ff5fb245b4908d34b
+Subproject commit dd05bab8b702cf67b9691f0eb38c655152672067