rpcdaemon: generate transaction receipts on-the-fly (#2375)

silkworm/rpc/commands/erigon_api.cpp

@@ -202,7 +202,7 @@ Task<void> ErigonRpcApi::handle_erigon_get_block_receipts_by_block_hash(const nl
             co_await tx->close();  // RAII not (yet) available with coroutines
             co_return;
         }
-        auto receipts{co_await core::get_receipts(*tx, *block_with_hash)};
+        auto receipts{co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_)};
         SILK_TRACE << "#receipts: " << receipts.size();
 
         const auto block{block_with_hash->block};
@@ -414,7 +414,7 @@ Task<void> ErigonRpcApi::handle_erigon_get_logs_by_hash(const nlohmann::json& re
             co_await tx->close();  // RAII not (yet) available with coroutines
             co_return;
         }
-        const auto receipts{co_await core::get_receipts(*tx, *block_with_hash)};
+        const auto receipts{co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_)};
         SILK_DEBUG << "receipts.size(): " << receipts.size();
         std::vector<Logs> logs{};
         logs.reserve(receipts.size());

silkworm/rpc/commands/erigon_api.hpp

@@ -25,6 +25,7 @@
 #include <silkworm/db/kv/api/state_cache.hpp>
 #include <silkworm/infra/concurrency/private_service.hpp>
 #include <silkworm/infra/concurrency/shared_service.hpp>
+#include <silkworm/rpc/common/worker_pool.hpp>
 #include <silkworm/rpc/ethbackend/backend.hpp>
 #include <silkworm/rpc/ethdb/database.hpp>
 #include <silkworm/rpc/json/types.hpp>
@@ -37,10 +38,11 @@ namespace silkworm::rpc::commands {
 
 class ErigonRpcApi {
   public:
-    explicit ErigonRpcApi(boost::asio::io_context& io_context)
+    explicit ErigonRpcApi(boost::asio::io_context& io_context, WorkerPool& workers)
         : block_cache_{must_use_shared_service<BlockCache>(io_context)},
           database_{must_use_private_service<ethdb::Database>(io_context)},
-          backend_{must_use_private_service<ethbackend::BackEnd>(io_context)} {}
+          backend_{must_use_private_service<ethbackend::BackEnd>(io_context)},
+          workers_{workers} {}
     virtual ~ErigonRpcApi() = default;
 
     ErigonRpcApi(const ErigonRpcApi&) = delete;
@@ -66,6 +68,7 @@ class ErigonRpcApi {
     BlockCache* block_cache_;
     ethdb::Database* database_;
     ethbackend::BackEnd* backend_;
+    WorkerPool& workers_;
 
     friend class silkworm::rpc::json_rpc::RequestHandler;
 };

silkworm/rpc/commands/erigon_api_test.cpp

@@ -26,7 +26,7 @@ namespace silkworm::rpc::commands {
 //! Utility class to expose handle hooks publicly just for tests
 class ErigonRpcApiForTest : public ErigonRpcApi {
   public:
-    explicit ErigonRpcApiForTest(boost::asio::io_context& io_context) : ErigonRpcApi{io_context} {}
+    explicit ErigonRpcApiForTest(boost::asio::io_context& io_context, WorkerPool& workers) : ErigonRpcApi{io_context, workers} {}
 
     // MSVC doesn't support using access declarations properly, so explicitly forward these public accessors
     Task<void> erigon_get_block_by_timestamp(const nlohmann::json& request, std::string& reply) {
@@ -52,7 +52,7 @@ class ErigonRpcApiForTest : public ErigonRpcApi {
     }
 };
 
-using ErigonRpcApiTest = test_util::JsonApiTestBase<ErigonRpcApiForTest>;
+using ErigonRpcApiTest = test_util::JsonApiWithWorkersTestBase<ErigonRpcApiForTest>;
 
 #ifndef SILKWORM_SANITIZE
 TEST_CASE_METHOD(ErigonRpcApiTest, "ErigonRpcApi::handle_erigon_get_block_by_timestamp", "[rpc][erigon_api]") {

silkworm/rpc/commands/eth_api.cpp

@@ -831,7 +831,7 @@ Task<void> EthereumRpcApi::handle_eth_get_transaction_receipt(const nlohmann::js
             co_await tx->close();  // RAII not (yet) available with coroutines
             co_return;
         }
-        auto receipts = co_await core::get_receipts(*tx, *block_with_hash);
+        auto receipts = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_);
         const auto& transactions = block_with_hash->block.transactions;
         if (receipts.size() != transactions.size()) {
             throw std::invalid_argument{"Unexpected size for receipts in handle_eth_get_transaction_receipt"};
@@ -2038,8 +2038,8 @@ Task<void> EthereumRpcApi::handle_fee_history(const nlohmann::json& request, nlo
         rpc::fee_history::BlockProvider block_provider = [this, &chain_storage](BlockNum block_number) {
             return core::read_block_by_number(*block_cache_, *chain_storage, block_number);
         };
-        rpc::fee_history::ReceiptsProvider receipts_provider = [&tx](const BlockWithHash& block_with_hash) {
-            return core::get_receipts(*tx, block_with_hash);
+        rpc::fee_history::ReceiptsProvider receipts_provider = [&tx, &chain_storage, this](const BlockWithHash& block_with_hash) {
+            return core::get_receipts(*tx, block_with_hash, *chain_storage, this->workers_);
         };
 
         rpc::fee_history::LatestBlockProvider latest_block_provider = [&tx]() {

silkworm/rpc/commands/ots_api.cpp

@@ -116,7 +116,7 @@ Task<void> OtsRpcApi::handle_ots_get_block_details(const nlohmann::json& request
             const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, *total_difficulty,
                                              block_with_hash->block.transactions.size(), block_with_hash->block.ommers,
                                              block_with_hash->block.withdrawals};
-            const auto receipts = co_await core::get_receipts(*tx, *block_with_hash);
+            const auto receipts = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_);
             const auto chain_config = co_await chain_storage->read_chain_config();
             const IssuanceDetails issuance = get_issuance(chain_config, *block_with_hash);
             const intx::uint256 total_fees = get_block_fees(*block_with_hash, receipts);
@@ -165,7 +165,7 @@ Task<void> OtsRpcApi::handle_ots_get_block_details_by_hash(const nlohmann::json&
             const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, *total_difficulty,
                                              block_with_hash->block.transactions.size(), block_with_hash->block.ommers,
                                              block_with_hash->block.withdrawals};
-            const auto receipts = co_await core::get_receipts(*tx, *block_with_hash);
+            const auto receipts = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_);
             const auto chain_config = co_await chain_storage->read_chain_config();
             const IssuanceDetails issuance = get_issuance(chain_config, *block_with_hash);
             const intx::uint256 total_fees = get_block_fees(*block_with_hash, receipts);
@@ -214,7 +214,7 @@ Task<void> OtsRpcApi::handle_ots_get_block_transactions(const nlohmann::json& re
             const auto total_difficulty{co_await chain_storage->read_total_difficulty(block_with_hash->hash, block_number)};
             ensure(total_difficulty.has_value(), [&]() { return "no total difficulty for block: " + std::to_string(block_number); });
             const Block extended_block{block_with_hash, *total_difficulty, false};
-            auto receipts = co_await core::get_receipts(*tx, *block_with_hash);
+            auto receipts = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_);
             auto block_size = extended_block.get_block_size();
             auto transaction_count = block_with_hash->block.transactions.size();
 
@@ -857,7 +857,7 @@ Task<void> OtsRpcApi::trace_block(db::kv::api::Transaction& tx, BlockNum block_n
 
     const auto total_difficulty{co_await chain_storage->read_total_difficulty(block_with_hash->hash, block_number)};
     ensure(total_difficulty.has_value(), [&]() { return "no total difficulty for block: " + std::to_string(block_number); });
-    const auto receipts = co_await core::get_receipts(tx, *block_with_hash);
+    const auto receipts = co_await core::get_receipts(tx, *block_with_hash, *chain_storage, workers_);
     const Block extended_block{block_with_hash, *total_difficulty, false};
 
     trace::TraceCallExecutor executor{*block_cache_, *chain_storage, workers_, tx};

silkworm/rpc/commands/parity_api.cpp

@@ -57,7 +57,7 @@ Task<void> ParityRpcApi::handle_parity_get_block_receipts(const nlohmann::json&
         const auto block_number = co_await core::get_block_number(bnoh, *tx);
         const auto block_with_hash = co_await core::read_block_by_number(*block_cache_, *chain_storage, block_number.first);
         if (block_with_hash) {
-            auto receipts{co_await core::get_receipts(*tx, *block_with_hash)};
+            auto receipts{co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_)};
             SILK_TRACE << "#receipts: " << receipts.size();
 
             const auto block{block_with_hash->block};

silkworm/rpc/commands/parity_api.hpp

@@ -24,6 +24,7 @@
 #include <silkworm/core/common/block_cache.hpp>
 #include <silkworm/infra/concurrency/private_service.hpp>
 #include <silkworm/infra/concurrency/shared_service.hpp>
+#include <silkworm/rpc/common/worker_pool.hpp>
 #include <silkworm/rpc/ethbackend/backend.hpp>
 #include <silkworm/rpc/ethdb/database.hpp>
 #include <silkworm/rpc/json/types.hpp>
@@ -36,10 +37,11 @@ namespace silkworm::rpc::commands {
 
 class ParityRpcApi {
   public:
-    explicit ParityRpcApi(boost::asio::io_context& io_context)
+    explicit ParityRpcApi(boost::asio::io_context& io_context, WorkerPool& workers)
         : block_cache_{must_use_shared_service<BlockCache>(io_context)},
           database_{must_use_private_service<ethdb::Database>(io_context)},
-          backend_{must_use_private_service<ethbackend::BackEnd>(io_context)} {}
+          backend_{must_use_private_service<ethbackend::BackEnd>(io_context)},
+          workers_{workers} {}
     virtual ~ParityRpcApi() = default;
 
     ParityRpcApi(const ParityRpcApi&) = delete;
@@ -54,6 +56,7 @@ class ParityRpcApi {
     BlockCache* block_cache_;
     ethdb::Database* database_;
     ethbackend::BackEnd* backend_;
+    WorkerPool& workers_;
 
     friend class silkworm::rpc::json_rpc::RequestHandler;
 };

silkworm/rpc/commands/rpc_api.hpp

@@ -60,8 +60,8 @@ class RpcApi : protected EthereumRpcApi,
           AdminRpcApi{io_context},
           Web3RpcApi{io_context},
           DebugRpcApi{io_context, workers},
-          ParityRpcApi{io_context},
-          ErigonRpcApi{io_context},
+          ParityRpcApi{io_context, workers},
+          ErigonRpcApi{io_context, workers},
           TraceRpcApi{io_context, workers},
           EngineRpcApi(io_context, std::move(build_info)),
           TxPoolRpcApi(io_context),

silkworm/rpc/core/evm_executor.cpp

@@ -321,6 +321,110 @@ ExecutionResult EVMExecutor::call(
     return exec_result;
 }
 
+ExecutionResult EVMExecutor::call_with_receipt(
+    const silkworm::Block& block,
+    const silkworm::Transaction& txn,
+    Receipt& receipt,
+    const Tracers& tracers,
+    bool refund,
+    bool gas_bailout) {
+    SILK_DEBUG << "EVMExecutor::call: blockNumber: " << block.header.number << " gas_limit: " << txn.gas_limit << " refund: " << refund
+               << " gas_bailout: " << gas_bailout << " transaction: " << rpc::Transaction{txn};
+
+    auto& svc = use_service<AnalysisCacheService>(workers_);
+    EVM evm{block, ibs_state_, config_, gas_bailout};
+    evm.analysis_cache = svc.get_analysis_cache();
+    evm.state_pool = svc.get_object_pool();
+    evm.beneficiary = rule_set_->get_beneficiary(block.header);
+    evm.transfer = rule_set_->transfer_func();
+
+    for (auto& tracer : tracers) {
+        evm.add_tracer(*tracer);
+    }
+
+    if (!txn.sender()) {
+        return {std::nullopt, txn.gas_limit, Bytes{}, "malformed transaction: cannot recover sender"};
+    }
+    ibs_state_.access_account(*txn.sender());
+
+    const evmc_revision rev{evm.revision()};
+    const intx::uint256 base_fee_per_gas{evm.block().header.base_fee_per_gas.value_or(0)};
+    const intx::uint128 g0{protocol::intrinsic_gas(txn, rev)};
+    SILKWORM_ASSERT(g0 <= UINT64_MAX);  // true due to the precondition (transaction must be valid)
+
+    if (const auto pre_check_result = pre_check(evm, txn, base_fee_per_gas, g0)) {
+        Bytes data{};
+        return {std::nullopt, txn.gas_limit, data, pre_check_result->pre_check_error, pre_check_result->pre_check_error_code};
+    }
+
+    if (const auto result = evm.deduct_entry_fees(txn); result.status != EVMC_SUCCESS) {
+        return {std::nullopt, txn.gas_limit, {}, result.error_message, PreCheckErrorCode::kInsufficientFunds};
+    }
+    if (txn.to.has_value()) {
+        ibs_state_.access_account(*txn.to);
+        // EVM itself increments the nonce for contract creation
+        ibs_state_.set_nonce(*txn.sender(), ibs_state_.get_nonce(*txn.sender()) + 1);
+    }
+    for (const AccessListEntry& ae : txn.access_list) {
+        ibs_state_.access_account(ae.account);
+        for (const evmc::bytes32& key : ae.storage_keys) {
+            ibs_state_.access_storage(ae.account, key);
+        }
+    }
+
+    CallResult result;
+    try {
+        SILK_DEBUG << "EVMExecutor::call execute on EVM txn: " << &txn << " g0: " << static_cast<uint64_t>(g0) << " start";
+        result = evm.execute(txn, txn.gas_limit - static_cast<uint64_t>(g0));
+        SILK_DEBUG << "EVMExecutor::call execute on EVM txn: " << &txn << " gas_left: " << result.gas_left << " end";
+    } catch (const std::exception& e) {
+        SILK_ERROR << "exception: evm_execute: " << e.what() << "\n";
+        std::string error_msg = "evm.execute: ";
+        error_msg.append(e.what());
+        return {std::nullopt, txn.gas_limit, /* data */ {}, error_msg};
+    } catch (...) {
+        SILK_ERROR << "exception: evm_execute: unexpected exception\n";
+        return {std::nullopt, txn.gas_limit, /* data */ {}, "evm.execute: unknown exception"};
+    }
+
+    uint64_t gas_left{result.gas_left};
+    uint64_t gas_used{txn.gas_limit - result.gas_left};
+
+    if (refund && !gas_bailout) {
+        gas_used = txn.gas_limit - refund_gas(evm, txn, result.gas_left, result.gas_refund);
+        gas_left = txn.gas_limit - gas_used;
+    }
+
+    // Reward the fee recipient
+    const intx::uint256 priority_fee_per_gas{txn.max_fee_per_gas >= base_fee_per_gas ? txn.priority_fee_per_gas(base_fee_per_gas)
+                                                                                     : txn.max_priority_fee_per_gas};
+    SILK_DEBUG << "EVMExecutor::call evm.beneficiary: " << evm.beneficiary << " balance: " << priority_fee_per_gas * gas_used;
+    ibs_state_.add_to_balance(evm.beneficiary, priority_fee_per_gas * gas_used);
+
+    for (auto& tracer : evm.tracers()) {
+        tracer.get().on_reward_granted(result, ibs_state_);
+    }
+    ibs_state_.finalize_transaction(rev);
+
+    receipt.success = result.status == EVMC_SUCCESS;
+    receipt.bloom = logs_bloom(ibs_state_.logs());
+    receipt.gas_used = gas_used;
+    receipt.type = static_cast<uint8_t>(txn.type);
+    for (size_t j{0}; j < ibs_state_.logs().size(); j++) {
+        Log rpc_log;
+        rpc_log.address = ibs_state_.logs()[j].address;
+        rpc_log.data = ibs_state_.logs()[j].data;
+        rpc_log.topics = ibs_state_.logs()[j].topics;
+        receipt.logs.push_back(rpc_log);
+    }
+
+    ExecutionResult exec_result{result.status, gas_left, result.data};
+
+    SILK_DEBUG << "EVMExecutor::call call_result: " << exec_result.error_message() << " #data: " << exec_result.data.size() << " end";
+
+    return exec_result;
+}
+
 Task<ExecutionResult> EVMExecutor::call(
     const silkworm::ChainConfig& config,
     const ChainStorage& chain_storage,

silkworm/rpc/core/evm_executor.hpp

@@ -33,10 +33,12 @@
 #include <silkworm/core/protocol/rule_set.hpp>
 #include <silkworm/core/state/state.hpp>
 #include <silkworm/core/types/block.hpp>
+#include <silkworm/core/types/receipt.hpp>
 #include <silkworm/core/types/transaction.hpp>
 #include <silkworm/db/chain/chain_storage.hpp>
 #include <silkworm/db/state/state_reader.hpp>
 #include <silkworm/rpc/common/worker_pool.hpp>
+#include <silkworm/rpc/types/receipt.hpp>
 
 namespace silkworm::rpc {
 
@@ -125,6 +127,14 @@ class EVMExecutor {
         bool refund = true,
         bool gas_bailout = false);
 
+    ExecutionResult call_with_receipt(
+        const silkworm::Block& block,
+        const silkworm::Transaction& txn,
+        Receipt& receipt,
+        const Tracers& tracers = {},
+        bool refund = true,
+        bool gas_bailout = false);
+
     void reset();
 
     void call_first_n(const silkworm::Block& block, uint64_t n, const Tracers& tracers = {}, bool refund = true, bool gas_bailout = false);