Use Local Payload if More Profitable than Builder (#3934)

* Use Local Payload if More Profitable than Builder

* Rename clone -> clone_from_ref

* Minimize Clones of GetPayloadResponse

* Cleanup & Fix Tests

* Added Tests for Payload Choice by Profit

* Fix Outdated Comments

beacon_node/beacon_chain/src/test_utils.rs

@@ -426,6 +426,7 @@ where
             DEFAULT_TERMINAL_BLOCK,
             shanghai_time,
             eip4844_time,
+            None,
             Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
             spec,
             None,
@@ -435,7 +436,11 @@ where
         self
     }
 
-    pub fn mock_execution_layer_with_builder(mut self, beacon_url: SensitiveUrl) -> Self {
+    pub fn mock_execution_layer_with_builder(
+        mut self,
+        beacon_url: SensitiveUrl,
+        builder_threshold: Option<u128>,
+    ) -> Self {
         // Get a random unused port
         let port = unused_port::unused_tcp_port().unwrap();
         let builder_url = SensitiveUrl::parse(format!("http://127.0.0.1:{port}").as_str()).unwrap();
@@ -452,6 +457,7 @@ where
             DEFAULT_TERMINAL_BLOCK,
             shanghai_time,
             eip4844_time,
+            builder_threshold,
             Some(JwtKey::from_slice(&DEFAULT_JWT_SECRET).unwrap()),
             spec.clone(),
             Some(builder_url.clone()),

beacon_node/execution_layer/src/engine_api.rs

@@ -14,8 +14,8 @@ use std::convert::TryFrom;
 use strum::IntoStaticStr;
 use superstruct::superstruct;
 pub use types::{
-    Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, FixedVector,
-    ForkName, Hash256, Uint256, VariableList, Withdrawal,
+    Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader,
+    ExecutionPayloadRef, FixedVector, ForkName, Hash256, Uint256, VariableList, Withdrawal,
 };
 use types::{ExecutionPayloadCapella, ExecutionPayloadEip4844, ExecutionPayloadMerge};
 
@@ -322,6 +322,8 @@ pub struct ProposeBlindedBlockResponse {
 #[superstruct(
     variants(Merge, Capella, Eip4844),
     variant_attributes(derive(Clone, Debug, PartialEq),),
+    map_into(ExecutionPayload),
+    map_ref_into(ExecutionPayloadRef),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
 )]
@@ -336,22 +338,47 @@ pub struct GetPayloadResponse<T: EthSpec> {
     pub block_value: Uint256,
 }
 
-impl<T: EthSpec> GetPayloadResponse<T> {
-    pub fn execution_payload(self) -> ExecutionPayload<T> {
-        match self {
-            GetPayloadResponse::Merge(response) => {
-                ExecutionPayload::Merge(response.execution_payload)
-            }
-            GetPayloadResponse::Capella(response) => {
-                ExecutionPayload::Capella(response.execution_payload)
-            }
-            GetPayloadResponse::Eip4844(response) => {
-                ExecutionPayload::Eip4844(response.execution_payload)
-            }
+impl<'a, T: EthSpec> From<GetPayloadResponseRef<'a, T>> for ExecutionPayloadRef<'a, T> {
+    fn from(response: GetPayloadResponseRef<'a, T>) -> Self {
+        map_get_payload_response_ref_into_execution_payload_ref!(&'a _, response, |inner, cons| {
+            cons(&inner.execution_payload)
+        })
+    }
+}
+
+impl<T: EthSpec> From<GetPayloadResponse<T>> for ExecutionPayload<T> {
+    fn from(response: GetPayloadResponse<T>) -> Self {
+        map_get_payload_response_into_execution_payload!(response, |inner, cons| {
+            cons(inner.execution_payload)
+        })
+    }
+}
+
+impl<T: EthSpec> From<GetPayloadResponse<T>> for (ExecutionPayload<T>, Uint256) {
+    fn from(response: GetPayloadResponse<T>) -> Self {
+        match response {
+            GetPayloadResponse::Merge(inner) => (
+                ExecutionPayload::Merge(inner.execution_payload),
+                inner.block_value,
+            ),
+            GetPayloadResponse::Capella(inner) => (
+                ExecutionPayload::Capella(inner.execution_payload),
+                inner.block_value,
+            ),
+            GetPayloadResponse::Eip4844(inner) => (
+                ExecutionPayload::Eip4844(inner.execution_payload),
+                inner.block_value,
+            ),
         }
     }
 }
 
+impl<T: EthSpec> GetPayloadResponse<T> {
+    pub fn execution_payload_ref(&self) -> ExecutionPayloadRef<T> {
+        self.to_ref().into()
+    }
+}
+
 #[derive(Clone, Copy, Debug)]
 pub struct EngineCapabilities {
     pub new_payload_v1: bool,

beacon_node/execution_layer/src/engine_api/http.rs

@@ -804,7 +804,7 @@ impl HttpJsonRpc {
     pub async fn get_payload_v1<T: EthSpec>(
         &self,
         payload_id: PayloadId,
-    ) -> Result<ExecutionPayload<T>, Error> {
+    ) -> Result<GetPayloadResponse<T>, Error> {
         let params = json!([JsonPayloadIdRequest::from(payload_id)]);
 
         let payload_v1: JsonExecutionPayloadV1<T> = self
@@ -815,7 +815,11 @@ impl HttpJsonRpc {
             )
             .await?;
 
-        Ok(JsonExecutionPayload::V1(payload_v1).into())
+        Ok(GetPayloadResponse::Merge(GetPayloadResponseMerge {
+            execution_payload: payload_v1.into(),
+            // Have to guess zero here as we don't know the value
+            block_value: Uint256::zero(),
+        }))
     }
 
     pub async fn get_payload_v2<T: EthSpec>(
@@ -1015,16 +1019,10 @@ impl HttpJsonRpc {
         &self,
         fork_name: ForkName,
         payload_id: PayloadId,
-    ) -> Result<ExecutionPayload<T>, Error> {
+    ) -> Result<GetPayloadResponse<T>, Error> {
         let engine_capabilities = self.get_engine_capabilities(None).await?;
         if engine_capabilities.get_payload_v2 {
-            // TODO: modify this method to return GetPayloadResponse instead
-            //       of throwing away the `block_value` and returning only the
-            //       ExecutionPayload
-            Ok(self
-                .get_payload_v2(fork_name, payload_id)
-                .await?
-                .execution_payload())
+            self.get_payload_v2(fork_name, payload_id).await
         } else if engine_capabilities.new_payload_v1 {
             self.get_payload_v1(payload_id).await
         } else {
@@ -1675,10 +1673,11 @@ mod test {
                     }
                 })],
                 |client| async move {
-                    let payload = client
+                    let payload: ExecutionPayload<_> = client
                         .get_payload_v1::<MainnetEthSpec>(str_to_payload_id("0xa247243752eb10b4"))
                         .await
-                        .unwrap();
+                        .unwrap()
+                        .into();
 
                     let expected = ExecutionPayload::Merge(ExecutionPayloadMerge {
                             parent_hash: ExecutionBlockHash::from_str("0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a").unwrap(),