feat(clis,specs): Tests marked as slow have greater t8n server `tim…

…eout` (#1037)

* feat(clis): Add parameter `slow_request` to t8n evaluation

* feat(specs): Marked slow tests pass `slow_request` parameter

* fix(clis): besu

* docs: Changelog

docs/CHANGELOG.md

@@ -63,6 +63,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 - ✨ Add the `eest make env` command that generates a default env file (`env.yaml`)([#996](https://github.com/ethereum/execution-spec-tests/pull/996)).
 - ✨ Generate Transaction Test type ([#933](https://github.com/ethereum/execution-spec-tests/pull/933)).
 - ✨ Add a default location for evm logs (`--evm-dump-dir`) when filling tests ([#999](https://github.com/ethereum/execution-spec-tests/pull/999)).
+- ✨ Slow tests now have greater timeout when making a request to the T8N server ([#1037](https://github.com/ethereum/execution-spec-tests/pull/1037)).
 
 ### 🔧 EVM Tools
 

src/ethereum_clis/clis/besu.py

@@ -109,6 +109,7 @@ def evaluate(
         eips: Optional[List[int]] = None,
         debug_output_path: str = "",
         state_test: bool = False,
+        slow_request: bool = False,
     ) -> TransitionToolOutput:
         """
         Executes `evm t8n` with the specified arguments.

src/ethereum_clis/transition_tool.py

@@ -30,6 +30,9 @@
 
 model_dump_config: Mapping = {"by_alias": True, "exclude_none": True}
 
+NORMAL_SERVER_TIMEOUT = 20
+SLOW_REQUEST_TIMEOUT = 60
+
 
 class TransitionTool(EthereumCLI, FixtureVerifier):
     """
@@ -276,6 +279,7 @@ def _evaluate_filesystem(
     def _server_post(
         self,
         data: Dict[str, Any],
+        timeout: int,
         url_args: Dict[str, List[str] | str] = {},
         retries: int = 5,
     ) -> Response:
@@ -288,7 +292,7 @@ def _server_post(
                 response = Session().post(
                     f"{self.server_url}?{urlencode(url_args, doseq=True)}",
                     json=data,
-                    timeout=20,
+                    timeout=timeout,
                 )
                 break
             except ConnectionError as e:
@@ -316,6 +320,7 @@ def _evaluate_server(
         *,
         t8n_data: TransitionToolData,
         debug_output_path: str = "",
+        timeout: int,
     ) -> TransitionToolOutput:
         """
         Executes the transition tool sending inputs and outputs via a server.
@@ -349,7 +354,9 @@ def _evaluate_server(
                 },
             )
 
-        response = self._server_post(data=post_data, url_args=self._generate_post_args(t8n_data))
+        response = self._server_post(
+            data=post_data, url_args=self._generate_post_args(t8n_data), timeout=timeout
+        )
         output: TransitionToolOutput = TransitionToolOutput.model_validate(response.json())
 
         if debug_output_path:
@@ -496,6 +503,7 @@ def evaluate(
         eips: Optional[List[int]] = None,
         debug_output_path: str = "",
         state_test: bool = False,
+        slow_request: bool = False,
     ) -> TransitionToolOutput:
         """
         Executes the relevant evaluate method as required by the `t8n` tool.
@@ -524,7 +532,11 @@ def evaluate(
         if self.t8n_use_server:
             if not self.process:
                 self.start_server()
-            return self._evaluate_server(t8n_data=t8n_data, debug_output_path=debug_output_path)
+            return self._evaluate_server(
+                t8n_data=t8n_data,
+                debug_output_path=debug_output_path,
+                timeout=SLOW_REQUEST_TIMEOUT if slow_request else NORMAL_SERVER_TIMEOUT,
+            )
 
         if self.t8n_use_stream:
             return self._evaluate_stream(t8n_data=t8n_data, debug_output_path=debug_output_path)

src/ethereum_test_specs/blockchain.py

@@ -45,7 +45,7 @@
 
 from .base import BaseTest, verify_result
 from .debugging import print_traces
-from .helpers import verify_transactions
+from .helpers import is_slow_test, verify_transactions
 
 
 def environment_from_parent_header(parent: "FixtureHeader") -> "Environment":
@@ -384,6 +384,7 @@ def generate_block_data(
         previous_env: Environment,
         previous_alloc: Alloc,
         eips: Optional[List[int]] = None,
+        slow: bool = False,
     ) -> Tuple[FixtureHeader, List[Transaction], List[Bytes] | None, Alloc, Environment]:
         """
         Generate common block data for both make_fixture and make_hive_fixture.
@@ -420,6 +421,7 @@ def generate_block_data(
             reward=fork.get_reward(env.number, env.timestamp),
             eips=eips,
             debug_output_path=self.get_next_transition_tool_output_path(),
+            slow_request=slow,
         )
 
         try:
@@ -528,6 +530,7 @@ def make_fixture(
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
+        slow: bool = False,
     ) -> Fixture:
         """
         Create a fixture from the blockchain test definition.
@@ -552,6 +555,7 @@ def make_fixture(
                     previous_env=env,
                     previous_alloc=alloc,
                     eips=eips,
+                    slow=slow,
                 )
                 fixture_block = FixtureBlockBase(
                     header=header,
@@ -610,6 +614,7 @@ def make_hive_fixture(
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
+        slow: bool = False,
     ) -> EngineFixture:
         """
         Create a hive fixture from the blocktest definition.
@@ -623,7 +628,13 @@ def make_hive_fixture(
 
         for block in self.blocks:
             header, txs, requests, new_alloc, new_env = self.generate_block_data(
-                t8n=t8n, fork=fork, block=block, previous_env=env, previous_alloc=alloc, eips=eips
+                t8n=t8n,
+                fork=fork,
+                block=block,
+                previous_env=env,
+                previous_alloc=alloc,
+                eips=eips,
+                slow=slow,
             )
             if block.rlp is None:
                 fixture_payloads.append(
@@ -701,9 +712,9 @@ def generate(
         """
         t8n.reset_traces()
         if fixture_format == BlockchainEngineFixture:
-            return self.make_hive_fixture(t8n, fork, eips)
+            return self.make_hive_fixture(t8n, fork, eips, slow=is_slow_test(request))
         elif fixture_format == BlockchainFixture:
-            return self.make_fixture(t8n, fork, eips)
+            return self.make_fixture(t8n, fork, eips, slow=is_slow_test(request))
 
         raise Exception(f"Unknown fixture format: {fixture_format}")
 

src/ethereum_test_specs/helpers.py

@@ -1,10 +1,11 @@
 """
 Helper functions
 """
-
 from dataclasses import dataclass
 from typing import Dict, List
 
+import pytest
+
 from ethereum_clis import Result
 from ethereum_test_exceptions import ExceptionBase, ExceptionMapper, UndefinedException
 from ethereum_test_types import Transaction
@@ -150,3 +151,12 @@ def verify_transactions(
         verify_transaction_exception(exception_mapper=exception_mapper, info=info)
 
     return list(rejected_txs.keys())
+
+
+def is_slow_test(request: pytest.FixtureRequest) -> bool:
+    """
+    Check if the test is slow
+    """
+    if hasattr(request, "node"):
+        return request.node.get_closest_marker("slow") is not None
+    return False

src/ethereum_test_specs/state.py

@@ -30,7 +30,7 @@
 from .base import BaseTest
 from .blockchain import Block, BlockchainTest, Header
 from .debugging import print_traces
-from .helpers import verify_transactions
+from .helpers import is_slow_test, verify_transactions
 
 TARGET_BLOB_GAS_PER_BLOCK = 393216
 
@@ -124,6 +124,7 @@ def make_state_test_fixture(
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
+        slow: bool = False,
     ) -> Fixture:
         """
         Create a fixture from the state test definition.
@@ -151,6 +152,7 @@ def make_state_test_fixture(
             eips=eips,
             debug_output_path=self.get_next_transition_tool_output_path(),
             state_test=True,
+            slow_request=slow,
         )
 
         try:
@@ -199,7 +201,7 @@ def generate(
                 request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
             )
         elif fixture_format == StateFixture:
-            return self.make_state_test_fixture(t8n, fork, eips)
+            return self.make_state_test_fixture(t8n, fork, eips, slow=is_slow_test(request))
 
         raise Exception(f"Unknown fixture format: {fixture_format}")