diff --git a/bittensor/core/extrinsics/registration.py b/bittensor/core/extrinsics/registration.py index fdb67619e6..e9836938d8 100644 --- a/bittensor/core/extrinsics/registration.py +++ b/bittensor/core/extrinsics/registration.py @@ -13,11 +13,8 @@ from bittensor.utils import format_error_message, unlock_key from bittensor.utils.btlogging import logging from bittensor.utils.networking import ensure_connected -from bittensor.utils.registration import ( - create_pow, - torch, - log_no_torch_error, -) +from bittensor.utils.registration import create_pow, torch, log_no_torch_error +from bittensor.core.extrinsics.utils import submit_extrinsic # For annotation and lazy import purposes if TYPE_CHECKING: @@ -68,8 +65,9 @@ def _do_pow_register( }, ) extrinsic = self.substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey) - response = self.substrate.submit_extrinsic( - extrinsic, + response = submit_extrinsic( + substrate=self.substrate, + extrinsic=extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, ) @@ -298,7 +296,8 @@ def _do_burned_register( extrinsic = self.substrate.create_signed_extrinsic( call=call, keypair=wallet.coldkey ) - response = self.substrate.submit_extrinsic( + response = submit_extrinsic( + self.substrate, extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/core/extrinsics/root.py b/bittensor/core/extrinsics/root.py index cbef1c4df7..e4fec236fc 100644 --- a/bittensor/core/extrinsics/root.py +++ b/bittensor/core/extrinsics/root.py @@ -5,6 +5,7 @@ from numpy.typing import NDArray from bittensor.core.settings import version_as_int +from bittensor.core.extrinsics.utils import submit_extrinsic from bittensor.utils import format_error_message, weight_utils, unlock_key from bittensor.utils.btlogging import logging from bittensor.utils.networking import ensure_connected @@ -31,7 +32,8 @@ def _do_root_register( extrinsic = self.substrate.create_signed_extrinsic( call=call, keypair=wallet.coldkey ) - response = self.substrate.submit_extrinsic( + response = submit_extrinsic( + self.substrate, extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, @@ -156,7 +158,8 @@ def _do_set_root_weights( keypair=wallet.coldkey, era={"period": period}, ) - response = self.substrate.submit_extrinsic( + response = submit_extrinsic( + self.substrate, extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/core/extrinsics/serving.py b/bittensor/core/extrinsics/serving.py index 864726b97c..1c26b87e76 100644 --- a/bittensor/core/extrinsics/serving.py +++ b/bittensor/core/extrinsics/serving.py @@ -288,7 +288,8 @@ def publish_metadata( ) extrinsic = substrate.create_signed_extrinsic(call=call, keypair=wallet.hotkey) - response = substrate.submit_extrinsic( + response = submit_extrinsic( + substrate, extrinsic, wait_for_inclusion=wait_for_inclusion, wait_for_finalization=wait_for_finalization, diff --git a/bittensor/core/extrinsics/utils.py b/bittensor/core/extrinsics/utils.py index 6c896372b6..6af14b25fb 100644 --- a/bittensor/core/extrinsics/utils.py +++ b/bittensor/core/extrinsics/utils.py @@ -1,21 +1,33 @@ """Module with helper functions for extrinsics.""" +import threading from typing import TYPE_CHECKING -from substrateinterface.exceptions import SubstrateRequestException + +from substrateinterface.exceptions import SubstrateRequestException, ExtrinsicNotFound + from bittensor.utils.btlogging import logging from bittensor.utils import format_error_message if TYPE_CHECKING: - from substrateinterface import SubstrateInterface + from substrateinterface import SubstrateInterface, ExtrinsicReceipt from scalecodec.types import GenericExtrinsic +class _ThreadingTimeoutException(Exception): + """ + Exception raised for timeout. Different from TimeoutException because this also triggers + a websocket failure. This exception should only be used with `threading` timer.. + """ + + pass + + def submit_extrinsic( substrate: "SubstrateInterface", extrinsic: "GenericExtrinsic", wait_for_inclusion: bool, wait_for_finalization: bool, -): +) -> "ExtrinsicReceipt": """ Submits an extrinsic to the substrate blockchain and handles potential exceptions. @@ -35,7 +47,22 @@ def submit_extrinsic( Raises: SubstrateRequestException: If the submission of the extrinsic fails, the error is logged and re-raised. """ + extrinsic_hash = extrinsic.extrinsic_hash + starting_block = substrate.get_block() + + def _handler(): + """ + Timeout handler for threading. Will raise a TimeoutError if timeout is exceeded. + """ + logging.error("Timed out waiting for extrinsic submission.") + raise _ThreadingTimeoutException + + # sets a timeout timer for the next call to 200 seconds + # will raise a _ThreadingTimeoutException if it reaches this point + timer = threading.Timer(200, _handler) + try: + timer.start() response = substrate.submit_extrinsic( extrinsic, wait_for_inclusion=wait_for_inclusion, @@ -46,4 +73,32 @@ def submit_extrinsic( # Re-rise the exception for retrying of the extrinsic call. If we remove the retry logic, the raise will need # to be removed. raise + + except _ThreadingTimeoutException: + after_timeout_block = substrate.get_block() + + response = None + for block_num in range( + starting_block["header"]["number"], + after_timeout_block["header"]["number"] + 1, + ): + block_hash = substrate.get_block_hash(block_num) + try: + response = substrate.retrieve_extrinsic_by_hash( + block_hash, f"0x{extrinsic_hash.hex()}" + ) + except ExtrinsicNotFound: + continue + if response: + break + finally: + timer.cancel() + + if response is None: + logging.error( + f"Extrinsic '0x{extrinsic_hash.hex()}' not submitted. " + f"Initially attempted to submit at block {starting_block['header']['number']}." + ) + raise SubstrateRequestException + return response diff --git a/tests/e2e_tests/utils/e2e_test_utils.py b/tests/e2e_tests/utils/e2e_test_utils.py index 2568908bea..7aac17a016 100644 --- a/tests/e2e_tests/utils/e2e_test_utils.py +++ b/tests/e2e_tests/utils/e2e_test_utils.py @@ -50,6 +50,7 @@ def clone_or_update_templates(specific_commit=None): os.chdir(install_dir) for repo, git_link in repo_mapping.items(): + print(os.path.abspath(repo)) if not os.path.exists(repo): print(f"\033[94mCloning {repo}...\033[0m") subprocess.run(["git", "clone", git_link, repo], check=True)