From c67c74096a023014912faa4bff714265481522b5 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Thu, 9 May 2024 23:08:26 +0200 Subject: [PATCH 1/6] add partial to query state enum --- dune_client/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dune_client/models.py b/dune_client/models.py index 270288f..0c62433 100644 --- a/dune_client/models.py +++ b/dune_client/models.py @@ -47,6 +47,7 @@ class ExecutionState(Enum): COMPLETED = "QUERY_STATE_COMPLETED" EXECUTING = "QUERY_STATE_EXECUTING" + PARTIAL = "QUERY_STATE_PARTIAL" PENDING = "QUERY_STATE_PENDING" CANCELLED = "QUERY_STATE_CANCELLED" FAILED = "QUERY_STATE_FAILED" From 462326ffba507113d0eec8a4f76a2f26e37bd472 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Fri, 10 May 2024 19:36:44 +0200 Subject: [PATCH 2/6] add allow_partial_results with default = True --- dune_client/api/base.py | 2 ++ dune_client/api/execution.py | 2 ++ dune_client/api/extensions.py | 7 ++++++- dune_client/models.py | 4 ++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dune_client/api/base.py b/dune_client/api/base.py index 14c40bb..dae2d64 100644 --- a/dune_client/api/base.py +++ b/dune_client/api/base.py @@ -96,6 +96,7 @@ def _build_parameters( sort_by: Optional[List[str]] = None, limit: Optional[int] = None, offset: Optional[int] = None, + allow_partial_results: bool = True, ) -> Dict[str, Union[str, int]]: """ Utility function that builds a dictionary of parameters to be used @@ -111,6 +112,7 @@ def _build_parameters( ), "sampling cannot be combined with filters or pagination" params = params or {} + params["allow_partial_results"] = allow_partial_results if columns is not None and len(columns) > 0: params["columns"] = ",".join(columns) if sample_count is not None: diff --git a/dune_client/api/execution.py b/dune_client/api/execution.py index ba7b244..a8cb3bb 100644 --- a/dune_client/api/execution.py +++ b/dune_client/api/execution.py @@ -80,6 +80,7 @@ def get_execution_results( sample_count: Optional[int] = None, filters: Optional[str] = None, sort_by: Optional[List[str]] = None, + allow_partial_results: bool = True, ) -> ResultsResponse: """GET results from Dune API for `job_id` (aka `execution_id`)""" params = self._build_parameters( @@ -89,6 +90,7 @@ def get_execution_results( sort_by=sort_by, limit=limit, offset=offset, + allow_partial_results=allow_partial_results, ) route = f"/execution/{job_id}/results" diff --git a/dune_client/api/extensions.py b/dune_client/api/extensions.py index 3aba004..e0eec49 100644 --- a/dune_client/api/extensions.py +++ b/dune_client/api/extensions.py @@ -54,6 +54,7 @@ def run_query( sample_count: Optional[int] = None, filters: Optional[str] = None, sort_by: Optional[List[str]] = None, + allow_partial_results: bool = True, ) -> ResultsResponse: """ Executes a Dune `query`, waits until execution completes, @@ -83,6 +84,7 @@ def run_query( filters=filters, sort_by=sort_by, limit=limit, + allow_partial_results=allow_partial_results, ), ) @@ -419,7 +421,10 @@ def _refresh( if status.state == ExecutionState.FAILED: self.logger.error(status) raise QueryFailed(f"Error data: {status.error}") - + if status.state == ExecutionState.PARTIAL: + self.logger.warning( + f"execution {job_id} resulted in a partial result set (i.e. results too large)." + ) return job_id def _fetch_entire_result( diff --git a/dune_client/models.py b/dune_client/models.py index 0c62433..15c96ff 100644 --- a/dune_client/models.py +++ b/dune_client/models.py @@ -47,7 +47,7 @@ class ExecutionState(Enum): COMPLETED = "QUERY_STATE_COMPLETED" EXECUTING = "QUERY_STATE_EXECUTING" - PARTIAL = "QUERY_STATE_PARTIAL" + PARTIAL = "QUERY_STATE_COMPLETED_PARTIAL" PENDING = "QUERY_STATE_PENDING" CANCELLED = "QUERY_STATE_CANCELLED" FAILED = "QUERY_STATE_FAILED" @@ -58,7 +58,7 @@ def terminal_states(cls) -> set[ExecutionState]: """ Returns the terminal states (i.e. when a query execution is no longer executing """ - return {cls.COMPLETED, cls.CANCELLED, cls.FAILED, cls.EXPIRED} + return {cls.COMPLETED, cls.CANCELLED, cls.FAILED, cls.EXPIRED, cls.PARTIAL} def is_complete(self) -> bool: """Returns True is state is completed, otherwise False.""" From ab6d42d7119700767054a21f94a33180d4c1e571 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Fri, 10 May 2024 19:40:49 +0200 Subject: [PATCH 3/6] move the warning log to get_results --- dune_client/api/execution.py | 12 ++++++++---- dune_client/api/extensions.py | 4 ---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dune_client/api/execution.py b/dune_client/api/execution.py index a8cb3bb..3d94e63 100644 --- a/dune_client/api/execution.py +++ b/dune_client/api/execution.py @@ -22,6 +22,7 @@ ResultsResponse, ExecutionResultCSV, DuneError, + ExecutionState, ) from dune_client.query import QueryBase @@ -128,9 +129,7 @@ def get_execution_results_csv( return self._get_execution_results_csv_by_url(url=url, params=params) def _get_execution_results_by_url( - self, - url: str, - params: Optional[Dict[str, Any]] = None, + self, url: str, params: Optional[Dict[str, Any]] = None, job_id=None ) -> ResultsResponse: """ GET results from Dune API with a given URL. This is particularly useful for pagination. @@ -139,7 +138,12 @@ def _get_execution_results_by_url( response_json = self._get(url=url, params=params) try: - return ResultsResponse.from_dict(response_json) + result = ResultsResponse.from_dict(response_json) + if result.state == ExecutionState.PARTIAL: + self.logger.warning( + f"execution {job_id} resulted in a partial result set (i.e. results too large)." + ) + return result except KeyError as err: raise DuneError(response_json, "ResultsResponse", err) from err diff --git a/dune_client/api/extensions.py b/dune_client/api/extensions.py index e0eec49..16dab45 100644 --- a/dune_client/api/extensions.py +++ b/dune_client/api/extensions.py @@ -421,10 +421,6 @@ def _refresh( if status.state == ExecutionState.FAILED: self.logger.error(status) raise QueryFailed(f"Error data: {status.error}") - if status.state == ExecutionState.PARTIAL: - self.logger.warning( - f"execution {job_id} resulted in a partial result set (i.e. results too large)." - ) return job_id def _fetch_entire_result( From 8a60a385ee6234ed104b174b343c1bf910415319 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Sat, 11 May 2024 11:03:58 +0200 Subject: [PATCH 4/6] revert accidental change --- dune_client/api/execution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dune_client/api/execution.py b/dune_client/api/execution.py index 3d94e63..a37eb8d 100644 --- a/dune_client/api/execution.py +++ b/dune_client/api/execution.py @@ -129,7 +129,7 @@ def get_execution_results_csv( return self._get_execution_results_csv_by_url(url=url, params=params) def _get_execution_results_by_url( - self, url: str, params: Optional[Dict[str, Any]] = None, job_id=None + self, url: str, params: Optional[Dict[str, Any]] = None ) -> ResultsResponse: """ GET results from Dune API with a given URL. This is particularly useful for pagination. @@ -141,7 +141,7 @@ def _get_execution_results_by_url( result = ResultsResponse.from_dict(response_json) if result.state == ExecutionState.PARTIAL: self.logger.warning( - f"execution {job_id} resulted in a partial result set (i.e. results too large)." + f"execution {result.execution_id} resulted in a partial result set (i.e. results too large)." ) return result except KeyError as err: From 065d19b4bf86f9f680ba0da8471dce8aa067b976 Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Mon, 26 Aug 2024 23:44:01 +0200 Subject: [PATCH 5/6] fix formatting --- dune_client/api/execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dune_client/api/execution.py b/dune_client/api/execution.py index a37eb8d..560289c 100644 --- a/dune_client/api/execution.py +++ b/dune_client/api/execution.py @@ -141,7 +141,8 @@ def _get_execution_results_by_url( result = ResultsResponse.from_dict(response_json) if result.state == ExecutionState.PARTIAL: self.logger.warning( - f"execution {result.execution_id} resulted in a partial result set (i.e. results too large)." + f"execution {result.execution_id} resulted in a partial " + f"result set (i.e. results too large)." ) return result except KeyError as err: From f40fdaba50e08684a83f45e91fea3207ba6d2c4e Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Tue, 27 Aug 2024 00:01:02 +0200 Subject: [PATCH 6/6] fix parameters (boolean as string) --- dune_client/api/base.py | 2 +- dune_client/api/execution.py | 2 +- dune_client/api/extensions.py | 4 +++- dune_client/client_async.py | 1 + dune_client/models.py | 1 + 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dune_client/api/base.py b/dune_client/api/base.py index dae2d64..79abddf 100644 --- a/dune_client/api/base.py +++ b/dune_client/api/base.py @@ -96,7 +96,7 @@ def _build_parameters( sort_by: Optional[List[str]] = None, limit: Optional[int] = None, offset: Optional[int] = None, - allow_partial_results: bool = True, + allow_partial_results: str = "true", ) -> Dict[str, Union[str, int]]: """ Utility function that builds a dictionary of parameters to be used diff --git a/dune_client/api/execution.py b/dune_client/api/execution.py index 560289c..e15e6bd 100644 --- a/dune_client/api/execution.py +++ b/dune_client/api/execution.py @@ -81,7 +81,7 @@ def get_execution_results( sample_count: Optional[int] = None, filters: Optional[str] = None, sort_by: Optional[List[str]] = None, - allow_partial_results: bool = True, + allow_partial_results: str = "true", ) -> ResultsResponse: """GET results from Dune API for `job_id` (aka `execution_id`)""" params = self._build_parameters( diff --git a/dune_client/api/extensions.py b/dune_client/api/extensions.py index 16dab45..44fda20 100644 --- a/dune_client/api/extensions.py +++ b/dune_client/api/extensions.py @@ -54,7 +54,7 @@ def run_query( sample_count: Optional[int] = None, filters: Optional[str] = None, sort_by: Optional[List[str]] = None, - allow_partial_results: bool = True, + allow_partial_results: str = "true", ) -> ResultsResponse: """ Executes a Dune `query`, waits until execution completes, @@ -418,6 +418,8 @@ def _refresh( ) time.sleep(ping_frequency) status = self.get_execution_status(job_id) + if status.state == ExecutionState.PENDING: + self.logger.warning("Partial result set retrieved.") if status.state == ExecutionState.FAILED: self.logger.error(status) raise QueryFailed(f"Error data: {status.error}") diff --git a/dune_client/client_async.py b/dune_client/client_async.py index 2db989e..7af0b20 100644 --- a/dune_client/client_async.py +++ b/dune_client/client_async.py @@ -175,6 +175,7 @@ async def _get() -> Any: headers=self.default_headers(), params=params, ) + print(response) if raw: return response return await self._handle_response(response) diff --git a/dune_client/models.py b/dune_client/models.py index 15c96ff..d648f08 100644 --- a/dune_client/models.py +++ b/dune_client/models.py @@ -312,6 +312,7 @@ class ResultsResponse: @classmethod def from_dict(cls, data: dict[str, str | int | ResultData]) -> ResultsResponse: """Constructor from dictionary. See unit test for sample input.""" + print(data) assert isinstance(data["execution_id"], str) assert isinstance(data["query_id"], int) assert isinstance(data["state"], str)