From cfafd3033cd1218f3ba2212a3e7d829ec2b63110 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 8 Mar 2023 15:55:53 -0800 Subject: [PATCH 01/46] Types in poller --- .../azure/core/polling/_async_poller.py | 4 ++-- .../azure-core/azure/core/polling/_poller.py | 21 +++++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index c502223635312..c4aca6f40e2f7 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -71,7 +71,7 @@ async def run(self): # pylint:disable=invalid-overridden-method """ -async def async_poller(client, initial_response, deserialization_callback, polling_method): +async def async_poller(client: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType], polling_method: AsyncPollingMethod[PollingReturnType]): """Async Poller for long running operations. .. deprecated:: 1.5.0 @@ -109,7 +109,7 @@ def __init__( self, client: Any, initial_response: Any, - deserialization_callback: Callable, + deserialization_callback: Callable[[Any], PollingReturnType], polling_method: AsyncPollingMethod[PollingReturnType], ): self._polling_method = polling_method diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 84067f3013926..02e0820590737 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -64,14 +64,16 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any raise TypeError("Polling method '{}' doesn't support from_continuation_token".format(cls.__name__)) -class NoPolling(PollingMethod): +class NoPolling(PollingMethod, Generic[PollingReturnType]): """An empty poller that returns the deserialized initial response.""" + _deserialization_callback: Callable[[Any], PollingReturnType] + """Deserialization callback passed during initialization""" + def __init__(self): self._initial_response = None - self._deserialization_callback = None - def initialize(self, _: Any, initial_response: Any, deserialization_callback: Callable) -> None: + def initialize(self, _: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType]) -> None: self._initial_response = initial_response self._deserialization_callback = deserialization_callback @@ -92,7 +94,7 @@ def finished(self) -> bool: """ return True - def resource(self) -> Any: + def resource(self) -> PollingReturnType: return self._deserialization_callback(self._initial_response) def get_continuation_token(self) -> str: @@ -130,7 +132,7 @@ def __init__( self, client: Any, initial_response: Any, - deserialization_callback: Callable, + deserialization_callback: Callable[[Any], PollingReturnType], polling_method: PollingMethod[PollingReturnType], ) -> None: self._callbacks: List[Callable] = [] @@ -147,10 +149,11 @@ def __init__( # Prepare thread execution self._thread = None - self._done = None + self._done = threading.Event() self._exception = None - if not self._polling_method.finished(): - self._done = threading.Event() + if self._polling_method.finished(): + self._done.set() + else: self._thread = threading.Thread( target=with_current_context(self._start), name="LROPoller({})".format(uuid.uuid4()), @@ -266,7 +269,7 @@ def add_done_callback(self, func: Callable) -> None: argument, a completed LongRunningOperation. """ # Still use "_done" and not "done", since CBs are executed inside the thread. - if self._done is None or self._done.is_set(): + if self._done.is_set(): func(self._polling_method) # Let's add them still, for consistency (if you wish to access to it for some reasons) self._callbacks.append(func) From dbe3d324202498456e8f30c9f9eb30bcc8fbe69e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 17 Mar 2023 13:35:21 -0700 Subject: [PATCH 02/46] Black it right --- sdk/core/azure-core/azure/core/polling/_async_poller.py | 7 ++++++- sdk/core/azure-core/azure/core/polling/_poller.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index c4aca6f40e2f7..d4469ffb0afd0 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -71,7 +71,12 @@ async def run(self): # pylint:disable=invalid-overridden-method """ -async def async_poller(client: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType], polling_method: AsyncPollingMethod[PollingReturnType]): +async def async_poller( + client: Any, + initial_response: Any, + deserialization_callback: Callable[[Any], PollingReturnType], + polling_method: AsyncPollingMethod[PollingReturnType], +): """Async Poller for long running operations. .. deprecated:: 1.5.0 diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 02e0820590737..24b1d5f00fd78 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -73,7 +73,9 @@ class NoPolling(PollingMethod, Generic[PollingReturnType]): def __init__(self): self._initial_response = None - def initialize(self, _: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType]) -> None: + def initialize( + self, _: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType] + ) -> None: self._initial_response = initial_response self._deserialization_callback = deserialization_callback From 6fc87dee78567b241d1c90bcf12cd24102c6c9d9 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 17 Mar 2023 17:46:40 -0700 Subject: [PATCH 03/46] Saving this week's work --- .../azure/core/pipeline/policies/_utils.py | 13 ++- .../azure/core/pipeline/transport/_base.py | 2 +- .../azure/core/polling/async_base_polling.py | 34 +++++- .../azure/core/polling/base_polling.py | 103 ++++++++++++------ 4 files changed, 107 insertions(+), 45 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py index 67bf5357a823d..f8386fec2cf91 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py @@ -25,21 +25,24 @@ # -------------------------------------------------------------------------- import datetime import email.utils +from typing import Optional + from ...utils._utils import _FixedOffset, case_insensitive_dict -def _parse_http_date(text): +def _parse_http_date(text: str) -> datetime.datetime: """Parse a HTTP date format into datetime.""" parsed_date = email.utils.parsedate_tz(text) - return datetime.datetime(*parsed_date[:6], tzinfo=_FixedOffset(parsed_date[9] / 60)) + return datetime.datetime(*parsed_date[:6], tzinfo=_FixedOffset(parsed_date[9] / 60)) # type: ignore -def parse_retry_after(retry_after): +def parse_retry_after(retry_after: str) -> float: """Helper to parse Retry-After and get value in seconds. :param str retry_after: Retry-After header - :rtype: int + :rtype: float """ + delay: float # Using the Mypy recommendation to use float for "int or float" try: delay = int(retry_after) except ValueError: @@ -49,7 +52,7 @@ def parse_retry_after(retry_after): return max(0, delay) -def get_retry_after(response): +def get_retry_after(response) -> Optional[float]: """Get the value of Retry-After in seconds. :param response: The PipelineResponse object diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py index d33485c823611..326a0bcafffe6 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_base.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_base.py @@ -133,7 +133,7 @@ def open(self): def close(self): """Close the session if it is not externally owned.""" - def sleep(self, duration): # pylint: disable=no-self-use + def sleep(self, duration: float) -> None: # pylint: disable=no-self-use time.sleep(duration) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index f57ae55eacdc4..6422140875a4c 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,6 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- +from typing import TYPE_CHECKING from ..exceptions import HttpResponseError from .base_polling import ( _failed, @@ -34,13 +35,38 @@ ) from ..pipeline._tools import is_rest +if TYPE_CHECKING: + from azure.core import AsyncPipelineClient + from azure.core.pipeline import PipelineResponse + from azure.core.pipeline.transport import ( + AsyncHttpResponse, + HttpRequest, + AsyncHttpTransport + ) + + AsyncPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] + + __all__ = ["AsyncLROBasePolling"] class AsyncLROBasePolling(LROBasePolling): """A subclass or LROBasePolling that redefine "run" as async.""" - async def run(self): # pylint:disable=invalid-overridden-method + _initial_response: "AsyncPipelineResponseType" + """Store the initial response.""" + + _pipeline_response: "AsyncPipelineResponseType" + """Store the latest received HTTP response, initialized by the first answer.""" + + _client: "AsyncPipelineClient" + """The Azure Core Async Pipeline client used to make request.""" + + @property + def _transport(self) -> "AsyncHttpTransport": + return self._client._pipeline._transport # pylint: disable=protected-access + + async def run(self) -> None: # pylint:disable=invalid-overridden-method try: await self._poll() @@ -59,7 +85,7 @@ async def run(self): # pylint:disable=invalid-overridden-method except OperationFailed as err: raise HttpResponseError(response=self._pipeline_response.http_response, error=err) - async def _poll(self): # pylint:disable=invalid-overridden-method + async def _poll(self) -> None: # pylint:disable=invalid-overridden-method """Poll status of operation so long as operation is incomplete and we have an endpoint to query. @@ -83,7 +109,7 @@ async def _poll(self): # pylint:disable=invalid-overridden-method self._pipeline_response = await self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - async def _sleep(self, delay): # pylint:disable=invalid-overridden-method + async def _sleep(self, delay: float): # pylint:disable=invalid-overridden-method await self._transport.sleep(delay) async def _delay(self): # pylint:disable=invalid-overridden-method @@ -99,7 +125,7 @@ async def update_status(self): # pylint:disable=invalid-overridden-method _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link): # pylint:disable=invalid-overridden-method + async def request_status(self, status_link: str): # pylint:disable=invalid-overridden-method """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 0d8c0730e18e0..c940ed7673aab 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -27,7 +27,7 @@ import base64 import json from enum import Enum -from typing import TYPE_CHECKING, Optional, Any, Union, Tuple, Callable, Dict +from typing import TYPE_CHECKING, Optional, Any, Union, Tuple, Callable, Dict, List, Generic, TypeVar from ..exceptions import HttpResponseError, DecodeError from . import PollingMethod @@ -36,11 +36,13 @@ from .._enum_meta import CaseInsensitiveEnumMeta if TYPE_CHECKING: + from azure.core import PipelineClient from azure.core.pipeline import PipelineResponse from azure.core.pipeline.transport import ( HttpResponse, AsyncHttpResponse, HttpRequest, + HttpTransport ) ResponseType = Union[HttpResponse, AsyncHttpResponse] @@ -48,6 +50,7 @@ ABC = abc.ABC +PollingReturnType = TypeVar("PollingReturnType") _FINISHED = frozenset(["succeeded", "canceled", "failed"]) _FAILED = frozenset(["canceled", "failed"]) @@ -183,13 +186,20 @@ class OperationResourcePolling(LongRunningOperation): https://aka.ms/azsdk/autorest/openapi/lro-options """ - def __init__(self, operation_location_header="operation-location", *, lro_options=None): + _async_url: str + """Url to resource monitor (AzureAsyncOperation or Operation-Location)""" + + _location_url: Optional[str] + """Location header if present""" + + _request: Any + """The initial request done""" + + def __init__( + self, operation_location_header: str = "operation-location", *, lro_options: Optional[Dict[str, Any]] = None + ): self._operation_location_header = operation_location_header - # Store the initial URLs - self._async_url = None - self._location_url = None - self._request = None self._lro_options = lro_options or {} def can_poll(self, pipeline_response): @@ -277,8 +287,9 @@ def get_status(self, pipeline_response: "PipelineResponseType") -> str: class LocationPolling(LongRunningOperation): """Implements a Location polling.""" - def __init__(self): - self._location_url = None + _location_url: str + """Location header""" + def can_poll(self, pipeline_response: "PipelineResponseType") -> bool: """Answer if this polling method could be used.""" @@ -354,7 +365,7 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None -class LROBasePolling(PollingMethod): # pylint: disable=too-many-instance-attributes +class LROBasePolling(PollingMethod, Generic[PollingReturnType]): # pylint: disable=too-many-instance-attributes """A base LRO poller. This assumes a basic flow: @@ -365,8 +376,31 @@ class LROBasePolling(PollingMethod): # pylint: disable=too-many-instance-attrib If your polling need are more specific, you could implement a PollingMethod directly """ + _initial_response: PipelineResponseType + """Store the initial response.""" + + _pipeline_response: PipelineResponseType + """Store the latest received HTTP response, initialized by the first answer.""" + + _deserialization_callback: Callable[[Any], PollingReturnType] + """The deserialization callback that returns the final instance.""" + + _operation: LongRunningOperation + """The algorithm this poller has currently decided to use. Will loop through 'can_poll' of the input algorithms to decide.""" + + _status: str + """Hold the current of this poller""" + + _client: "PipelineClient" + """The Azure Core Pipeline client used to make request.""" + def __init__( - self, timeout=30, lro_algorithms=None, lro_options=None, path_format_arguments=None, **operation_config + self, + timeout: float = 30, + lro_algorithms: Optional[List[LongRunningOperation]] = None, + lro_options: Optional[Dict[str, Any]] = None, + path_format_arguments: Optional[Dict[str, str]] = None, + **operation_config ): self._lro_algorithms = lro_algorithms or [ OperationResourcePolling(lro_options=lro_options), @@ -375,17 +409,11 @@ def __init__( ] self._timeout = timeout - self._client = None # Will hold the Pipelineclient - self._operation = None # Will hold an instance of LongRunningOperation - self._initial_response = None # Will hold the initial response - self._pipeline_response = None # Will hold latest received response - self._deserialization_callback = None # Will hold the deserialization callback self._operation_config = operation_config self._lro_options = lro_options self._path_format_arguments = path_format_arguments - self._status = None - def status(self): + def status(self) -> str: """Return the current status as a string. :rtype: str """ @@ -393,21 +421,26 @@ def status(self): raise ValueError("set_initial_status was never called. Did you give this instance to a poller?") return self._status - def finished(self): + def finished(self) -> bool: """Is this polling finished? :rtype: bool """ return _finished(self.status()) - def resource(self): + def resource(self) -> Optional[PollingReturnType]: """Return the built resource.""" return self._parse_resource(self._pipeline_response) @property - def _transport(self): + def _transport(self) -> "HttpTransport": return self._client._pipeline._transport # pylint: disable=protected-access - def initialize(self, client, initial_response, deserialization_callback): + def initialize( + self, + client: "PipelineClient", + initial_response: Any, + deserialization_callback: Callable[[Any], PollingReturnType], + ) -> None: """Set the initial status of this LRO. :param initial_response: The initial response of the poller @@ -443,7 +476,9 @@ def get_continuation_token(self) -> str: return base64.b64encode(pickle.dumps(self._initial_response)).decode("ascii") @classmethod - def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any, Any, Callable]: + def from_continuation_token( + cls, continuation_token: str, **kwargs + ) -> Tuple[Any, Any, Callable[[Any], PollingReturnType]]: try: client = kwargs["client"] except KeyError: @@ -461,7 +496,7 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any initial_response.context.transport = client._pipeline._transport # pylint: disable=protected-access return client, initial_response, deserialization_callback - def run(self): + def run(self) -> None: try: self._poll() @@ -480,7 +515,7 @@ def run(self): except OperationFailed as err: raise HttpResponseError(response=self._pipeline_response.http_response, error=err) - def _poll(self): + def _poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. @@ -504,7 +539,7 @@ def _poll(self): self._pipeline_response = self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - def _parse_resource(self, pipeline_response: "PipelineResponseType") -> Optional[Any]: + def _parse_resource(self, pipeline_response: "PipelineResponseType") -> Optional[PollingReturnType]: """Assuming this response is a resource, use the deserialization callback to parse it. If body is empty, assuming no resource to return. """ @@ -513,34 +548,32 @@ def _parse_resource(self, pipeline_response: "PipelineResponseType") -> Optional return self._deserialization_callback(pipeline_response) return None - def _sleep(self, delay): + def _sleep(self, delay: float) -> None: self._transport.sleep(delay) - def _extract_delay(self): - if self._pipeline_response is None: - return None + def _extract_delay(self) -> float: delay = get_retry_after(self._pipeline_response) if delay: return delay return self._timeout - def _delay(self): + def _delay(self) -> None: """Check for a 'retry-after' header to set timeout, otherwise use configured timeout. """ delay = self._extract_delay() self._sleep(delay) - def update_status(self): + def update_status(self) -> None: """Update the current status of the LRO.""" self._pipeline_response = self.request_status(self._operation.get_polling_url()) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - def _get_request_id(self): + def _get_request_id(self) -> str: return self._pipeline_response.http_response.request.headers["x-ms-client-request-id"] - def request_status(self, status_link): + def request_status(self, status_link: str): """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -557,8 +590,8 @@ def request_status(self, status_link): # want to keep making azure.core.rest calls from azure.core.rest import HttpRequest as RestHttpRequest - request = RestHttpRequest("GET", status_link) - return self._client.send_request(request, _return_pipeline_response=True, **self._operation_config) + rest_request = RestHttpRequest("GET", status_link) + return self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config) # if I am a azure.core.pipeline.transport.HttpResponse request = self._client.get(status_link) return self._client._pipeline.run( # pylint: disable=protected-access From 1facfcf998046f3ae4fc00870fcd277fb08c515d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 20 Mar 2023 11:46:52 -0700 Subject: [PATCH 04/46] Move typing --- sdk/core/azure-core/azure/core/polling/_poller.py | 2 +- .../azure-core/azure/core/polling/async_base_polling.py | 5 +++-- sdk/core/azure-core/azure/core/polling/base_polling.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 24b1d5f00fd78..48fff9bb5e6c8 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -64,7 +64,7 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any raise TypeError("Polling method '{}' doesn't support from_continuation_token".format(cls.__name__)) -class NoPolling(PollingMethod, Generic[PollingReturnType]): +class NoPolling(PollingMethod[PollingReturnType], Generic[PollingReturnType]): """An empty poller that returns the deserialized initial response.""" _deserialization_callback: Callable[[Any], PollingReturnType] diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 6422140875a4c..cdf1e87f17029 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Generic, TypeVar from ..exceptions import HttpResponseError from .base_polling import ( _failed, @@ -46,11 +46,12 @@ AsyncPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] +PollingReturnType = TypeVar("PollingReturnType") __all__ = ["AsyncLROBasePolling"] -class AsyncLROBasePolling(LROBasePolling): +class AsyncLROBasePolling(LROBasePolling[PollingReturnType], Generic[PollingReturnType]): """A subclass or LROBasePolling that redefine "run" as async.""" _initial_response: "AsyncPipelineResponseType" diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index c940ed7673aab..f74e3d7603577 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -365,7 +365,7 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None -class LROBasePolling(PollingMethod, Generic[PollingReturnType]): # pylint: disable=too-many-instance-attributes +class LROBasePolling(PollingMethod[PollingReturnType], Generic[PollingReturnType]): # pylint: disable=too-many-instance-attributes """A base LRO poller. This assumes a basic flow: @@ -427,7 +427,7 @@ def finished(self) -> bool: """ return _finished(self.status()) - def resource(self) -> Optional[PollingReturnType]: + def resource(self) -> PollingReturnType: """Return the built resource.""" return self._parse_resource(self._pipeline_response) @@ -539,7 +539,7 @@ def _poll(self) -> None: self._pipeline_response = self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - def _parse_resource(self, pipeline_response: "PipelineResponseType") -> Optional[PollingReturnType]: + def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingReturnType: """Assuming this response is a resource, use the deserialization callback to parse it. If body is empty, assuming no resource to return. """ From 7f7f56015a919b5926c1c722a591042930ca2bb0 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 27 Mar 2023 16:23:34 -0700 Subject: [PATCH 05/46] Split base polling in two --- .../azure/core/polling/async_base_polling.py | 29 +++-- .../azure/core/polling/base_polling.py | 110 +++++++++--------- 2 files changed, 76 insertions(+), 63 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index cdf1e87f17029..0f090e2831494 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -30,9 +30,10 @@ BadStatus, BadResponse, OperationFailed, - LROBasePolling, + _SansIOLROBasePolling, _raise_if_bad_http_status_and_method, ) +from ._async_poller import AsyncPollingMethod from ..pipeline._tools import is_rest if TYPE_CHECKING: @@ -51,8 +52,16 @@ __all__ = ["AsyncLROBasePolling"] -class AsyncLROBasePolling(LROBasePolling[PollingReturnType], Generic[PollingReturnType]): - """A subclass or LROBasePolling that redefine "run" as async.""" +class AsyncLROBasePolling(_SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient"], AsyncPollingMethod[PollingReturnType]): + """A base LRO async poller. + + This assumes a basic flow: + - I analyze the response to decide the polling approach + - I poll + - I ask the final resource depending of the polling approach + + If your polling need are more specific, you could implement a PollingMethod directly + """ _initial_response: "AsyncPipelineResponseType" """Store the initial response.""" @@ -60,14 +69,12 @@ class AsyncLROBasePolling(LROBasePolling[PollingReturnType], Generic[PollingRetu _pipeline_response: "AsyncPipelineResponseType" """Store the latest received HTTP response, initialized by the first answer.""" - _client: "AsyncPipelineClient" - """The Azure Core Async Pipeline client used to make request.""" @property def _transport(self) -> "AsyncHttpTransport": return self._client._pipeline._transport # pylint: disable=protected-access - async def run(self) -> None: # pylint:disable=invalid-overridden-method + async def run(self) -> None: try: await self._poll() @@ -86,7 +93,7 @@ async def run(self) -> None: # pylint:disable=invalid-overridden-method except OperationFailed as err: raise HttpResponseError(response=self._pipeline_response.http_response, error=err) - async def _poll(self) -> None: # pylint:disable=invalid-overridden-method + async def _poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. @@ -110,23 +117,23 @@ async def _poll(self) -> None: # pylint:disable=invalid-overridden-method self._pipeline_response = await self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - async def _sleep(self, delay: float): # pylint:disable=invalid-overridden-method + async def _sleep(self, delay: float): await self._transport.sleep(delay) - async def _delay(self): # pylint:disable=invalid-overridden-method + async def _delay(self): """Check for a 'retry-after' header to set timeout, otherwise use configured timeout. """ delay = self._extract_delay() await self._sleep(delay) - async def update_status(self): # pylint:disable=invalid-overridden-method + async def update_status(self): """Update the current status of the LRO.""" self._pipeline_response = await self.request_status(self._operation.get_polling_url()) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link: str): # pylint:disable=invalid-overridden-method + async def request_status(self, status_link: str): """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index f74e3d7603577..7a74ff5da1595 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -51,6 +51,7 @@ ABC = abc.ABC PollingReturnType = TypeVar("PollingReturnType") +PipelineClientType = TypeVar("PipelineClientType") _FINISHED = frozenset(["succeeded", "canceled", "failed"]) _FAILED = frozenset(["canceled", "failed"]) @@ -199,7 +200,7 @@ def __init__( self, operation_location_header: str = "operation-location", *, lro_options: Optional[Dict[str, Any]] = None ): self._operation_location_header = operation_location_header - + self._location_url = None self._lro_options = lro_options or {} def can_poll(self, pipeline_response): @@ -365,22 +366,8 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None -class LROBasePolling(PollingMethod[PollingReturnType], Generic[PollingReturnType]): # pylint: disable=too-many-instance-attributes - """A base LRO poller. - - This assumes a basic flow: - - I analyze the response to decide the polling approach - - I poll - - I ask the final resource depending of the polling approach - - If your polling need are more specific, you could implement a PollingMethod directly - """ - - _initial_response: PipelineResponseType - """Store the initial response.""" - - _pipeline_response: PipelineResponseType - """Store the latest received HTTP response, initialized by the first answer.""" +class _SansIOLROBasePolling(Generic[PollingReturnType, PipelineClientType]): + """A base class that has no opinion on IO, to help mypy be accurate.""" _deserialization_callback: Callable[[Any], PollingReturnType] """The deserialization callback that returns the final instance.""" @@ -391,7 +378,7 @@ class LROBasePolling(PollingMethod[PollingReturnType], Generic[PollingReturnType _status: str """Hold the current of this poller""" - _client: "PipelineClient" + _client: PipelineClientType """The Azure Core Pipeline client used to make request.""" def __init__( @@ -413,31 +400,10 @@ def __init__( self._lro_options = lro_options self._path_format_arguments = path_format_arguments - def status(self) -> str: - """Return the current status as a string. - :rtype: str - """ - if not self._operation: - raise ValueError("set_initial_status was never called. Did you give this instance to a poller?") - return self._status - - def finished(self) -> bool: - """Is this polling finished? - :rtype: bool - """ - return _finished(self.status()) - - def resource(self) -> PollingReturnType: - """Return the built resource.""" - return self._parse_resource(self._pipeline_response) - - @property - def _transport(self) -> "HttpTransport": - return self._client._pipeline._transport # pylint: disable=protected-access def initialize( self, - client: "PipelineClient", + client: PipelineClientType, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType], ) -> None: @@ -496,6 +462,58 @@ def from_continuation_token( initial_response.context.transport = client._pipeline._transport # pylint: disable=protected-access return client, initial_response, deserialization_callback + def status(self) -> str: + """Return the current status as a string. + :rtype: str + """ + if not self._operation: + raise ValueError("set_initial_status was never called. Did you give this instance to a poller?") + return self._status + + def finished(self) -> bool: + """Is this polling finished? + :rtype: bool + """ + return _finished(self.status()) + + def resource(self) -> PollingReturnType: + """Return the built resource.""" + return self._parse_resource(self._pipeline_response) + + def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingReturnType: + """Assuming this response is a resource, use the deserialization callback to parse it. + If body is empty, assuming no resource to return. + """ + response = pipeline_response.http_response + if not _is_empty(response): + return self._deserialization_callback(pipeline_response) + return None + + def _get_request_id(self) -> str: + return self._pipeline_response.http_response.request.headers["x-ms-client-request-id"] + + +class LROBasePolling(_SansIOLROBasePolling[PollingReturnType, "PipelineClient"], PollingMethod[PollingReturnType]): # pylint: disable=too-many-instance-attributes + """A base LRO poller. + + This assumes a basic flow: + - I analyze the response to decide the polling approach + - I poll + - I ask the final resource depending of the polling approach + + If your polling need are more specific, you could implement a PollingMethod directly + """ + + _initial_response: "PipelineResponseType" + """Store the initial response.""" + + _pipeline_response: "PipelineResponseType" + """Store the latest received HTTP response, initialized by the first answer.""" + + @property + def _transport(self) -> "HttpTransport": + return self._client._pipeline._transport # pylint: disable=protected-access + def run(self) -> None: try: self._poll() @@ -539,15 +557,6 @@ def _poll(self) -> None: self._pipeline_response = self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingReturnType: - """Assuming this response is a resource, use the deserialization callback to parse it. - If body is empty, assuming no resource to return. - """ - response = pipeline_response.http_response - if not _is_empty(response): - return self._deserialization_callback(pipeline_response) - return None - def _sleep(self, delay: float) -> None: self._transport.sleep(delay) @@ -570,9 +579,6 @@ def update_status(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - def _get_request_id(self) -> str: - return self._pipeline_response.http_response.request.headers["x-ms-client-request-id"] - def request_status(self, status_link: str): """Do a simple GET to this status link. From 13168972134e7d68cd219a6ed323b981d474f37f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 15:48:58 -0700 Subject: [PATCH 06/46] Typing fixes --- .../core/pipeline/policies/_universal.py | 13 ++++++++-- .../azure/core/polling/async_base_polling.py | 14 +++++----- .../azure/core/polling/base_polling.py | 26 ++++++++++++------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py index 9751be46de7cf..63d066a844541 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py @@ -35,7 +35,7 @@ import types import re import uuid -from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping +from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping, runtime_checkable import urllib.parse from typing_extensions import Protocol @@ -49,15 +49,17 @@ _LOGGER = logging.getLogger(__name__) +@runtime_checkable class HTTPRequestType(Protocol): """Protocol compatible with new rest request and legacy transport request""" headers: MutableMapping[str, str] url: str method: str - body: bytes + body: Optional[Union[bytes, Dict[str, Union[str, int]]]] +@runtime_checkable class HTTPResponseType(Protocol): """Protocol compatible with new rest response and legacy transport response""" @@ -73,9 +75,16 @@ def status_code(self) -> int: def content_type(self) -> Optional[str]: ... + @property + def request(self) -> HTTPRequestType: + ... + def text(self, encoding: Optional[str] = None) -> str: ... + def body(self) -> bytes: + ... + class HeadersPolicy(SansIOHTTPPolicy[HTTPRequestType, HTTPResponseType]): """A simple policy that sends the given headers with the request. diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 0f090e2831494..3d9d60f98df61 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -36,23 +36,25 @@ from ._async_poller import AsyncPollingMethod from ..pipeline._tools import is_rest + if TYPE_CHECKING: from azure.core import AsyncPipelineClient from azure.core.pipeline import PipelineResponse from azure.core.pipeline.transport import ( - AsyncHttpResponse, - HttpRequest, AsyncHttpTransport ) + from azure.core._pipeline_client_async import _AsyncContextManagerCloseable + from azure.core.pipeline.policies._universal import HTTPRequestType - AsyncPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] + AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType", bound="_AsyncContextManagerCloseable") + AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] PollingReturnType = TypeVar("PollingReturnType") __all__ = ["AsyncLROBasePolling"] -class AsyncLROBasePolling(_SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient"], AsyncPollingMethod[PollingReturnType]): +class AsyncLROBasePolling(_SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]"], AsyncPollingMethod[PollingReturnType]): """A base LRO async poller. This assumes a basic flow: @@ -153,10 +155,10 @@ async def request_status(self, status_link: str): request = RestHttpRequest("GET", status_link) return await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config) # if I am a azure.core.pipeline.transport.HttpResponse - request = self._client.get(status_link) + legacy_request = self._client.get(status_link) return await self._client._pipeline.run( # pylint: disable=protected-access - request, stream=False, **self._operation_config + legacy_request, stream=False, **self._operation_config ) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 7a74ff5da1595..a0abdc4a0d75b 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -35,18 +35,17 @@ from ..pipeline._tools import is_rest from .._enum_meta import CaseInsensitiveEnumMeta +HTTPRequestType = TypeVar("HTTPRequestType") + if TYPE_CHECKING: from azure.core import PipelineClient from azure.core.pipeline import PipelineResponse from azure.core.pipeline.transport import ( - HttpResponse, - AsyncHttpResponse, - HttpRequest, HttpTransport ) + from azure.core.pipeline.policies._universal import HTTPResponseType - ResponseType = Union[HttpResponse, AsyncHttpResponse] - PipelineResponseType = PipelineResponse[HttpRequest, ResponseType] + PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] ABC = abc.ABC @@ -88,7 +87,7 @@ class OperationFailed(Exception): pass -def _as_json(response: "ResponseType") -> Dict[str, Any]: +def _as_json(response: "HTTPResponseType") -> Dict[str, Any]: """Assuming this is not empty, return the content as JSON. Result/exceptions is not determined if you call this method without testing _is_empty. @@ -101,7 +100,7 @@ def _as_json(response: "ResponseType") -> Dict[str, Any]: raise DecodeError("Error occurred in deserializing the response body.") -def _raise_if_bad_http_status_and_method(response: "ResponseType") -> None: +def _raise_if_bad_http_status_and_method(response: "HTTPResponseType") -> None: """Check response status code is valid. Must be 200, 201, 202, or 204. @@ -114,7 +113,7 @@ def _raise_if_bad_http_status_and_method(response: "ResponseType") -> None: raise BadStatus("Invalid return status {!r} for {!r} operation".format(code, response.request.method)) -def _is_empty(response: "ResponseType") -> bool: +def _is_empty(response: "HTTPResponseType") -> bool: """Check if response body contains meaningful content. :rtype: bool @@ -261,7 +260,7 @@ def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: return "InProgress" raise OperationFailed("Operation failed or canceled") - def _set_async_url_if_present(self, response: "ResponseType") -> None: + def _set_async_url_if_present(self, response: "HTTPResponseType") -> None: self._async_url = response.headers[self._operation_location_header] location_url = response.headers.get("location") @@ -487,7 +486,14 @@ def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingR response = pipeline_response.http_response if not _is_empty(response): return self._deserialization_callback(pipeline_response) - return None + + # This "type ignore" has been discussed with architects. + # We have a typing problem that if the Swagger/TSP describes a return type (PollingReturnType is not None), BUT + # the returned payload is actually empty, we don't want to fail, but return None. + # To make it clean, we would have to make the polling return type Optional "just in case the Swagger/TSP is wrong" + # This is reducing the quality and the value of the typing annotations for a case that is not supposed to happen + # in the first place. So we decided to ignore the type error here. + return None # type: ignore def _get_request_id(self) -> str: return self._pipeline_response.http_response.request.headers["x-ms-client-request-id"] From f3c42e975529884fd6e4b9bcf07b6d03c1e34938 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 16:19:19 -0700 Subject: [PATCH 07/46] Typing update --- .../azure/core/polling/async_base_polling.py | 10 ++++++---- .../azure/core/polling/base_polling.py | 20 ++++++++++--------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 3d9d60f98df61..938e3426406c8 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar, cast from ..exceptions import HttpResponseError from .base_polling import ( _failed, @@ -44,7 +44,7 @@ AsyncHttpTransport ) from azure.core._pipeline_client_async import _AsyncContextManagerCloseable - from azure.core.pipeline.policies._universal import HTTPRequestType + from azure.core.pipeline.policies._universal import HTTPRequestType, HTTPResponseType AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType", bound="_AsyncContextManagerCloseable") AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] @@ -135,7 +135,7 @@ async def update_status(self): _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link: str): + async def request_status(self, status_link: str) -> AsyncPipelineResponseType: """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -153,7 +153,9 @@ async def request_status(self, status_link: str): from azure.core.rest import HttpRequest as RestHttpRequest request = RestHttpRequest("GET", status_link) - return await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config) + # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not + # declared in the typing of "send_request" + return cast(AsyncPipelineResponseType, await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config)) # if I am a azure.core.pipeline.transport.HttpResponse legacy_request = self._client.get(status_link) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index a0abdc4a0d75b..ca5f6ce097252 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -27,7 +27,7 @@ import base64 import json from enum import Enum -from typing import TYPE_CHECKING, Optional, Any, Union, Tuple, Callable, Dict, List, Generic, TypeVar +from typing import TYPE_CHECKING, Optional, Any, Union, Tuple, Callable, Dict, List, Generic, TypeVar, cast from ..exceptions import HttpResponseError, DecodeError from . import PollingMethod @@ -498,6 +498,12 @@ def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingR def _get_request_id(self) -> str: return self._pipeline_response.http_response.request.headers["x-ms-client-request-id"] + def _extract_delay(self) -> float: + delay = get_retry_after(self._pipeline_response) + if delay: + return delay + return self._timeout + class LROBasePolling(_SansIOLROBasePolling[PollingReturnType, "PipelineClient"], PollingMethod[PollingReturnType]): # pylint: disable=too-many-instance-attributes """A base LRO poller. @@ -566,12 +572,6 @@ def _poll(self) -> None: def _sleep(self, delay: float) -> None: self._transport.sleep(delay) - def _extract_delay(self) -> float: - delay = get_retry_after(self._pipeline_response) - if delay: - return delay - return self._timeout - def _delay(self) -> None: """Check for a 'retry-after' header to set timeout, otherwise use configured timeout. @@ -585,7 +585,7 @@ def update_status(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - def request_status(self, status_link: str): + def request_status(self, status_link: str) -> "PipelineResponseType": """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -603,7 +603,9 @@ def request_status(self, status_link: str): from azure.core.rest import HttpRequest as RestHttpRequest rest_request = RestHttpRequest("GET", status_link) - return self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config) + # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not + # declared in the typing of "send_request" + return cast("PipelineResponseType", self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config)) # if I am a azure.core.pipeline.transport.HttpResponse request = self._client.get(status_link) return self._client._pipeline.run( # pylint: disable=protected-access From bfd8432f1a37b899af84fda77a8c0d646f25cf1f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 16:26:05 -0700 Subject: [PATCH 08/46] Black --- .../azure/core/polling/async_base_polling.py | 15 +++++++----- .../azure/core/polling/base_polling.py | 23 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 938e3426406c8..00ae8f2c9dc66 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -40,9 +40,7 @@ if TYPE_CHECKING: from azure.core import AsyncPipelineClient from azure.core.pipeline import PipelineResponse - from azure.core.pipeline.transport import ( - AsyncHttpTransport - ) + from azure.core.pipeline.transport import AsyncHttpTransport from azure.core._pipeline_client_async import _AsyncContextManagerCloseable from azure.core.pipeline.policies._universal import HTTPRequestType, HTTPResponseType @@ -54,7 +52,10 @@ __all__ = ["AsyncLROBasePolling"] -class AsyncLROBasePolling(_SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]"], AsyncPollingMethod[PollingReturnType]): +class AsyncLROBasePolling( + _SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]"], + AsyncPollingMethod[PollingReturnType], +): """A base LRO async poller. This assumes a basic flow: @@ -71,7 +72,6 @@ class AsyncLROBasePolling(_SansIOLROBasePolling[PollingReturnType, "AsyncPipelin _pipeline_response: "AsyncPipelineResponseType" """Store the latest received HTTP response, initialized by the first answer.""" - @property def _transport(self) -> "AsyncHttpTransport": return self._client._pipeline._transport # pylint: disable=protected-access @@ -155,7 +155,10 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: request = RestHttpRequest("GET", status_link) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" - return cast(AsyncPipelineResponseType, await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config)) + return cast( + AsyncPipelineResponseType, + await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config), + ) # if I am a azure.core.pipeline.transport.HttpResponse legacy_request = self._client.get(status_link) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index ca5f6ce097252..4311c7d350f7a 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -40,9 +40,7 @@ if TYPE_CHECKING: from azure.core import PipelineClient from azure.core.pipeline import PipelineResponse - from azure.core.pipeline.transport import ( - HttpTransport - ) + from azure.core.pipeline.transport import HttpTransport from azure.core.pipeline.policies._universal import HTTPResponseType PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] @@ -290,7 +288,6 @@ class LocationPolling(LongRunningOperation): _location_url: str """Location header""" - def can_poll(self, pipeline_response: "PipelineResponseType") -> bool: """Answer if this polling method could be used.""" response = pipeline_response.http_response @@ -372,7 +369,7 @@ class _SansIOLROBasePolling(Generic[PollingReturnType, PipelineClientType]): """The deserialization callback that returns the final instance.""" _operation: LongRunningOperation - """The algorithm this poller has currently decided to use. Will loop through 'can_poll' of the input algorithms to decide.""" + """The algorithm this poller has decided to use. Will loop through 'can_poll' of the input algorithms to decide.""" _status: str """Hold the current of this poller""" @@ -399,7 +396,6 @@ def __init__( self._lro_options = lro_options self._path_format_arguments = path_format_arguments - def initialize( self, client: PipelineClientType, @@ -490,9 +486,9 @@ def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingR # This "type ignore" has been discussed with architects. # We have a typing problem that if the Swagger/TSP describes a return type (PollingReturnType is not None), BUT # the returned payload is actually empty, we don't want to fail, but return None. - # To make it clean, we would have to make the polling return type Optional "just in case the Swagger/TSP is wrong" - # This is reducing the quality and the value of the typing annotations for a case that is not supposed to happen - # in the first place. So we decided to ignore the type error here. + # To make it clean, we would have to make the polling return type Optional + # "just in case the Swagger/TSP is wrong". This is reducing the quality and the value of the typing annotations + # for a case that is not supposed to happen in the first place. So we decided to ignore the type error here. return None # type: ignore def _get_request_id(self) -> str: @@ -505,7 +501,9 @@ def _extract_delay(self) -> float: return self._timeout -class LROBasePolling(_SansIOLROBasePolling[PollingReturnType, "PipelineClient"], PollingMethod[PollingReturnType]): # pylint: disable=too-many-instance-attributes +class LROBasePolling( + _SansIOLROBasePolling[PollingReturnType, "PipelineClient"], PollingMethod[PollingReturnType] +): # pylint: disable=too-many-instance-attributes """A base LRO poller. This assumes a basic flow: @@ -605,7 +603,10 @@ def request_status(self, status_link: str) -> "PipelineResponseType": rest_request = RestHttpRequest("GET", status_link) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" - return cast("PipelineResponseType", self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config)) + return cast( + "PipelineResponseType", + self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), + ) # if I am a azure.core.pipeline.transport.HttpResponse request = self._client.get(status_link) return self._client._pipeline.run( # pylint: disable=protected-access From bae1d5fc04bfcc5f70f6a37adc38cfed5a314ba2 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 16:35:20 -0700 Subject: [PATCH 09/46] Unecessary Generic --- sdk/core/azure-core/azure/core/polling/_poller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 48fff9bb5e6c8..52f93cdac6fd6 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -64,7 +64,7 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any raise TypeError("Polling method '{}' doesn't support from_continuation_token".format(cls.__name__)) -class NoPolling(PollingMethod[PollingReturnType], Generic[PollingReturnType]): +class NoPolling(PollingMethod[PollingReturnType]): """An empty poller that returns the deserialized initial response.""" _deserialization_callback: Callable[[Any], PollingReturnType] From e494ee05a11bfc34ccf533bd644fdb661dd39b62 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 16:37:26 -0700 Subject: [PATCH 10/46] Stringify types --- sdk/core/azure-core/azure/core/polling/async_base_polling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 00ae8f2c9dc66..aa69cb6205a86 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -135,7 +135,7 @@ async def update_status(self): _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link: str) -> AsyncPipelineResponseType: + async def request_status(self, status_link: str) -> "AsyncPipelineResponseType": """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -156,7 +156,7 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - AsyncPipelineResponseType, + "AsyncPipelineResponseType", await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config), ) # if I am a azure.core.pipeline.transport.HttpResponse From c7d8aa80d855695051f7a7b8ed3d105a91cd7d2d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 17:08:22 -0700 Subject: [PATCH 11/46] Fix import --- .../azure-core/azure/core/pipeline/policies/_universal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py index 63d066a844541..dc58e6930f712 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py @@ -35,9 +35,9 @@ import types import re import uuid -from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping, runtime_checkable +from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMappinggst import urllib.parse -from typing_extensions import Protocol +from typing_extensions import Protocol, runtime_checkable from azure.core import __version__ as azcore_version from azure.core.exceptions import DecodeError, raise_with_traceback From bbe53f25c912c402d18b79556424d797163f0f70 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 17:10:32 -0700 Subject: [PATCH 12/46] Spellcheck --- .vscode/cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index bd025731a5b18..783163c2cc65a 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -212,6 +212,7 @@ "ints", "iohttp", "IOHTTP", + "IOLRO", "inprogress", "ipconfiguration", "ipconfigurations", @@ -765,7 +766,7 @@ "xoperex", "Zocor" ] - }, + }, { "filename": "sdk/keyvault/**", "words": [ From e39cccd34ce8b1a22dcba468957cb0f63f049aba Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Apr 2023 17:38:25 -0700 Subject: [PATCH 13/46] Weird typo... --- sdk/core/azure-core/azure/core/pipeline/policies/_universal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py index dc58e6930f712..e98e7d5ff9934 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py @@ -35,7 +35,7 @@ import types import re import uuid -from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMappinggst +from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping import urllib.parse from typing_extensions import Protocol, runtime_checkable From 03a6d5ffaf2d898d26607bdedf82bb0680c8c5bb Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 21 Apr 2023 13:55:17 -0700 Subject: [PATCH 14/46] PyLint --- .../azure-core/azure/core/polling/async_base_polling.py | 2 +- sdk/core/azure-core/azure/core/polling/base_polling.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index aa69cb6205a86..2e0ccb88f3ef2 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING, Generic, TypeVar, cast +from typing import TYPE_CHECKING, TypeVar, cast from ..exceptions import HttpResponseError from .base_polling import ( _failed, diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 4311c7d350f7a..a599c961e58e3 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -27,7 +27,7 @@ import base64 import json from enum import Enum -from typing import TYPE_CHECKING, Optional, Any, Union, Tuple, Callable, Dict, List, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Optional, Any, Tuple, Callable, Dict, List, Generic, TypeVar, cast from ..exceptions import HttpResponseError, DecodeError from . import PollingMethod @@ -362,7 +362,7 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None -class _SansIOLROBasePolling(Generic[PollingReturnType, PipelineClientType]): +class _SansIOLROBasePolling(Generic[PollingReturnType, PipelineClientType]): # pylint: disable=too-many-instance-attributes """A base class that has no opinion on IO, to help mypy be accurate.""" _deserialization_callback: Callable[[Any], PollingReturnType] @@ -408,7 +408,7 @@ def initialize( :raises: HttpResponseError if initial status is incorrect LRO state """ self._client = client - self._pipeline_response = self._initial_response = initial_response + self._pipeline_response = self._initial_response = initial_response # pylint: disable=attribute-defined-outside-init self._deserialization_callback = deserialization_callback for operation in self._lro_algorithms: From bd00b834084637818eb9985ae96d0b913bfc039d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 21 Apr 2023 17:48:39 -0700 Subject: [PATCH 15/46] More types --- sdk/core/azure-core/azure/core/polling/base_polling.py | 7 +++---- sdk/core/azure-core/azure/core/rest/_helpers.py | 8 +++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index a599c961e58e3..1b486aa4d63c9 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -35,13 +35,12 @@ from ..pipeline._tools import is_rest from .._enum_meta import CaseInsensitiveEnumMeta -HTTPRequestType = TypeVar("HTTPRequestType") if TYPE_CHECKING: from azure.core import PipelineClient from azure.core.pipeline import PipelineResponse from azure.core.pipeline.transport import HttpTransport - from azure.core.pipeline.policies._universal import HTTPResponseType + from azure.core.pipeline.policies._universal import HTTPResponseType, HTTPRequestType PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] @@ -502,7 +501,7 @@ def _extract_delay(self) -> float: class LROBasePolling( - _SansIOLROBasePolling[PollingReturnType, "PipelineClient"], PollingMethod[PollingReturnType] + _SansIOLROBasePolling[PollingReturnType, "PipelineClient[HTTPRequestType, HTTPResponseType]"], PollingMethod[PollingReturnType] ): # pylint: disable=too-many-instance-attributes """A base LRO poller. @@ -521,7 +520,7 @@ class LROBasePolling( """Store the latest received HTTP response, initialized by the first answer.""" @property - def _transport(self) -> "HttpTransport": + def _transport(self) -> "HttpTransport[HTTPRequestType, HTTPResponseType]": return self._client._pipeline._transport # pylint: disable=protected-access def run(self) -> None: diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index bcc2a8c71c062..f1934bcbd8bd3 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -67,6 +67,8 @@ ContentTypeBase = Union[str, bytes, Iterable[bytes]] ContentType = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] +DataType = Optional[Union[bytes, Dict[str, Union[str, int]]]] + ########################### HELPER SECTION ################################# @@ -179,7 +181,7 @@ def decode_to_text(encoding: Optional[str], content: bytes) -> str: class HttpRequestBackcompatMixin: - def __getattr__(self, attr): + def __getattr__(self, attr: str): backcompat_attrs = [ "files", "data", @@ -238,14 +240,14 @@ def _query(self): return {} @property - def _body(self): + def _body(self) -> DataType: """DEPRECATED: Body of the request. You should use the `content` property instead This is deprecated and will be removed in a later release. """ return self._data @_body.setter - def _body(self, val): + def _body(self, val: DataType): """DEPRECATED: Set the body of the request This is deprecated and will be removed in a later release. """ From e149c704e7ef87dc02e003cb59e7958564804397 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 2 May 2023 11:34:49 -0700 Subject: [PATCH 16/46] Update sdk/core/azure-core/azure/core/polling/async_base_polling.py Co-authored-by: Kashif Khan <361477+kashifkhan@users.noreply.github.com> --- sdk/core/azure-core/azure/core/polling/async_base_polling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 2e0ccb88f3ef2..8adbd35deffd4 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -119,7 +119,7 @@ async def _poll(self) -> None: self._pipeline_response = await self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - async def _sleep(self, delay: float): + async def _sleep(self, delay: float) -> None: await self._transport.sleep(delay) async def _delay(self): From 34a63404c174c71dbf3799d78daecc56ef60d89d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 24 May 2023 12:57:47 -0700 Subject: [PATCH 17/46] Missing type --- sdk/core/azure-core/azure/core/polling/base_polling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 1b486aa4d63c9..5a3e0eca6b543 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -199,7 +199,7 @@ def __init__( self._location_url = None self._lro_options = lro_options or {} - def can_poll(self, pipeline_response): + def can_poll(self, pipeline_response: "PipelineResponseType"): """Answer if this polling method could be used.""" response = pipeline_response.http_response return self._operation_location_header in response.headers From 6465b24f86614207e61ce0100ccad74539ec9007 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 14 Jun 2023 17:00:39 -0700 Subject: [PATCH 18/46] Typing of the day --- .../core/pipeline/policies/_universal.py | 45 +++-------------- .../azure/core/polling/async_base_polling.py | 45 +++++++++-------- .../azure/core/polling/base_polling.py | 50 ++++++++++++------- 3 files changed, 64 insertions(+), 76 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py index e98e7d5ff9934..a32a32692a315 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py @@ -35,7 +35,7 @@ import types import re import uuid -from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping +from typing import IO, cast, Union, Optional, AnyStr, Dict, MutableMapping, Any import urllib.parse from typing_extensions import Protocol, runtime_checkable @@ -45,45 +45,16 @@ from azure.core.pipeline import PipelineRequest, PipelineResponse from ._base import SansIOHTTPPolicy +from ..transport import HttpRequest as LegacyHttpRequest +from ..transport._base import _HttpResponseBase as LegacySansIOHttpResponse +from ...rest import HttpRequest +from ...rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse _LOGGER = logging.getLogger(__name__) - -@runtime_checkable -class HTTPRequestType(Protocol): - """Protocol compatible with new rest request and legacy transport request""" - - headers: MutableMapping[str, str] - url: str - method: str - body: Optional[Union[bytes, Dict[str, Union[str, int]]]] - - -@runtime_checkable -class HTTPResponseType(Protocol): - """Protocol compatible with new rest response and legacy transport response""" - - @property - def headers(self) -> MutableMapping[str, str]: - ... - - @property - def status_code(self) -> int: - ... - - @property - def content_type(self) -> Optional[str]: - ... - - @property - def request(self) -> HTTPRequestType: - ... - - def text(self, encoding: Optional[str] = None) -> str: - ... - - def body(self) -> bytes: - ... +HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] +HTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] +PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] class HeadersPolicy(SansIOHTTPPolicy[HTTPRequestType, HTTPResponseType]): diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 8adbd35deffd4..8df0b1e8510c3 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING, TypeVar, cast +from typing import TYPE_CHECKING, TypeVar, cast, Union from ..exceptions import HttpResponseError from .base_polling import ( _failed, @@ -35,17 +35,17 @@ ) from ._async_poller import AsyncPollingMethod from ..pipeline._tools import is_rest - - -if TYPE_CHECKING: - from azure.core import AsyncPipelineClient - from azure.core.pipeline import PipelineResponse - from azure.core.pipeline.transport import AsyncHttpTransport - from azure.core._pipeline_client_async import _AsyncContextManagerCloseable - from azure.core.pipeline.policies._universal import HTTPRequestType, HTTPResponseType - - AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType", bound="_AsyncContextManagerCloseable") - AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] +from .. import AsyncPipelineClient +from ..pipeline import PipelineResponse +from ..pipeline.transport import HttpRequest as LegacyHttpRequest, AsyncHttpTransport +from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse +from .._pipeline_client_async import _AsyncContextManagerCloseable +from ..rest import HttpRequest +from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse + +HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] +AsyncHTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] +AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] PollingReturnType = TypeVar("PollingReturnType") @@ -66,10 +66,10 @@ class AsyncLROBasePolling( If your polling need are more specific, you could implement a PollingMethod directly """ - _initial_response: "AsyncPipelineResponseType" + _initial_response: AsyncPipelineResponseType """Store the initial response.""" - _pipeline_response: "AsyncPipelineResponseType" + _pipeline_response: AsyncPipelineResponseType """Store the latest received HTTP response, initialized by the first answer.""" @property @@ -135,7 +135,7 @@ async def update_status(self): _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link: str) -> "AsyncPipelineResponseType": + async def request_status(self, status_link: str) -> AsyncPipelineResponseType: """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -150,20 +150,21 @@ async def request_status(self, status_link: str) -> "AsyncPipelineResponseType": if is_rest(self._initial_response.http_response): # if I am a azure.core.rest.HttpResponse # want to keep making azure.core.rest calls - from azure.core.rest import HttpRequest as RestHttpRequest - request = RestHttpRequest("GET", status_link) + rest_request = HttpRequest("GET", status_link) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - "AsyncPipelineResponseType", - await self._client.send_request(request, _return_pipeline_response=True, **self._operation_config), + AsyncPipelineResponseType, + await self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) - # if I am a azure.core.pipeline.transport.HttpResponse - legacy_request = self._client.get(status_link) + # Legacy HttpRequest and AsyncHttpResponse from azure.core.pipeline.transport + # "Type ignore"-ing things here, as we don't want the typing system to know + # about the legacy APIs. + request = self._client.get(status_link) return await self._client._pipeline.run( # pylint: disable=protected-access - legacy_request, stream=False, **self._operation_config + request, stream=False, **self._operation_config ) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 5a3e0eca6b543..b1667357ebc7d 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -27,22 +27,24 @@ import base64 import json from enum import Enum -from typing import TYPE_CHECKING, Optional, Any, Tuple, Callable, Dict, List, Generic, TypeVar, cast +from typing import Optional, Any, Tuple, Callable, Dict, List, Generic, TypeVar, cast, Union from ..exceptions import HttpResponseError, DecodeError from . import PollingMethod from ..pipeline.policies._utils import get_retry_after from ..pipeline._tools import is_rest from .._enum_meta import CaseInsensitiveEnumMeta +from .. import PipelineClient +from ..pipeline import PipelineResponse +from ..pipeline.transport import HttpRequest as LegacyHttpRequest, HttpTransport +from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse +from ..rest import HttpRequest +from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse -if TYPE_CHECKING: - from azure.core import PipelineClient - from azure.core.pipeline import PipelineResponse - from azure.core.pipeline.transport import HttpTransport - from azure.core.pipeline.policies._universal import HTTPResponseType, HTTPRequestType - - PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] +HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] +HTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] +PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] ABC = abc.ABC @@ -54,6 +56,19 @@ _SUCCEEDED = frozenset(["succeeded"]) + +def _get_content(response: HTTPResponseType) -> bytes: + """Get the content of this response. This is designed specifically to avoid + a warning of mypy for body() access, as this method is deprecated. + + :param response: The response object. + :rtype: bytes + """ + if isinstance(response, LegacySansIOHttpResponse): + return response.body() + return response.content + + def _finished(status): if hasattr(status, "value"): status = status.value @@ -84,7 +99,7 @@ class OperationFailed(Exception): pass -def _as_json(response: "HTTPResponseType") -> Dict[str, Any]: +def _as_json(response: HTTPResponseType) -> Dict[str, Any]: """Assuming this is not empty, return the content as JSON. Result/exceptions is not determined if you call this method without testing _is_empty. @@ -97,7 +112,7 @@ def _as_json(response: "HTTPResponseType") -> Dict[str, Any]: raise DecodeError("Error occurred in deserializing the response body.") -def _raise_if_bad_http_status_and_method(response: "HTTPResponseType") -> None: +def _raise_if_bad_http_status_and_method(response: HTTPResponseType) -> None: """Check response status code is valid. Must be 200, 201, 202, or 204. @@ -110,12 +125,12 @@ def _raise_if_bad_http_status_and_method(response: "HTTPResponseType") -> None: raise BadStatus("Invalid return status {!r} for {!r} operation".format(code, response.request.method)) -def _is_empty(response: "HTTPResponseType") -> bool: +def _is_empty(response: HTTPResponseType) -> bool: """Check if response body contains meaningful content. :rtype: bool """ - return not bool(response.body()) + return not bool(_get_content(response)) class LongRunningOperation(ABC): @@ -257,7 +272,7 @@ def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: return "InProgress" raise OperationFailed("Operation failed or canceled") - def _set_async_url_if_present(self, response: "HTTPResponseType") -> None: + def _set_async_url_if_present(self, response: HTTPResponseType) -> None: self._async_url = response.headers[self._operation_location_header] location_url = response.headers.get("location") @@ -597,16 +612,17 @@ def request_status(self, status_link: str) -> "PipelineResponseType": if is_rest(self._initial_response.http_response): # if I am a azure.core.rest.HttpResponse # want to keep making azure.core.rest calls - from azure.core.rest import HttpRequest as RestHttpRequest - - rest_request = RestHttpRequest("GET", status_link) + rest_request = HttpRequest("GET", status_link) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( "PipelineResponseType", self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) - # if I am a azure.core.pipeline.transport.HttpResponse + + # Legacy HttpRequest and HttpResponse from azure.core.pipeline.transport + # "Type ignore"-ing things here, as we don't want the typing system to know + # about the legacy APIs. request = self._client.get(status_link) return self._client._pipeline.run( # pylint: disable=protected-access request, stream=False, **self._operation_config From 2a5f3acf401669215001baf3e50d09c781ccb92c Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Sat, 17 Jun 2023 16:50:48 -0700 Subject: [PATCH 19/46] Re-enable verifytypes --- sdk/core/azure-core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/pyproject.toml b/sdk/core/azure-core/pyproject.toml index 04cfdf63a644e..a8848916dda3d 100644 --- a/sdk/core/azure-core/pyproject.toml +++ b/sdk/core/azure-core/pyproject.toml @@ -1,7 +1,7 @@ [tool.azure-sdk-build] mypy = true type_check_samples = true -verifytypes = false +verifytypes = true pyright = false # For test environments or static checks where a check should be run by default, not explicitly disabling will enable the check. # pylint is enabled by default, so there is no reason for a pylint = true in every pyproject.toml. From b3fb232ce6c2173df70c21111f73eadaa980d9ba Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 19 Jun 2023 11:51:33 -0700 Subject: [PATCH 20/46] Simplify the expectations async pipeline has on the response --- .../azure/core/_pipeline_client_async.py | 21 +++---------------- .../azure/core/polling/async_base_polling.py | 3 +-- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index cbc70d4c0d53c..7945a32353089 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -35,7 +35,6 @@ Generic, Optional, cast, - TYPE_CHECKING, ) from typing_extensions import Protocol from .configuration import Configuration @@ -50,17 +49,8 @@ ) -if TYPE_CHECKING: # Protocol and non-Protocol can't mix in Python 3.7 - - class _AsyncContextManagerCloseable(AsyncContextManager, Protocol): - """Defines a context manager that is closeable at the same time.""" - - async def close(self): - ... - - HTTPRequestType = TypeVar("HTTPRequestType") -AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType", bound="_AsyncContextManagerCloseable") +AsyncHTTPResponseType = TypeVar("AsyncHTTPResponseType", bound="AsyncContextManager") _LOGGER = logging.getLogger(__name__) @@ -79,11 +69,9 @@ class _Coroutine(Awaitable[AsyncHTTPResponseType]): This allows the dev to either use the "async with" syntax, or simply the object directly. It's also why "send_request" is not declared as async, since it couldn't be both easily. - "wrapped" must be an awaitable that returns an object that: - - has an async "close()" - - has an "__aexit__" method (IOW, is an async context manager) + "wrapped" must be an awaitable object that returns an object implements the async context manager protocol. - This permits this code to work for both requests. + This permits this code to work for both following requests. ```python from azure.core import AsyncPipelineClient @@ -122,9 +110,6 @@ async def __aenter__(self) -> AsyncHTTPResponseType: async def __aexit__(self, *args) -> None: await self._response.__aexit__(*args) - async def close(self) -> None: - await self._response.close() - class AsyncPipelineClient( PipelineClientBase, diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 8df0b1e8510c3..1ebf2c964b828 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -23,7 +23,7 @@ # IN THE SOFTWARE. # # -------------------------------------------------------------------------- -from typing import TYPE_CHECKING, TypeVar, cast, Union +from typing import TypeVar, cast, Union from ..exceptions import HttpResponseError from .base_polling import ( _failed, @@ -39,7 +39,6 @@ from ..pipeline import PipelineResponse from ..pipeline.transport import HttpRequest as LegacyHttpRequest, AsyncHttpTransport from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse -from .._pipeline_client_async import _AsyncContextManagerCloseable from ..rest import HttpRequest from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse From 4f94c883c207b5ec94cbaa3c9dc904fbdbee9dac Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 19 Jun 2023 13:24:52 -0700 Subject: [PATCH 21/46] Async Cxt Manager --- .../azure/core/pipeline/transport/_base_async.py | 5 ++++- .../azure/core/polling/async_base_polling.py | 10 +++++----- sdk/core/azure-core/azure/core/polling/base_polling.py | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_base_async.py b/sdk/core/azure-core/azure/core/pipeline/transport/_base_async.py index 543e5977020f0..fa66e7e76f8a5 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_base_async.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_base_async.py @@ -55,7 +55,7 @@ def _iterate_response_content(iterator): raise _ResponseStopIteration() -class AsyncHttpResponse(_HttpResponseBase): # pylint: disable=abstract-method +class AsyncHttpResponse(_HttpResponseBase, AbstractAsyncContextManager): # pylint: disable=abstract-method """An AsyncHttpResponse ABC. Allows for the asynchronous streaming of data from the response. @@ -85,6 +85,9 @@ def parts(self) -> AsyncIterator: return _PartGenerator(self, default_http_response_type=AsyncHttpClientTransportResponse) + async def __aexit__(self, exc_type, exc_value, traceback): + return None + class AsyncHttpClientTransportResponse( # pylint: disable=abstract-method _HttpClientTransportResponse, AsyncHttpResponse diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 1ebf2c964b828..a185cb7b909f8 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -37,13 +37,13 @@ from ..pipeline._tools import is_rest from .. import AsyncPipelineClient from ..pipeline import PipelineResponse -from ..pipeline.transport import HttpRequest as LegacyHttpRequest, AsyncHttpTransport +from ..pipeline.transport import HttpRequest as LegacyHttpRequest, AsyncHttpTransport, AsyncHttpResponse as LegacyAsyncHttpResponse from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse -from ..rest import HttpRequest +from ..rest import HttpRequest, AsyncHttpResponse from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] -AsyncHTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] +AsyncHTTPResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] PollingReturnType = TypeVar("PollingReturnType") @@ -52,7 +52,7 @@ class AsyncLROBasePolling( - _SansIOLROBasePolling[PollingReturnType, "AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]"], + _SansIOLROBasePolling[PollingReturnType, AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]], AsyncPollingMethod[PollingReturnType], ): """A base LRO async poller. @@ -72,7 +72,7 @@ class AsyncLROBasePolling( """Store the latest received HTTP response, initialized by the first answer.""" @property - def _transport(self) -> "AsyncHttpTransport": + def _transport(self) -> AsyncHttpTransport[HTTPRequestType, AsyncHTTPResponseType]: return self._client._pipeline._transport # pylint: disable=protected-access async def run(self) -> None: diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index b1667357ebc7d..a56c0c2bb442f 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -171,7 +171,7 @@ def get_status(self, pipeline_response: "PipelineResponseType") -> str: def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Optional[str]: """If a final GET is needed, returns the URL. - :rtype: str + :rtype: str or None """ raise NotImplementedError() From ca61aa0ed8276b2e85f145fd010ecb23f0c437c6 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 19 Jun 2023 17:54:45 -0700 Subject: [PATCH 22/46] Final Typing? --- .../azure/core/pipeline/__init__.py | 4 +-- .../azure/core/polling/async_base_polling.py | 22 ++++++++++----- .../azure/core/polling/base_polling.py | 27 ++++++++++++------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/__init__.py b/sdk/core/azure-core/azure/core/pipeline/__init__.py index fcadb2b3518da..646ceb3b05784 100644 --- a/sdk/core/azure-core/azure/core/pipeline/__init__.py +++ b/sdk/core/azure-core/azure/core/pipeline/__init__.py @@ -26,8 +26,8 @@ from typing import TypeVar, Generic, Dict, Any -HTTPResponseType = TypeVar("HTTPResponseType") -HTTPRequestType = TypeVar("HTTPRequestType") +HTTPResponseType = TypeVar("HTTPResponseType", covariant=True) +HTTPRequestType = TypeVar("HTTPRequestType", covariant=True) class PipelineContext(Dict[str, Any]): diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index a185cb7b909f8..c8eefe64647a0 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -37,14 +37,21 @@ from ..pipeline._tools import is_rest from .. import AsyncPipelineClient from ..pipeline import PipelineResponse -from ..pipeline.transport import HttpRequest as LegacyHttpRequest, AsyncHttpTransport, AsyncHttpResponse as LegacyAsyncHttpResponse +from ..pipeline.transport import ( + HttpRequest as LegacyHttpRequest, + AsyncHttpTransport, + AsyncHttpResponse as LegacyAsyncHttpResponse, +) from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse from ..rest import HttpRequest, AsyncHttpResponse from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] AsyncHTTPResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] -AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] +LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyAsyncHttpResponse] +NewPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] +AsyncPipelineResponseType = Union[LegacyPipelineResponseType, NewPipelineResponseType] + PollingReturnType = TypeVar("PollingReturnType") @@ -154,16 +161,19 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - AsyncPipelineResponseType, + LegacyPipelineResponseType, await self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) # Legacy HttpRequest and AsyncHttpResponse from azure.core.pipeline.transport - # "Type ignore"-ing things here, as we don't want the typing system to know + # casting things here, as we don't want the typing system to know # about the legacy APIs. request = self._client.get(status_link) - return await self._client._pipeline.run( # pylint: disable=protected-access - request, stream=False, **self._operation_config + return cast( + NewPipelineResponseType, + await self._client._pipeline.run( # pylint: disable=protected-access + request, stream=False, **self._operation_config + ), ) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index a56c0c2bb442f..0e32c03503ab9 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -44,7 +44,9 @@ HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] HTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] -PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] +LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacySansIOHttpResponse] +NewPipelineResponseType = PipelineResponse[HttpRequest, SansIOHttpResponse] +PipelineResponseType = Union[LegacyPipelineResponseType, NewPipelineResponseType] ABC = abc.ABC @@ -56,7 +58,6 @@ _SUCCEEDED = frozenset(["succeeded"]) - def _get_content(response: HTTPResponseType) -> bytes: """Get the content of this response. This is designed specifically to avoid a warning of mypy for body() access, as this method is deprecated. @@ -376,7 +377,9 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None -class _SansIOLROBasePolling(Generic[PollingReturnType, PipelineClientType]): # pylint: disable=too-many-instance-attributes +class _SansIOLROBasePolling( + Generic[PollingReturnType, PipelineClientType] +): # pylint: disable=too-many-instance-attributes """A base class that has no opinion on IO, to help mypy be accurate.""" _deserialization_callback: Callable[[Any], PollingReturnType] @@ -422,7 +425,9 @@ def initialize( :raises: HttpResponseError if initial status is incorrect LRO state """ self._client = client - self._pipeline_response = self._initial_response = initial_response # pylint: disable=attribute-defined-outside-init + self._pipeline_response = ( + self._initial_response + ) = initial_response # pylint: disable=attribute-defined-outside-init self._deserialization_callback = deserialization_callback for operation in self._lro_algorithms: @@ -516,7 +521,8 @@ def _extract_delay(self) -> float: class LROBasePolling( - _SansIOLROBasePolling[PollingReturnType, "PipelineClient[HTTPRequestType, HTTPResponseType]"], PollingMethod[PollingReturnType] + _SansIOLROBasePolling[PollingReturnType, "PipelineClient[HTTPRequestType, HTTPResponseType]"], + PollingMethod[PollingReturnType], ): # pylint: disable=too-many-instance-attributes """A base LRO poller. @@ -616,16 +622,19 @@ def request_status(self, status_link: str) -> "PipelineResponseType": # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - "PipelineResponseType", + PipelineResponseType, self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) # Legacy HttpRequest and HttpResponse from azure.core.pipeline.transport - # "Type ignore"-ing things here, as we don't want the typing system to know + # casting things here, as we don't want the typing system to know # about the legacy APIs. request = self._client.get(status_link) - return self._client._pipeline.run( # pylint: disable=protected-access - request, stream=False, **self._operation_config + return cast( + LegacyPipelineResponseType, + self._client._pipeline.run( # pylint: disable=protected-access + request, stream=False, **self._operation_config + ), ) From c82c94674265300050def88fc490e124bf9efc2e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 19 Jun 2023 18:05:38 -0700 Subject: [PATCH 23/46] More covariant --- sdk/core/azure-core/azure/core/polling/_async_poller.py | 2 +- sdk/core/azure-core/azure/core/polling/_poller.py | 2 +- sdk/core/azure-core/azure/core/polling/async_base_polling.py | 2 +- sdk/core/azure-core/azure/core/polling/base_polling.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index d4469ffb0afd0..dfec7574627f1 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -31,7 +31,7 @@ from ._poller import NoPolling as _NoPolling -PollingReturnType = TypeVar("PollingReturnType") +PollingReturnType = TypeVar("PollingReturnType", covariant=True) _LOGGER = logging.getLogger(__name__) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 52f93cdac6fd6..99f8beac3ffcf 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -33,7 +33,7 @@ from azure.core.tracing.common import with_current_context -PollingReturnType = TypeVar("PollingReturnType") +PollingReturnType = TypeVar("PollingReturnType", covariant=True) _LOGGER = logging.getLogger(__name__) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index c8eefe64647a0..80e9262b973e8 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -53,7 +53,7 @@ AsyncPipelineResponseType = Union[LegacyPipelineResponseType, NewPipelineResponseType] -PollingReturnType = TypeVar("PollingReturnType") +PollingReturnType = TypeVar("PollingReturnType", covariant=True) __all__ = ["AsyncLROBasePolling"] diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 0e32c03503ab9..214d33ee16a44 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -50,7 +50,7 @@ ABC = abc.ABC -PollingReturnType = TypeVar("PollingReturnType") +PollingReturnType = TypeVar("PollingReturnType", covariant=True) PipelineClientType = TypeVar("PipelineClientType") _FINISHED = frozenset(["succeeded", "canceled", "failed"]) From 4dd11c7070f54568afb11f138e9a93f2deda0e5a Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 19 Jun 2023 19:19:18 -0700 Subject: [PATCH 24/46] Upside down --- sdk/core/azure-core/azure/core/polling/async_base_polling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 80e9262b973e8..4295ebf5dd610 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -161,7 +161,7 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - LegacyPipelineResponseType, + NewPipelineResponseType, await self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) @@ -170,7 +170,7 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: # about the legacy APIs. request = self._client.get(status_link) return cast( - NewPipelineResponseType, + LegacyPipelineResponseType, await self._client._pipeline.run( # pylint: disable=protected-access request, stream=False, **self._operation_config ), From a049070b2f376514402f1056f26717d47bf86078 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 20 Jun 2023 10:38:02 -0700 Subject: [PATCH 25/46] Fix tests --- .../tests/async_tests/test_base_polling_async.py | 7 +++++-- sdk/core/azure-core/tests/test_base_polling.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py index e737b8804f110..14243d0cbf74a 100644 --- a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py @@ -293,7 +293,11 @@ def mock_send(http_request, http_response, method, status, headers=None, body=RE headers = {} response = Response() response._content_consumed = True - response._content = json.dumps(body).encode("ascii") if body is not None else None + # "requests" never returns None for content. Make sure it's empty bytes at worst + # In [4]: r=requests.get("https://httpbin.org/status/200") + # In [5]: r.content + # Out[5]: b'' + response._content = json.dumps(body).encode("ascii") if body is not None else b"" response.request = Request() response.request.method = method response.request.url = RESOURCE_URL @@ -366,7 +370,6 @@ def mock_update(http_request, http_response, url, headers=None): response = create_transport_response(http_response, request, response) if is_rest(http_response): response.body() - return PipelineResponse(request, response, None) # context @staticmethod diff --git a/sdk/core/azure-core/tests/test_base_polling.py b/sdk/core/azure-core/tests/test_base_polling.py index b470f088ee757..0d855025c1e9e 100644 --- a/sdk/core/azure-core/tests/test_base_polling.py +++ b/sdk/core/azure-core/tests/test_base_polling.py @@ -315,7 +315,11 @@ def mock_send(http_request, http_response, method, status, headers=None, body=RE headers = {} response = Response() response._content_consumed = True - response._content = json.dumps(body).encode("ascii") if body is not None else None + # "requests" never returns None for content. Make sure it's empty bytes at worst + # In [4]: r=requests.get("https://httpbin.org/status/200") + # In [5]: r.content + # Out[5]: b'' + response._content = json.dumps(body).encode("ascii") if body is not None else b"" response.request = Request() response.request.method = method response.request.url = RESOURCE_URL @@ -347,6 +351,8 @@ def mock_send(http_request, http_response, method, status, headers=None, body=RE request, response, ) + if is_rest(response): + response._body() return PipelineResponse(request, response, None) # context @staticmethod @@ -390,6 +396,9 @@ def mock_update(http_request, http_response, url, headers=None): request, response, ) + # Make sure body is loaded if this is "rest" + if is_rest(response): + response._body() return PipelineResponse(request, response, None) # context @staticmethod From bf3aa737905467be7759b49ee07a0b11bd304614 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 20 Jun 2023 14:55:52 -0700 Subject: [PATCH 26/46] Messed up merge --- sdk/core/azure-core/azure/core/polling/_async_poller.py | 5 ----- sdk/core/azure-core/azure/core/polling/_poller.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index f747f0db1d1e5..c92df23a9c53e 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -114,13 +114,8 @@ def __init__( self, client: Any, initial_response: Any, -<<<<<<< HEAD deserialization_callback: Callable[[Any], PollingReturnType_co], polling_method: AsyncPollingMethod[PollingReturnType_co], -======= - deserialization_callback: Callable, - polling_method: AsyncPollingMethod[PollingReturnType_co], ->>>>>>> origin/main ): self._polling_method = polling_method self._done = False diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 397c810f8e503..036da46d6735c 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -134,13 +134,8 @@ def __init__( self, client: Any, initial_response: Any, -<<<<<<< HEAD deserialization_callback: Callable[[Any], PollingReturnType_co], polling_method: PollingMethod[PollingReturnType_co], -======= - deserialization_callback: Callable, - polling_method: PollingMethod[PollingReturnType_co], ->>>>>>> origin/main ) -> None: self._callbacks: List[Callable] = [] self._polling_method = polling_method From 7644c15570bd7b768a0368c8a7a945a90b0a5b40 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Tue, 20 Jun 2023 16:05:46 -0700 Subject: [PATCH 27/46] Pylint --- .../azure/core/_pipeline_client_async.py | 1 - .../core/pipeline/policies/_universal.py | 1 - .../azure/core/polling/async_base_polling.py | 2 - .../azure/core/polling/base_polling.py | 44 +++++++++---------- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/sdk/core/azure-core/azure/core/_pipeline_client_async.py b/sdk/core/azure-core/azure/core/_pipeline_client_async.py index 7945a32353089..493c8787ee0a5 100644 --- a/sdk/core/azure-core/azure/core/_pipeline_client_async.py +++ b/sdk/core/azure-core/azure/core/_pipeline_client_async.py @@ -36,7 +36,6 @@ Optional, cast, ) -from typing_extensions import Protocol from .configuration import Configuration from .pipeline import AsyncPipeline from .pipeline.transport._base import PipelineClientBase diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py index 00894ceedc67c..82bb101f99922 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_universal.py @@ -37,7 +37,6 @@ import uuid from typing import IO, cast, Union, Optional, AnyStr, Dict, Any, Set, Mapping import urllib.parse -from typing_extensions import Protocol, runtime_checkable from azure.core import __version__ as azcore_version from azure.core.exceptions import DecodeError, raise_with_traceback diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 7b49de430a490..f86ed06ab8b0d 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -42,9 +42,7 @@ AsyncHttpTransport, AsyncHttpResponse as LegacyAsyncHttpResponse, ) -from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse from ..rest import HttpRequest, AsyncHttpResponse -from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] AsyncHTTPResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index ef071ab74c280..1f8452f5e2709 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -146,7 +146,7 @@ class LongRunningOperation(ABC): """ @abc.abstractmethod - def can_poll(self, pipeline_response: "PipelineResponseType") -> bool: + def can_poll(self, pipeline_response: PipelineResponseType) -> bool: """Answer if this polling method could be used.""" raise NotImplementedError() @@ -156,7 +156,7 @@ def get_polling_url(self) -> str: raise NotImplementedError() @abc.abstractmethod - def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: + def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """Process first response after initiating long running operation. :param azure.core.pipeline.PipelineResponse response: initial REST call response. @@ -164,12 +164,12 @@ def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: raise NotImplementedError() @abc.abstractmethod - def get_status(self, pipeline_response: "PipelineResponseType") -> str: + def get_status(self, pipeline_response: PipelineResponseType) -> str: """Return the status string extracted from this response.""" raise NotImplementedError() @abc.abstractmethod - def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str or None @@ -215,7 +215,7 @@ def __init__( self._location_url = None self._lro_options = lro_options or {} - def can_poll(self, pipeline_response: "PipelineResponseType"): + def can_poll(self, pipeline_response: PipelineResponseType): """Answer if this polling method could be used.""" response = pipeline_response.http_response return self._operation_location_header in response.headers @@ -224,7 +224,7 @@ def get_polling_url(self) -> str: """Return the polling URL.""" return self._async_url - def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str @@ -259,7 +259,7 @@ def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Option return None - def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: + def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """Process first response after initiating long running operation. :param azure.core.pipeline.PipelineResponse response: initial REST call response. @@ -280,7 +280,7 @@ def _set_async_url_if_present(self, response: HTTPResponseType) -> None: if location_url: self._location_url = location_url - def get_status(self, pipeline_response: "PipelineResponseType") -> str: + def get_status(self, pipeline_response: PipelineResponseType) -> str: """Process the latest status update retrieved from an "Operation-Location" header. :param azure.core.pipeline.PipelineResponse response: The response to extract the status. @@ -303,7 +303,7 @@ class LocationPolling(LongRunningOperation): _location_url: str """Location header""" - def can_poll(self, pipeline_response: "PipelineResponseType") -> bool: + def can_poll(self, pipeline_response: PipelineResponseType) -> bool: """Answer if this polling method could be used.""" response = pipeline_response.http_response return "location" in response.headers @@ -312,14 +312,14 @@ def get_polling_url(self) -> str: """Return the polling URL.""" return self._location_url - def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str """ return None - def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: + def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """Process first response after initiating long running operation. :param azure.core.pipeline.PipelineResponse response: initial REST call response. @@ -332,7 +332,7 @@ def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: return "InProgress" raise OperationFailed("Operation failed or canceled") - def get_status(self, pipeline_response: "PipelineResponseType") -> str: + def get_status(self, pipeline_response: PipelineResponseType) -> str: """Process the latest status update retrieved from a 'location' header. :param azure.core.pipeline.PipelineResponse response: latest REST call response. @@ -350,7 +350,7 @@ class StatusCheckPolling(LongRunningOperation): if not other polling are detected and status code is 2xx. """ - def can_poll(self, pipeline_response: "PipelineResponseType") -> bool: + def can_poll(self, pipeline_response: PipelineResponseType) -> bool: """Answer if this polling method could be used.""" return True @@ -358,7 +358,7 @@ def get_polling_url(self) -> str: """Return the polling URL.""" raise ValueError("This polling doesn't support polling") - def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: + def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """Process first response after initiating long running operation and set self.status attribute. @@ -366,10 +366,10 @@ def set_initial_status(self, pipeline_response: "PipelineResponseType") -> str: """ return "Succeeded" - def get_status(self, pipeline_response: "PipelineResponseType") -> str: + def get_status(self, pipeline_response: PipelineResponseType) -> str: return "Succeeded" - def get_final_get_url(self, pipeline_response: "PipelineResponseType") -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str @@ -494,7 +494,7 @@ def resource(self) -> PollingReturnType_co: """Return the built resource.""" return self._parse_resource(self._pipeline_response) - def _parse_resource(self, pipeline_response: "PipelineResponseType") -> PollingReturnType_co: + def _parse_resource(self, pipeline_response: PipelineResponseType) -> PollingReturnType_co: """Assuming this response is a resource, use the deserialization callback to parse it. If body is empty, assuming no resource to return. """ @@ -521,7 +521,7 @@ def _extract_delay(self) -> float: class LROBasePolling( - _SansIOLROBasePolling[PollingReturnType_co, "PipelineClient[HTTPRequestType, HTTPResponseType]"], + _SansIOLROBasePolling[PollingReturnType_co, PipelineClient[HTTPRequestType, HTTPResponseType]], PollingMethod[PollingReturnType_co], ): # pylint: disable=too-many-instance-attributes """A base LRO poller. @@ -534,14 +534,14 @@ class LROBasePolling( If your polling need are more specific, you could implement a PollingMethod directly """ - _initial_response: "PipelineResponseType" + _initial_response: PipelineResponseType """Store the initial response.""" - _pipeline_response: "PipelineResponseType" + _pipeline_response: PipelineResponseType """Store the latest received HTTP response, initialized by the first answer.""" @property - def _transport(self) -> "HttpTransport[HTTPRequestType, HTTPResponseType]": + def _transport(self) -> HttpTransport[HTTPRequestType, HTTPResponseType]: return self._client._pipeline._transport # pylint: disable=protected-access def run(self) -> None: @@ -603,7 +603,7 @@ def update_status(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - def request_status(self, status_link: str) -> "PipelineResponseType": + def request_status(self, status_link: str) -> PipelineResponseType: """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. From d229aae31011abb3a0122a67e008c5ffd7d7e363 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 21 Jun 2023 17:39:04 -0700 Subject: [PATCH 28/46] Better Typing --- .../azure-core/azure/core/polling/_poller.py | 5 +- .../azure/core/polling/async_base_polling.py | 7 +- .../azure/core/polling/base_polling.py | 73 ++++++++++++++----- .../azure/ai/translation/document/_polling.py | 2 +- 4 files changed, 63 insertions(+), 24 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 036da46d6735c..21f174c870547 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -74,7 +74,10 @@ def __init__(self): self._initial_response = None def initialize( - self, _: Any, initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType_co] + self, + _: Any, + initial_response: Any, + deserialization_callback: Callable[[Any], PollingReturnType_co], ) -> None: self._initial_response = initial_response self._deserialization_callback = deserialization_callback diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index f86ed06ab8b0d..8c5f68b5a5ba9 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -48,7 +48,7 @@ AsyncHTTPResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyAsyncHttpResponse] NewPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] -AsyncPipelineResponseType = Union[LegacyPipelineResponseType, NewPipelineResponseType] +AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] PollingReturnType_co = TypeVar("PollingReturnType_co", covariant=True) @@ -57,7 +57,10 @@ class AsyncLROBasePolling( - _SansIOLROBasePolling[PollingReturnType_co, AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType]], + _SansIOLROBasePolling[ + PollingReturnType_co, + AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType], + ], AsyncPollingMethod[PollingReturnType_co], ): """A base LRO async poller. diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 1f8452f5e2709..52596a6553114 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -27,7 +27,20 @@ import base64 import json from enum import Enum -from typing import Optional, Any, Tuple, Callable, Dict, List, Generic, TypeVar, cast, Union +from typing import ( + Optional, + Any, + Tuple, + Callable, + Dict, + List, + Generic, + TypeVar, + cast, + Union, + Protocol, + runtime_checkable, +) from ..exceptions import HttpResponseError, DecodeError from . import PollingMethod @@ -36,36 +49,44 @@ from .._enum_meta import CaseInsensitiveEnumMeta from .. import PipelineClient from ..pipeline import PipelineResponse -from ..pipeline.transport import HttpRequest as LegacyHttpRequest, HttpTransport -from ..pipeline.transport._base import _HttpResponseBase as LegacySansIOHttpResponse -from ..rest import HttpRequest -from ..rest._rest_py3 import _HttpResponseBase as SansIOHttpResponse +from ..pipeline.transport import ( + HttpTransport, + HttpRequest as LegacyHttpRequest, + HttpResponse as LegacyHttpResponse, + AsyncHttpResponse as LegacyAsyncHttpResponse, +) +from ..rest import HttpRequest, HttpResponse, AsyncHttpResponse HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] -HTTPResponseType = Union[LegacySansIOHttpResponse, SansIOHttpResponse] -LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacySansIOHttpResponse] -NewPipelineResponseType = PipelineResponse[HttpRequest, SansIOHttpResponse] -PipelineResponseType = Union[LegacyPipelineResponseType, NewPipelineResponseType] +HTTPResponseType = Union[LegacyHttpResponse, HttpResponse] +LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyHttpResponse] +NewPipelineResponseType = PipelineResponse[HttpRequest, HttpResponse] +PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] +# Some tool functions here works for all HttpResponse, and need to be typed as such to be used in all context +AllHTTPResponseType = Union[LegacyHttpResponse, HttpResponse, LegacyAsyncHttpResponse, AsyncHttpResponse] ABC = abc.ABC PollingReturnType_co = TypeVar("PollingReturnType_co", covariant=True) PipelineClientType = TypeVar("PipelineClientType") +HTTPResponseType_co = TypeVar("HTTPResponseType_co", covariant=True) +HTTPRequestType_co = TypeVar("HTTPRequestType_co", covariant=True) + _FINISHED = frozenset(["succeeded", "canceled", "failed"]) _FAILED = frozenset(["canceled", "failed"]) _SUCCEEDED = frozenset(["succeeded"]) -def _get_content(response: HTTPResponseType) -> bytes: +def _get_content(response: AllHTTPResponseType) -> bytes: """Get the content of this response. This is designed specifically to avoid a warning of mypy for body() access, as this method is deprecated. :param response: The response object. :rtype: bytes """ - if isinstance(response, LegacySansIOHttpResponse): + if isinstance(response, (LegacyHttpResponse, LegacyAsyncHttpResponse)): return response.body() return response.content @@ -100,7 +121,7 @@ class OperationFailed(Exception): pass -def _as_json(response: HTTPResponseType) -> Dict[str, Any]: +def _as_json(response: AllHTTPResponseType) -> Dict[str, Any]: """Assuming this is not empty, return the content as JSON. Result/exceptions is not determined if you call this method without testing _is_empty. @@ -113,7 +134,7 @@ def _as_json(response: HTTPResponseType) -> Dict[str, Any]: raise DecodeError("Error occurred in deserializing the response body.") -def _raise_if_bad_http_status_and_method(response: HTTPResponseType) -> None: +def _raise_if_bad_http_status_and_method(response: AllHTTPResponseType) -> None: """Check response status code is valid. Must be 200, 201, 202, or 204. @@ -126,7 +147,7 @@ def _raise_if_bad_http_status_and_method(response: HTTPResponseType) -> None: raise BadStatus("Invalid return status {!r} for {!r} operation".format(code, response.request.method)) -def _is_empty(response: HTTPResponseType) -> bool: +def _is_empty(response: AllHTTPResponseType) -> bool: """Check if response body contains meaningful content. :rtype: bool @@ -134,7 +155,7 @@ def _is_empty(response: HTTPResponseType) -> bool: return not bool(_get_content(response)) -class LongRunningOperation(ABC): +class LongRunningOperation(ABC, Generic[HTTPRequestType_co, HTTPResponseType_co]): """LongRunningOperation Provides default logic for interpreting operation responses and status updates. @@ -146,7 +167,10 @@ class LongRunningOperation(ABC): """ @abc.abstractmethod - def can_poll(self, pipeline_response: PipelineResponseType) -> bool: + def can_poll( + self, + pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], + ) -> bool: """Answer if this polling method could be used.""" raise NotImplementedError() @@ -156,7 +180,10 @@ def get_polling_url(self) -> str: raise NotImplementedError() @abc.abstractmethod - def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: + def set_initial_status( + self, + pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], + ) -> str: """Process first response after initiating long running operation. :param azure.core.pipeline.PipelineResponse response: initial REST call response. @@ -164,12 +191,18 @@ def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: raise NotImplementedError() @abc.abstractmethod - def get_status(self, pipeline_response: PipelineResponseType) -> str: + def get_status( + self, + pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], + ) -> str: """Return the status string extracted from this response.""" raise NotImplementedError() @abc.abstractmethod - def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: + def get_final_get_url( + self, + pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], + ) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str or None @@ -191,7 +224,7 @@ class _FinalStateViaOption(str, Enum, metaclass=CaseInsensitiveEnumMeta): OPERATION_LOCATION_FINAL_STATE = "operation-location" -class OperationResourcePolling(LongRunningOperation): +class OperationResourcePolling(LongRunningOperation[HTTPRequestType, HTTPResponseType]): """Implements a operation resource polling, typically from Operation-Location. :param str operation_location_header: Name of the header to return operation format (default 'operation-location') diff --git a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py index e9ae5b86c2d36..6d5785c95027f 100644 --- a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py +++ b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py @@ -17,7 +17,7 @@ from azure.core.exceptions import HttpResponseError, ODataV4Format from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import ( +from azure.core.rest import ( HttpResponse, AsyncHttpResponse, HttpRequest, From 7a8c137fd01fdd3457807abc48b20f59e64efa06 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 22 Jun 2023 13:22:51 -0700 Subject: [PATCH 29/46] Final typing? --- .../azure/core/polling/async_base_polling.py | 27 ++++++------ .../azure/core/polling/base_polling.py | 42 ++++++++++--------- .../azure/ai/translation/document/_polling.py | 2 +- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 8c5f68b5a5ba9..83d5987e493c7 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -44,11 +44,14 @@ ) from ..rest import HttpRequest, AsyncHttpResponse -HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] -AsyncHTTPResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] +HttpRequestType = Union[LegacyHttpRequest, HttpRequest] +AsyncHttpResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyAsyncHttpResponse] NewPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] -AsyncPipelineResponseType = PipelineResponse[HTTPRequestType, AsyncHTTPResponseType] +AsyncPipelineResponseType = PipelineResponse[HttpRequestType, AsyncHttpResponseType] +HttpRequestTypeVar = TypeVar("HttpRequestTypeVar", bound=HttpRequestType) +AsyncHttpResponseTypeVar = TypeVar("AsyncHttpResponseTypeVar", bound=AsyncHttpResponseType) +AsyncPipelineResponseTypeVar = PipelineResponse[HttpRequestTypeVar, AsyncHttpResponseTypeVar] PollingReturnType_co = TypeVar("PollingReturnType_co", covariant=True) @@ -59,7 +62,7 @@ class AsyncLROBasePolling( _SansIOLROBasePolling[ PollingReturnType_co, - AsyncPipelineClient[HTTPRequestType, AsyncHTTPResponseType], + AsyncPipelineClient[HttpRequestTypeVar, AsyncHttpResponseTypeVar], ], AsyncPollingMethod[PollingReturnType_co], ): @@ -80,7 +83,7 @@ class AsyncLROBasePolling( """Store the latest received HTTP response, initialized by the first answer.""" @property - def _transport(self) -> AsyncHttpTransport[HTTPRequestType, AsyncHTTPResponseType]: + def _transport(self) -> AsyncHttpTransport[HttpRequestTypeVar, AsyncHttpResponseTypeVar]: return self._client._pipeline._transport # pylint: disable=protected-access async def run(self) -> None: @@ -142,7 +145,7 @@ async def update_status(self): _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - async def request_status(self, status_link: str) -> AsyncPipelineResponseType: + async def request_status(self, status_link: str) -> AsyncPipelineResponseTypeVar: """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -154,24 +157,22 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseType: # Re-inject 'x-ms-client-request-id' while polling if "request_id" not in self._operation_config: self._operation_config["request_id"] = self._get_request_id() - if is_rest(self._initial_response.http_response): - # if I am a azure.core.rest.HttpResponse - # want to keep making azure.core.rest calls - rest_request = HttpRequest("GET", status_link) + if is_rest(self._initial_response.http_response): + rest_request = cast(HttpRequestTypeVar, HttpRequest("GET", status_link)) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - NewPipelineResponseType, + AsyncPipelineResponseTypeVar, await self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) # Legacy HttpRequest and AsyncHttpResponse from azure.core.pipeline.transport # casting things here, as we don't want the typing system to know # about the legacy APIs. - request = self._client.get(status_link) + request = cast(HttpRequestTypeVar, self._client.get(status_link)) return cast( - LegacyPipelineResponseType, + AsyncPipelineResponseTypeVar, await self._client._pipeline.run( # pylint: disable=protected-access request, stream=False, **self._operation_config ), diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 52596a6553114..683373c26c37c 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -58,11 +58,14 @@ from ..rest import HttpRequest, HttpResponse, AsyncHttpResponse -HTTPRequestType = Union[LegacyHttpRequest, HttpRequest] -HTTPResponseType = Union[LegacyHttpResponse, HttpResponse] +HttpRequestType = Union[LegacyHttpRequest, HttpRequest] +HttpResponseType = Union[LegacyHttpResponse, HttpResponse] LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyHttpResponse] NewPipelineResponseType = PipelineResponse[HttpRequest, HttpResponse] -PipelineResponseType = PipelineResponse[HTTPRequestType, HTTPResponseType] +PipelineResponseType = PipelineResponse[HttpRequestType, HttpResponseType] +HttpRequestTypeVar = TypeVar("HttpRequestTypeVar", bound=HttpRequestType) +HttpResponseTypeVar = TypeVar("HttpResponseTypeVar", bound=HttpResponseType) +PipelineResponseTypeVar = PipelineResponse[HttpRequestTypeVar, HttpResponseTypeVar] # Some tool functions here works for all HttpResponse, and need to be typed as such to be used in all context AllHTTPResponseType = Union[LegacyHttpResponse, HttpResponse, LegacyAsyncHttpResponse, AsyncHttpResponse] @@ -224,7 +227,7 @@ class _FinalStateViaOption(str, Enum, metaclass=CaseInsensitiveEnumMeta): OPERATION_LOCATION_FINAL_STATE = "operation-location" -class OperationResourcePolling(LongRunningOperation[HTTPRequestType, HTTPResponseType]): +class OperationResourcePolling(LongRunningOperation[HttpRequestTypeVar, HttpResponseTypeVar]): """Implements a operation resource polling, typically from Operation-Location. :param str operation_location_header: Name of the header to return operation format (default 'operation-location') @@ -248,7 +251,7 @@ def __init__( self._location_url = None self._lro_options = lro_options or {} - def can_poll(self, pipeline_response: PipelineResponseType): + def can_poll(self, pipeline_response: PipelineResponseTypeVar): """Answer if this polling method could be used.""" response = pipeline_response.http_response return self._operation_location_header in response.headers @@ -257,7 +260,7 @@ def get_polling_url(self) -> str: """Return the polling URL.""" return self._async_url - def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseTypeVar) -> Optional[str]: """If a final GET is needed, returns the URL. :rtype: str @@ -292,7 +295,7 @@ def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional return None - def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: + def set_initial_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process first response after initiating long running operation. :param azure.core.pipeline.PipelineResponse response: initial REST call response. @@ -306,14 +309,14 @@ def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: return "InProgress" raise OperationFailed("Operation failed or canceled") - def _set_async_url_if_present(self, response: HTTPResponseType) -> None: + def _set_async_url_if_present(self, response: HttpResponseTypeVar) -> None: self._async_url = response.headers[self._operation_location_header] location_url = response.headers.get("location") if location_url: self._location_url = location_url - def get_status(self, pipeline_response: PipelineResponseType) -> str: + def get_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process the latest status update retrieved from an "Operation-Location" header. :param azure.core.pipeline.PipelineResponse response: The response to extract the status. @@ -554,7 +557,7 @@ def _extract_delay(self) -> float: class LROBasePolling( - _SansIOLROBasePolling[PollingReturnType_co, PipelineClient[HTTPRequestType, HTTPResponseType]], + _SansIOLROBasePolling[PollingReturnType_co, PipelineClient[HttpRequestTypeVar, HttpResponseTypeVar]], PollingMethod[PollingReturnType_co], ): # pylint: disable=too-many-instance-attributes """A base LRO poller. @@ -567,14 +570,14 @@ class LROBasePolling( If your polling need are more specific, you could implement a PollingMethod directly """ - _initial_response: PipelineResponseType + _initial_response: PipelineResponseTypeVar """Store the initial response.""" - _pipeline_response: PipelineResponseType + _pipeline_response: PipelineResponseTypeVar """Store the latest received HTTP response, initialized by the first answer.""" @property - def _transport(self) -> HttpTransport[HTTPRequestType, HTTPResponseType]: + def _transport(self) -> HttpTransport[HttpRequestTypeVar, HttpResponseTypeVar]: return self._client._pipeline._transport # pylint: disable=protected-access def run(self) -> None: @@ -636,7 +639,7 @@ def update_status(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) self._status = self._operation.get_status(self._pipeline_response) - def request_status(self, status_link: str) -> PipelineResponseType: + def request_status(self, status_link: str) -> PipelineResponseTypeVar: """Do a simple GET to this status link. This method re-inject 'x-ms-client-request-id'. @@ -648,23 +651,22 @@ def request_status(self, status_link: str) -> PipelineResponseType: # Re-inject 'x-ms-client-request-id' while polling if "request_id" not in self._operation_config: self._operation_config["request_id"] = self._get_request_id() + if is_rest(self._initial_response.http_response): - # if I am a azure.core.rest.HttpResponse - # want to keep making azure.core.rest calls - rest_request = HttpRequest("GET", status_link) + rest_request = cast(HttpRequestTypeVar, HttpRequest("GET", status_link)) # Need a cast, as "_return_pipeline_response" mutate the return type, and that return type is not # declared in the typing of "send_request" return cast( - PipelineResponseType, + PipelineResponseTypeVar, self._client.send_request(rest_request, _return_pipeline_response=True, **self._operation_config), ) # Legacy HttpRequest and HttpResponse from azure.core.pipeline.transport # casting things here, as we don't want the typing system to know # about the legacy APIs. - request = self._client.get(status_link) + request = cast(HttpRequestTypeVar, self._client.get(status_link)) return cast( - LegacyPipelineResponseType, + PipelineResponseTypeVar, self._client._pipeline.run( # pylint: disable=protected-access request, stream=False, **self._operation_config ), diff --git a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py index 6d5785c95027f..9e68d5acc850b 100644 --- a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py +++ b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py @@ -165,7 +165,7 @@ def _poll(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) -class TranslationPolling(OperationResourcePolling): +class TranslationPolling(OperationResourcePolling[HttpRequest, HttpResponse]): """Implements a Location polling.""" def can_poll(self, pipeline_response: PipelineResponseType) -> bool: From 48872c988b407c55275d96fc6effe710ee202039 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 22 Jun 2023 15:18:32 -0700 Subject: [PATCH 30/46] Pylint --- .../azure/core/polling/base_polling.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 683373c26c37c..fa5e3e17b5a4e 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -38,8 +38,6 @@ TypeVar, cast, Union, - Protocol, - runtime_checkable, ) from ..exceptions import HttpResponseError, DecodeError @@ -461,9 +459,9 @@ def initialize( :raises: HttpResponseError if initial status is incorrect LRO state """ self._client = client - self._pipeline_response = ( - self._initial_response - ) = initial_response # pylint: disable=attribute-defined-outside-init + self._pipeline_response = ( # pylint: disable=attribute-defined-outside-init + self._initial_response # pylint: disable=attribute-defined-outside-init + ) = initial_response self._deserialization_callback = deserialization_callback for operation in self._lro_algorithms: @@ -539,10 +537,10 @@ def _parse_resource(self, pipeline_response: PipelineResponseType) -> PollingRet return self._deserialization_callback(pipeline_response) # This "type ignore" has been discussed with architects. - # We have a typing problem that if the Swagger/TSP describes a return type (PollingReturnType_co is not None), BUT - # the returned payload is actually empty, we don't want to fail, but return None. - # To make it clean, we would have to make the polling return type Optional - # "just in case the Swagger/TSP is wrong". This is reducing the quality and the value of the typing annotations + # We have a typing problem that if the Swagger/TSP describes a return type (PollingReturnType_co is not None), + # BUT the returned payload is actually empty, we don't want to fail, but return None. + # To be clean, we would have to make the polling return type Optional "just in case the Swagger/TSP is wrong". + # This is reducing the quality and the value of the typing annotations # for a case that is not supposed to happen in the first place. So we decided to ignore the type error here. return None # type: ignore From 36e1e788d043f7a2ca1e650d41cfc78d3ad3f763 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Thu, 22 Jun 2023 15:39:32 -0700 Subject: [PATCH 31/46] Simplify translation typing for now --- .../azure/ai/translation/document/_polling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py index 9e68d5acc850b..6d5785c95027f 100644 --- a/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py +++ b/sdk/translation/azure-ai-translation-document/azure/ai/translation/document/_polling.py @@ -165,7 +165,7 @@ def _poll(self) -> None: _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) -class TranslationPolling(OperationResourcePolling[HttpRequest, HttpResponse]): +class TranslationPolling(OperationResourcePolling): """Implements a Location polling.""" def can_poll(self, pipeline_response: PipelineResponseType) -> bool: From 33e612524b56e86ab507826f78f851ff7e797a09 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 23 Jun 2023 12:43:28 -0700 Subject: [PATCH 32/46] Fix backcompat with azure-mgmt-core --- .../azure/core/polling/async_base_polling.py | 12 ++-- .../azure/core/polling/base_polling.py | 16 ++++++ .../async_tests/test_base_polling_async.py | 57 +++++++++++++++++-- 3 files changed, 73 insertions(+), 12 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 83d5987e493c7..5b36142b02b15 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -88,7 +88,7 @@ def _transport(self) -> AsyncHttpTransport[HttpRequestTypeVar, AsyncHttpResponse async def run(self) -> None: try: - await self._poll() + await self._async_poll() except BadStatus as err: self._status = "Failed" @@ -105,7 +105,7 @@ async def run(self) -> None: except OperationFailed as err: raise HttpResponseError(response=self._pipeline_response.http_response, error=err) - async def _poll(self) -> None: + async def _async_poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. @@ -118,7 +118,7 @@ async def _poll(self) -> None: if not self.finished(): await self.update_status() while not self.finished(): - await self._delay() + await self._async_delay() await self.update_status() if _failed(self.status()): @@ -129,15 +129,15 @@ async def _poll(self) -> None: self._pipeline_response = await self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - async def _sleep(self, delay: float) -> None: + async def _async_sleep(self, delay: float) -> None: await self._transport.sleep(delay) - async def _delay(self): + async def _async_delay(self): """Check for a 'retry-after' header to set timeout, otherwise use configured timeout. """ delay = self._extract_delay() - await self._sleep(delay) + await self._async_sleep(delay) async def update_status(self): """Update the current status of the LRO.""" diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index fa5e3e17b5a4e..b7dc78768b849 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -578,6 +578,22 @@ class LROBasePolling( def _transport(self) -> HttpTransport[HttpRequestTypeVar, HttpResponseTypeVar]: return self._client._pipeline._transport # pylint: disable=protected-access + def __getattribute__(self, name: str) -> Any: + """Find the right method for the job. + + This contains a workaround for azure-mgmt-core 1.0.0 to 1.4.0, where the MRO + is changing when azure-core was refactored in 1.27.0. The MRO change was causing + AsyncARMPolling to look-up the wrong methods and find the non-async ones. + + :param str name: The name of the attribute to retrieve. + :rtype: Any + :return: The attribute value. + """ + cls = object.__getattribute__(self, "__class__") + if cls.__name__ == "AsyncARMPolling" and name in ["run", "update_status", "request_status"]: + return getattr(super(LROBasePolling, self), name) + return super().__getattribute__(name) + def run(self) -> None: try: self._poll() diff --git a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py index 14243d0cbf74a..a563d8c34071b 100644 --- a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py @@ -30,12 +30,7 @@ from utils import HTTP_REQUESTS from azure.core.pipeline._tools import is_rest import types -import unittest - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock import pytest @@ -47,6 +42,7 @@ from azure.core.pipeline import PipelineResponse, AsyncPipeline, PipelineContext from azure.core.pipeline.transport import AsyncioRequestsTransportResponse, AsyncHttpTransport +from azure.core.polling.base_polling import LROBasePolling from azure.core.polling.async_base_polling import ( AsyncLROBasePolling, ) @@ -749,3 +745,52 @@ async def test_final_get_via_location(port, http_request, deserialization_cb): ) result = await poller.result() assert result == {"returnedFrom": "locationHeaderUrl"} + + + +"""Reproduce the bad design of azure-mgmt-core 1.0.0-1.4.0""" + +class ARMPolling(LROBasePolling): + pass + +class AsyncARMPolling(ARMPolling, AsyncLROBasePolling): + pass + +@pytest.mark.asyncio +async def test_async_polling_inheritance(async_pipeline_client_builder, deserialization_cb): + rest_http = request_and_responses_product(ASYNCIO_REQUESTS_TRANSPORT_RESPONSES)[1] + async def send(request, **kwargs): + assert request.method == "GET" + + if request.url == "http://example.org/location": + return TestBasePolling.mock_send(rest_http[0], rest_http[1], "GET", 200, body={"success": True}).http_response + elif request.url == "http://example.org/async_monitor": + return TestBasePolling.mock_send( + rest_http[0], rest_http[1], "GET", 200, body={"status": "Succeeded"} + ).http_response + else: + pytest.fail("No other query allowed") + + client = async_pipeline_client_builder(send) + + initial_response = TestBasePolling.mock_send( + rest_http[0], + rest_http[1], + "POST", + 200, + { + "location": "http://example.org/location", + "operation-location": "http://example.org/async_monitor", + }, + "", + ) + + polling = AsyncARMPolling() + polling.initialize( + client=client, + initial_response=initial_response, + deserialization_callback=deserialization_cb, + ) + await polling.run() + resource = polling.resource() + assert resource["success"] From 8aea6b10dce9a44bf0ad5e5d1174f13ecd548502 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 23 Jun 2023 14:33:39 -0700 Subject: [PATCH 33/46] Revert renaming private methods --- .../azure/core/polling/async_base_polling.py | 12 ++++++------ .../azure-core/azure/core/polling/base_polling.py | 9 ++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 5b36142b02b15..83d5987e493c7 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -88,7 +88,7 @@ def _transport(self) -> AsyncHttpTransport[HttpRequestTypeVar, AsyncHttpResponse async def run(self) -> None: try: - await self._async_poll() + await self._poll() except BadStatus as err: self._status = "Failed" @@ -105,7 +105,7 @@ async def run(self) -> None: except OperationFailed as err: raise HttpResponseError(response=self._pipeline_response.http_response, error=err) - async def _async_poll(self) -> None: + async def _poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. @@ -118,7 +118,7 @@ async def _async_poll(self) -> None: if not self.finished(): await self.update_status() while not self.finished(): - await self._async_delay() + await self._delay() await self.update_status() if _failed(self.status()): @@ -129,15 +129,15 @@ async def _async_poll(self) -> None: self._pipeline_response = await self.request_status(final_get_url) _raise_if_bad_http_status_and_method(self._pipeline_response.http_response) - async def _async_sleep(self, delay: float) -> None: + async def _sleep(self, delay: float) -> None: await self._transport.sleep(delay) - async def _async_delay(self): + async def _delay(self): """Check for a 'retry-after' header to set timeout, otherwise use configured timeout. """ delay = self._extract_delay() - await self._async_sleep(delay) + await self._sleep(delay) async def update_status(self): """Update the current status of the LRO.""" diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index b7dc78768b849..b8ab470bda427 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -590,7 +590,14 @@ def __getattribute__(self, name: str) -> Any: :return: The attribute value. """ cls = object.__getattribute__(self, "__class__") - if cls.__name__ == "AsyncARMPolling" and name in ["run", "update_status", "request_status"]: + if cls.__name__ == "AsyncARMPolling" and name in [ + "run", + "update_status", + "request_status", + "_sleep", + "_delay", + "_poll", + ]: return getattr(super(LROBasePolling, self), name) return super().__getattribute__(name) From aa9c89c6334fb494f200eb38e3038585b43dee08 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 23 Jun 2023 15:20:46 -0700 Subject: [PATCH 34/46] Black --- .../tests/async_tests/test_base_polling_async.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py index a563d8c34071b..12e9ed765a215 100644 --- a/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_base_polling_async.py @@ -747,23 +747,28 @@ async def test_final_get_via_location(port, http_request, deserialization_cb): assert result == {"returnedFrom": "locationHeaderUrl"} - """Reproduce the bad design of azure-mgmt-core 1.0.0-1.4.0""" + class ARMPolling(LROBasePolling): pass + class AsyncARMPolling(ARMPolling, AsyncLROBasePolling): pass + @pytest.mark.asyncio async def test_async_polling_inheritance(async_pipeline_client_builder, deserialization_cb): rest_http = request_and_responses_product(ASYNCIO_REQUESTS_TRANSPORT_RESPONSES)[1] + async def send(request, **kwargs): assert request.method == "GET" if request.url == "http://example.org/location": - return TestBasePolling.mock_send(rest_http[0], rest_http[1], "GET", 200, body={"success": True}).http_response + return TestBasePolling.mock_send( + rest_http[0], rest_http[1], "GET", 200, body={"success": True} + ).http_response elif request.url == "http://example.org/async_monitor": return TestBasePolling.mock_send( rest_http[0], rest_http[1], "GET", 200, body={"status": "Succeeded"} From 188d7abed1d22e25c2af9c40b8747201f3b1524e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 26 Jun 2023 12:05:38 -0700 Subject: [PATCH 35/46] Feedback from @kristapratico --- .../azure-core/azure/core/polling/async_base_polling.py | 7 ++----- sdk/core/azure-core/azure/core/polling/base_polling.py | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 83d5987e493c7..472ef53b24da2 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -46,9 +46,6 @@ HttpRequestType = Union[LegacyHttpRequest, HttpRequest] AsyncHttpResponseType = Union[LegacyAsyncHttpResponse, AsyncHttpResponse] -LegacyPipelineResponseType = PipelineResponse[LegacyHttpRequest, LegacyAsyncHttpResponse] -NewPipelineResponseType = PipelineResponse[HttpRequest, AsyncHttpResponse] -AsyncPipelineResponseType = PipelineResponse[HttpRequestType, AsyncHttpResponseType] HttpRequestTypeVar = TypeVar("HttpRequestTypeVar", bound=HttpRequestType) AsyncHttpResponseTypeVar = TypeVar("AsyncHttpResponseTypeVar", bound=AsyncHttpResponseType) AsyncPipelineResponseTypeVar = PipelineResponse[HttpRequestTypeVar, AsyncHttpResponseTypeVar] @@ -76,10 +73,10 @@ class AsyncLROBasePolling( If your polling need are more specific, you could implement a PollingMethod directly """ - _initial_response: AsyncPipelineResponseType + _initial_response: AsyncPipelineResponseTypeVar """Store the initial response.""" - _pipeline_response: AsyncPipelineResponseType + _pipeline_response: AsyncPipelineResponseTypeVar """Store the latest received HTTP response, initialized by the first answer.""" @property diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index b8ab470bda427..378d3eaf58e4b 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -423,7 +423,7 @@ class _SansIOLROBasePolling( """The algorithm this poller has decided to use. Will loop through 'can_poll' of the input algorithms to decide.""" _status: str - """Hold the current of this poller""" + """Hold the current status of this poller""" _client: PipelineClientType """The Azure Core Pipeline client used to make request.""" @@ -434,7 +434,7 @@ def __init__( lro_algorithms: Optional[List[LongRunningOperation]] = None, lro_options: Optional[Dict[str, Any]] = None, path_format_arguments: Optional[Dict[str, str]] = None, - **operation_config + **operation_config: Any ): self._lro_algorithms = lro_algorithms or [ OperationResourcePolling(lro_options=lro_options), From dee0db529e21492fffbbe98922e67d172904ef72 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 26 Jun 2023 14:16:06 -0700 Subject: [PATCH 36/46] Docstrings part 1 --- .../azure/core/polling/_async_poller.py | 11 ++++++++-- .../azure-core/azure/core/polling/_poller.py | 22 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index c92df23a9c53e..b636b7187857c 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -76,7 +76,7 @@ async def async_poller( initial_response: Any, deserialization_callback: Callable[[Any], PollingReturnType_co], polling_method: AsyncPollingMethod[PollingReturnType_co], -): +) -> PollingReturnType_co: """Async Poller for long running operations. .. deprecated:: 1.5.0 @@ -91,6 +91,8 @@ async def async_poller( :type deserialization_callback: callable or msrest.serialization.Model :param polling_method: The polling strategy to adopt :type polling_method: ~azure.core.polling.PollingMethod + :returns: The final resource at the end of the polling. + :rtype: any or None """ poller = AsyncLROPoller(client, initial_response, deserialization_callback, polling_method) return await poller @@ -129,7 +131,11 @@ def __init__( self._polling_method.initialize(client, initial_response, deserialization_callback) def polling_method(self) -> AsyncPollingMethod[PollingReturnType_co]: - """Return the polling method associated to this poller.""" + """Return the polling method associated to this poller. + + :returns: The polling method associated to this poller. + :rtype: ~azure.core.polling.AsyncPollingMethod + """ return self._polling_method def continuation_token(self) -> str: @@ -163,6 +169,7 @@ async def result(self) -> PollingReturnType_co: """Return the result of the long running operation. :returns: The deserialized resource of the long running operation, if one is available. + :rtype: any or None :raises ~azure.core.exceptions.HttpResponseError: Server problem with the query. """ await self.wait() diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 21f174c870547..4115e49fe9831 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -86,9 +86,10 @@ def run(self) -> None: """Empty run, no polling.""" def status(self) -> str: - """Return the current status as a string. + """Return the current status. :rtype: str + :return: The current status """ return "succeeded" @@ -96,6 +97,7 @@ def finished(self) -> bool: """Is this polling finished? :rtype: bool + :return: Whether this polling is finished """ return True @@ -112,7 +114,9 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError("Need kwarg 'deserialization_callback' to be recreated from continuation_token") + raise ValueError( # pylint: disable=raise-missing-from + "Need kwarg 'deserialization_callback' to be recreated from continuation_token" + ) import pickle initial_response = pickle.loads(base64.b64decode(continuation_token)) # nosec @@ -169,9 +173,6 @@ def __init__( def _start(self): """Start the long running operation. On completion, runs any callbacks. - - :param callable update_cmd: The API request to check the status of - the operation. """ try: self._polling_method.run() @@ -197,7 +198,11 @@ def _start(self): callbacks, self._callbacks = self._callbacks, [] def polling_method(self) -> PollingMethod[PollingReturnType_co]: - """Return the polling method associated to this poller.""" + """Return the polling method associated to this poller. + + :returns: The polling method + :rtype: ~azure.core.polling.PollingMethod + """ return self._polling_method def continuation_token(self) -> str: @@ -231,8 +236,9 @@ def result(self, timeout: Optional[float] = None) -> PollingReturnType_co: """Return the result of the long running operation, or the result available after the specified timeout. - :returns: The deserialized resource of the long running operation, - if one is available. + :param float timeout: Period of time to wait before getting back control. + :returns: The deserialized resource of the long running operation, if one is available. + :rtype: any or None :raises ~azure.core.exceptions.HttpResponseError: Server problem with the query. """ self.wait(timeout) From 9127c89733a5f48eeb7f9971ed39a31ddf6309e9 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 26 Jun 2023 15:04:12 -0700 Subject: [PATCH 37/46] Polling pylint part 2 --- .../azure/core/polling/async_base_polling.py | 10 +- .../azure/core/polling/base_polling.py | 219 ++++++++++++++---- 2 files changed, 179 insertions(+), 50 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/async_base_polling.py b/sdk/core/azure-core/azure/core/polling/async_base_polling.py index 472ef53b24da2..2e05b52b24d0a 100644 --- a/sdk/core/azure-core/azure/core/polling/async_base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/async_base_polling.py @@ -89,7 +89,7 @@ async def run(self) -> None: except BadStatus as err: self._status = "Failed" - raise HttpResponseError(response=self._pipeline_response.http_response, error=err) + raise HttpResponseError(response=self._pipeline_response.http_response, error=err) from err except BadResponse as err: self._status = "Failed" @@ -97,17 +97,15 @@ async def run(self) -> None: response=self._pipeline_response.http_response, message=str(err), error=err, - ) + ) from err except OperationFailed as err: - raise HttpResponseError(response=self._pipeline_response.http_response, error=err) + raise HttpResponseError(response=self._pipeline_response.http_response, error=err) from err async def _poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. - :param callable update_cmd: The function to call to retrieve the - latest status of the long running operation. :raises: OperationFailed if operation status 'Failed' or 'Canceled'. :raises: BadStatus if response status invalid. :raises: BadResponse if response invalid. @@ -147,7 +145,9 @@ async def request_status(self, status_link: str) -> AsyncPipelineResponseTypeVar This method re-inject 'x-ms-client-request-id'. + :param str status_link: URL to poll. :rtype: azure.core.pipeline.PipelineResponse + :return: The response of the status request. """ if self._path_format_arguments: status_link = self._client.format_url(status_link, **self._path_format_arguments) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 378d3eaf58e4b..f9612670cdf5e 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -85,6 +85,8 @@ def _get_content(response: AllHTTPResponseType) -> bytes: a warning of mypy for body() access, as this method is deprecated. :param response: The response object. + :type response: any + :return: The content of this response. :rtype: bytes """ if isinstance(response, (LegacyHttpResponse, LegacyAsyncHttpResponse)): @@ -127,12 +129,16 @@ def _as_json(response: AllHTTPResponseType) -> Dict[str, Any]: Result/exceptions is not determined if you call this method without testing _is_empty. + :param response: The response object. + :type response: any + :return: The content of this response as dict. + :rtype: dict :raises: DecodeError if response body contains invalid json data. """ try: return json.loads(response.text()) - except ValueError: - raise DecodeError("Error occurred in deserializing the response body.") + except ValueError as err: + raise DecodeError("Error occurred in deserializing the response body.") from err def _raise_if_bad_http_status_and_method(response: AllHTTPResponseType) -> None: @@ -140,6 +146,8 @@ def _raise_if_bad_http_status_and_method(response: AllHTTPResponseType) -> None: Must be 200, 201, 202, or 204. + :param response: The response object. + :type response: any :raises: BadStatus if invalid status. """ code = response.status_code @@ -151,20 +159,16 @@ def _raise_if_bad_http_status_and_method(response: AllHTTPResponseType) -> None: def _is_empty(response: AllHTTPResponseType) -> bool: """Check if response body contains meaningful content. + :param response: The response object. + :type response: any + :return: True if response body is empty, False otherwise. :rtype: bool """ return not bool(_get_content(response)) class LongRunningOperation(ABC, Generic[HTTPRequestType_co, HTTPResponseType_co]): - """LongRunningOperation - Provides default logic for interpreting operation responses - and status updates. - - :param azure.core.pipeline.PipelineResponse response: The initial pipeline response. - :param callable deserialization_callback: The deserialization callaback. - :param dict lro_options: LRO options. - :param kwargs: Unused for now + """Protocol to implement for a long running operation algorithm. """ @abc.abstractmethod @@ -172,12 +176,22 @@ def can_poll( self, pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], ) -> bool: - """Answer if this polling method could be used.""" + """Answer if this polling method could be used. + + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: True if this polling method could be used, False otherwise. + :rtype: bool + """ raise NotImplementedError() @abc.abstractmethod def get_polling_url(self) -> str: - """Return the polling URL.""" + """Return the polling URL. + + :return: The polling URL. + :rtype: str + """ raise NotImplementedError() @abc.abstractmethod @@ -187,7 +201,10 @@ def set_initial_status( ) -> str: """Process first response after initiating long running operation. - :param azure.core.pipeline.PipelineResponse response: initial REST call response. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The initial status. + :rtype: str """ raise NotImplementedError() @@ -196,7 +213,13 @@ def get_status( self, pipeline_response: PipelineResponse[HTTPRequestType_co, HTTPResponseType_co], ) -> str: - """Return the status string extracted from this response.""" + """Return the status string extracted from this response. + + :param pipeline_response: The response object. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The status string. + :rtype: str + """ raise NotImplementedError() @abc.abstractmethod @@ -206,6 +229,9 @@ def get_final_get_url( ) -> Optional[str]: """If a final GET is needed, returns the URL. + :param pipeline_response: Success REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The URL to the final GET, or None if no final GET is needed. :rtype: str or None """ raise NotImplementedError() @@ -250,18 +276,33 @@ def __init__( self._lro_options = lro_options or {} def can_poll(self, pipeline_response: PipelineResponseTypeVar): - """Answer if this polling method could be used.""" + """Check if status monitor header (e.g. Operation-Location) is present. + + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: True if this polling method could be used, False otherwise. + :rtype: bool + """ response = pipeline_response.http_response return self._operation_location_header in response.headers def get_polling_url(self) -> str: - """Return the polling URL.""" + """Return the polling URL. + + Will extract it from the defined header to read (e.g. Operation-Location) + + :return: The polling URL. + :rtype: str + """ return self._async_url def get_final_get_url(self, pipeline_response: PipelineResponseTypeVar) -> Optional[str]: """If a final GET is needed, returns the URL. - :rtype: str + :param pipeline_response: Success REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The URL to the final GET, or None if no final GET is needed. + :rtype: str or None """ if ( self._lro_options.get(_LroOption.FINAL_STATE_VIA) == _FinalStateViaOption.LOCATION_FINAL_STATE @@ -296,7 +337,10 @@ def get_final_get_url(self, pipeline_response: PipelineResponseTypeVar) -> Optio def set_initial_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process first response after initiating long running operation. - :param azure.core.pipeline.PipelineResponse response: initial REST call response. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The initial status. + :rtype: str """ self._request = pipeline_response.http_response.request response = pipeline_response.http_response @@ -317,7 +361,10 @@ def _set_async_url_if_present(self, response: HttpResponseTypeVar) -> None: def get_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process the latest status update retrieved from an "Operation-Location" header. - :param azure.core.pipeline.PipelineResponse response: The response to extract the status. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The status string. + :rtype: str :raises: BadResponse if response has no body, or body does not contain status. """ response = pipeline_response.http_response @@ -338,25 +385,43 @@ class LocationPolling(LongRunningOperation): """Location header""" def can_poll(self, pipeline_response: PipelineResponseType) -> bool: - """Answer if this polling method could be used.""" + """True if contains a Location header + + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: True if this polling method could be used, False otherwise. + :rtype: bool + """ response = pipeline_response.http_response return "location" in response.headers def get_polling_url(self) -> str: - """Return the polling URL.""" + """Return the Location header value. + + :return: The polling URL. + :rtype: str + """ return self._location_url def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. - :rtype: str + Always return None for a Location polling. + + :param pipeline_response: Success REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: Always None for this implementation. + :rtype: None """ return None def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """Process first response after initiating long running operation. - :param azure.core.pipeline.PipelineResponse response: initial REST call response. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The initial status. + :rtype: str """ response = pipeline_response.http_response @@ -367,10 +432,14 @@ def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: raise OperationFailed("Operation failed or canceled") def get_status(self, pipeline_response: PipelineResponseType) -> str: - """Process the latest status update retrieved from a 'location' header. + """Return the status string extracted from this response. + + For Location polling, it means the status monitor returns 202. - :param azure.core.pipeline.PipelineResponse response: latest REST call response. - :raises: BadResponse if response has no body and not status 202. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The status string. + :rtype: str """ response = pipeline_response.http_response if "location" in response.headers: @@ -385,28 +454,58 @@ class StatusCheckPolling(LongRunningOperation): """ def can_poll(self, pipeline_response: PipelineResponseType) -> bool: - """Answer if this polling method could be used.""" + """Answer if this polling method could be used. + + For this implementation, always True. + + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: True if this polling method could be used, False otherwise. + :rtype: bool + """ return True def get_polling_url(self) -> str: - """Return the polling URL.""" - raise ValueError("This polling doesn't support polling") + """Return the polling URL. + + This is not implemented for this polling, since we're never supposed to loop. + + :return: The polling URL. + :rtype: str + """ + raise ValueError("This polling doesn't support polling url") def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: - """Process first response after initiating long running - operation and set self.status attribute. + """Process first response after initiating long running operation. + + Will succeed immediately. - :param azure.core.pipeline.PipelineResponse response: initial REST call response. + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The initial status. + :rtype: str """ return "Succeeded" def get_status(self, pipeline_response: PipelineResponseType) -> str: + """Return the status string extracted from this response. + + Only possible status is success. + + :param pipeline_response: Initial REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The status string. + :rtype: str + """ return "Succeeded" def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: """If a final GET is needed, returns the URL. + :param pipeline_response: Success REST call response. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse :rtype: str + :return: Always None for this implementation. """ return None @@ -414,7 +513,15 @@ def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional class _SansIOLROBasePolling( Generic[PollingReturnType_co, PipelineClientType] ): # pylint: disable=too-many-instance-attributes - """A base class that has no opinion on IO, to help mypy be accurate.""" + """A base class that has no opinion on IO, to help mypy be accurate. + + :param float timeout: Default polling internal in absence of Retry-After header, in seconds. + :param list[LongRunningOperation] lro_algorithms: Ordered list of LRO algorithms to use. + :param lro_options: Additional options for LRO. For more information, see the algorithm's docstring. + :type lro_options: dict[str, any] + :param path_format_arguments: A dictionary of the format arguments to be used to format the URL. + :type path_format_arguments: dict[str, str] + """ _deserialization_callback: Callable[[Any], PollingReturnType_co] """The deserialization callback that returns the final instance.""" @@ -455,7 +562,12 @@ def initialize( ) -> None: """Set the initial status of this LRO. - :param initial_response: The initial response of the poller + :param client: The Azure Core Pipeline client used to make request. + :type client: ~azure.core.pipeline.PipelineClient + :param initial_response: The initial response for the call. + :type initial_response: ~azure.core.pipeline.PipelineResponse + :param deserialization_callback: A callback function to deserialize the final response. + :type deserialization_callback: callable :raises: HttpResponseError if initial status is incorrect LRO state """ self._client = client @@ -477,12 +589,12 @@ def initialize( except BadStatus as err: self._status = "Failed" - raise HttpResponseError(response=initial_response.http_response, error=err) + raise HttpResponseError(response=initial_response.http_response, error=err) from err except BadResponse as err: self._status = "Failed" - raise HttpResponseError(response=initial_response.http_response, message=str(err), error=err) + raise HttpResponseError(response=initial_response.http_response, message=str(err), error=err) from err except OperationFailed as err: - raise HttpResponseError(response=initial_response.http_response, error=err) + raise HttpResponseError(response=initial_response.http_response, error=err) from err def get_continuation_token(self) -> str: import pickle @@ -496,12 +608,16 @@ def from_continuation_token( try: client = kwargs["client"] except KeyError: - raise ValueError("Need kwarg 'client' to be recreated from continuation_token") + raise ValueError( # pylint: disable=raise-missing-from + "Need kwarg 'client' to be recreated from continuation_token" + ) try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError("Need kwarg 'deserialization_callback' to be recreated from continuation_token") + raise ValueError( # pylint: disable=raise-missing-from + "Need kwarg 'deserialization_callback' to be recreated from continuation_token" + ) import pickle @@ -512,7 +628,9 @@ def from_continuation_token( def status(self) -> str: """Return the current status as a string. + :rtype: str + :return: The current status. """ if not self._operation: raise ValueError("set_initial_status was never called. Did you give this instance to a poller?") @@ -520,17 +638,28 @@ def status(self) -> str: def finished(self) -> bool: """Is this polling finished? + :rtype: bool + :return: True if finished, False otherwise. """ return _finished(self.status()) def resource(self) -> PollingReturnType_co: - """Return the built resource.""" + """Return the built resource. + + :rtype: any + :return: The built resource. + """ return self._parse_resource(self._pipeline_response) def _parse_resource(self, pipeline_response: PipelineResponseType) -> PollingReturnType_co: """Assuming this response is a resource, use the deserialization callback to parse it. If body is empty, assuming no resource to return. + + :param pipeline_response: The response object. + :type pipeline_response: ~azure.core.pipeline.PipelineResponse + :return: The parsed resource. + :rtype: any """ response = pipeline_response.http_response if not _is_empty(response): @@ -607,7 +736,7 @@ def run(self) -> None: except BadStatus as err: self._status = "Failed" - raise HttpResponseError(response=self._pipeline_response.http_response, error=err) + raise HttpResponseError(response=self._pipeline_response.http_response, error=err) from err except BadResponse as err: self._status = "Failed" @@ -615,17 +744,15 @@ def run(self) -> None: response=self._pipeline_response.http_response, message=str(err), error=err, - ) + ) from err except OperationFailed as err: - raise HttpResponseError(response=self._pipeline_response.http_response, error=err) + raise HttpResponseError(response=self._pipeline_response.http_response, error=err) from err def _poll(self) -> None: """Poll status of operation so long as operation is incomplete and we have an endpoint to query. - :param callable update_cmd: The function to call to retrieve the - latest status of the long running operation. :raises: OperationFailed if operation status 'Failed' or 'Canceled'. :raises: BadStatus if response status invalid. :raises: BadResponse if response invalid. @@ -665,7 +792,9 @@ def request_status(self, status_link: str) -> PipelineResponseTypeVar: This method re-inject 'x-ms-client-request-id'. + :param str status_link: The URL to poll. :rtype: azure.core.pipeline.PipelineResponse + :return: The response of the status request. """ if self._path_format_arguments: status_link = self._client.format_url(status_link, **self._path_format_arguments) From 1467a211f6a57fb97b847531727167b29bdf3dff Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 26 Jun 2023 15:50:42 -0700 Subject: [PATCH 38/46] Black --- sdk/core/azure-core/azure/core/polling/base_polling.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index f9612670cdf5e..5e7288ae23cb9 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -168,8 +168,7 @@ def _is_empty(response: AllHTTPResponseType) -> bool: class LongRunningOperation(ABC, Generic[HTTPRequestType_co, HTTPResponseType_co]): - """Protocol to implement for a long running operation algorithm. - """ + """Protocol to implement for a long running operation algorithm.""" @abc.abstractmethod def can_poll( From 5b83d9c50d02200bb40052432da22f5f930a7427 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 28 Jun 2023 17:39:47 -0700 Subject: [PATCH 39/46] All LRO impl should use TypeVar --- .../azure/core/polling/base_polling.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 5e7288ae23cb9..73075cb7daa2d 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -377,13 +377,13 @@ def get_status(self, pipeline_response: PipelineResponseTypeVar) -> str: return status -class LocationPolling(LongRunningOperation): +class LocationPolling(LongRunningOperation[HttpRequestTypeVar, HttpResponseTypeVar]): """Implements a Location polling.""" _location_url: str """Location header""" - def can_poll(self, pipeline_response: PipelineResponseType) -> bool: + def can_poll(self, pipeline_response: PipelineResponseTypeVar) -> bool: """True if contains a Location header :param pipeline_response: Initial REST call response. @@ -402,7 +402,7 @@ def get_polling_url(self) -> str: """ return self._location_url - def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseTypeVar) -> Optional[str]: """If a final GET is needed, returns the URL. Always return None for a Location polling. @@ -414,7 +414,7 @@ def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional """ return None - def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: + def set_initial_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process first response after initiating long running operation. :param pipeline_response: Initial REST call response. @@ -430,7 +430,7 @@ def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: return "InProgress" raise OperationFailed("Operation failed or canceled") - def get_status(self, pipeline_response: PipelineResponseType) -> str: + def get_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Return the status string extracted from this response. For Location polling, it means the status monitor returns 202. @@ -447,12 +447,12 @@ def get_status(self, pipeline_response: PipelineResponseType) -> str: return "InProgress" if response.status_code == 202 else "Succeeded" -class StatusCheckPolling(LongRunningOperation): +class StatusCheckPolling(LongRunningOperation[HttpRequestTypeVar, HttpResponseTypeVar]): """Should be the fallback polling, that don't poll but exit successfully if not other polling are detected and status code is 2xx. """ - def can_poll(self, pipeline_response: PipelineResponseType) -> bool: + def can_poll(self, pipeline_response: PipelineResponseTypeVar) -> bool: """Answer if this polling method could be used. For this implementation, always True. @@ -474,7 +474,7 @@ def get_polling_url(self) -> str: """ raise ValueError("This polling doesn't support polling url") - def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: + def set_initial_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Process first response after initiating long running operation. Will succeed immediately. @@ -486,7 +486,7 @@ def set_initial_status(self, pipeline_response: PipelineResponseType) -> str: """ return "Succeeded" - def get_status(self, pipeline_response: PipelineResponseType) -> str: + def get_status(self, pipeline_response: PipelineResponseTypeVar) -> str: """Return the status string extracted from this response. Only possible status is success. @@ -498,7 +498,7 @@ def get_status(self, pipeline_response: PipelineResponseType) -> str: """ return "Succeeded" - def get_final_get_url(self, pipeline_response: PipelineResponseType) -> Optional[str]: + def get_final_get_url(self, pipeline_response: PipelineResponseTypeVar) -> Optional[str]: """If a final GET is needed, returns the URL. :param pipeline_response: Success REST call response. From f559db943b7173cb4a56cacbc572c6e4396cae4d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 5 Jul 2023 14:34:47 -0700 Subject: [PATCH 40/46] Feedback --- .../azure-core/azure/core/pipeline/policies/_utils.py | 7 +++++-- sdk/core/azure-core/azure/core/polling/_poller.py | 4 ++-- sdk/core/azure-core/azure/core/polling/base_polling.py | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py index 1a9824531b674..b78dab777df25 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py @@ -25,7 +25,7 @@ # -------------------------------------------------------------------------- import datetime import email.utils -from typing import Optional +from typing import Optional, cast from ...utils._utils import _FixedOffset, case_insensitive_dict @@ -38,7 +38,10 @@ def _parse_http_date(text: str) -> datetime.datetime: :return: The parsed datetime """ parsed_date = email.utils.parsedate_tz(text) - return datetime.datetime(*parsed_date[:6], tzinfo=_FixedOffset(parsed_date[9] / 60)) # type: ignore + if not parsed_date: + raise ValueError("Invalid HTTP date") + tz_offset = cast(int, parsed_date[9]) # Look at the code, tzoffset is always an int, at worst 0 + return datetime.datetime(*parsed_date[:6], tzinfo=_FixedOffset(tz_offset / 60)) def parse_retry_after(retry_after: str) -> float: diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 4115e49fe9831..3bd586a893c0d 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -114,9 +114,9 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError( # pylint: disable=raise-missing-from + raise ValueError( "Need kwarg 'deserialization_callback' to be recreated from continuation_token" - ) + ) from None import pickle initial_response = pickle.loads(base64.b64decode(continuation_token)) # nosec diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 73075cb7daa2d..901e3a906990d 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -607,16 +607,16 @@ def from_continuation_token( try: client = kwargs["client"] except KeyError: - raise ValueError( # pylint: disable=raise-missing-from + raise ValueError( "Need kwarg 'client' to be recreated from continuation_token" - ) + ) from None try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError( # pylint: disable=raise-missing-from + raise ValueError( "Need kwarg 'deserialization_callback' to be recreated from continuation_token" - ) + ) from None import pickle From 5035836cd1e8c11b8c470e9108aa670aa91378a7 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 5 Jul 2023 14:58:19 -0700 Subject: [PATCH 41/46] Convert some Anyu after feedback --- sdk/core/azure-core/azure/core/polling/base_polling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 901e3a906990d..94c3c579491c9 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -556,8 +556,8 @@ def __init__( def initialize( self, client: PipelineClientType, - initial_response: Any, - deserialization_callback: Callable[[Any], PollingReturnType_co], + initial_response: PipelineResponseType, + deserialization_callback: Callable[[PipelineResponseType], PollingReturnType_co], ) -> None: """Set the initial status of this LRO. From e7c52a012cb84decfbec71bc580258ca647986de Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 5 Jul 2023 15:53:56 -0700 Subject: [PATCH 42/46] Spellcheck --- sdk/core/azure-core/azure/core/pipeline/policies/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py index 27b162b0bf3da..ee710fc5a2c0b 100644 --- a/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py +++ b/sdk/core/azure-core/azure/core/pipeline/policies/_utils.py @@ -41,7 +41,7 @@ def _parse_http_date(text: str) -> datetime.datetime: parsed_date = email.utils.parsedate_tz(text) if not parsed_date: raise ValueError("Invalid HTTP date") - tz_offset = cast(int, parsed_date[9]) # Look at the code, tzoffset is always an int, at worst 0 + tz_offset = cast(int, parsed_date[9]) # Look at the code, tz_offset is always an int, at worst 0 return datetime.datetime(*parsed_date[:6], tzinfo=_FixedOffset(tz_offset / 60)) From 3501c2239ba749e935159a92257987faab9146eb Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Wed, 5 Jul 2023 17:37:46 -0700 Subject: [PATCH 43/46] Black --- sdk/core/azure-core/azure/core/polling/_poller.py | 4 +--- sdk/core/azure-core/azure/core/polling/base_polling.py | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 3bd586a893c0d..6cf4394c8d011 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -114,9 +114,7 @@ def from_continuation_token(cls, continuation_token: str, **kwargs) -> Tuple[Any try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError( - "Need kwarg 'deserialization_callback' to be recreated from continuation_token" - ) from None + raise ValueError("Need kwarg 'deserialization_callback' to be recreated from continuation_token") from None import pickle initial_response = pickle.loads(base64.b64decode(continuation_token)) # nosec diff --git a/sdk/core/azure-core/azure/core/polling/base_polling.py b/sdk/core/azure-core/azure/core/polling/base_polling.py index 94c3c579491c9..001c02aea91ca 100644 --- a/sdk/core/azure-core/azure/core/polling/base_polling.py +++ b/sdk/core/azure-core/azure/core/polling/base_polling.py @@ -607,16 +607,12 @@ def from_continuation_token( try: client = kwargs["client"] except KeyError: - raise ValueError( - "Need kwarg 'client' to be recreated from continuation_token" - ) from None + raise ValueError("Need kwarg 'client' to be recreated from continuation_token") from None try: deserialization_callback = kwargs["deserialization_callback"] except KeyError: - raise ValueError( - "Need kwarg 'deserialization_callback' to be recreated from continuation_token" - ) from None + raise ValueError("Need kwarg 'deserialization_callback' to be recreated from continuation_token") from None import pickle From c5b06c1db78895c379e35ed40eb603b7632034c5 Mon Sep 17 00:00:00 2001 From: Kashif Khan <361477+kashifkhan@users.noreply.github.com> Date: Thu, 6 Jul 2023 13:32:14 -0500 Subject: [PATCH 44/46] Update sdk/core/azure-core/azure/core/polling/_async_poller.py --- sdk/core/azure-core/azure/core/polling/_async_poller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index b636b7187857c..e745b441155b2 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -91,7 +91,7 @@ async def async_poller( :type deserialization_callback: callable or msrest.serialization.Model :param polling_method: The polling strategy to adopt :type polling_method: ~azure.core.polling.PollingMethod - :returns: The final resource at the end of the polling. + :return: The final resource at the end of the polling. :rtype: any or None """ poller = AsyncLROPoller(client, initial_response, deserialization_callback, polling_method) From d62044571ade662a6fc6d7a73b6e8dd0374b00d0 Mon Sep 17 00:00:00 2001 From: Kashif Khan <361477+kashifkhan@users.noreply.github.com> Date: Thu, 6 Jul 2023 13:33:38 -0500 Subject: [PATCH 45/46] Update sdk/core/azure-core/azure/core/polling/_async_poller.py --- sdk/core/azure-core/azure/core/polling/_async_poller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/_async_poller.py b/sdk/core/azure-core/azure/core/polling/_async_poller.py index e745b441155b2..88cb29375c44b 100644 --- a/sdk/core/azure-core/azure/core/polling/_async_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_async_poller.py @@ -133,7 +133,7 @@ def __init__( def polling_method(self) -> AsyncPollingMethod[PollingReturnType_co]: """Return the polling method associated to this poller. - :returns: The polling method associated to this poller. + :return: The polling method associated to this poller. :rtype: ~azure.core.polling.AsyncPollingMethod """ return self._polling_method From c7a6d9376d70996f078aa1d4a4ec4626d576f0d7 Mon Sep 17 00:00:00 2001 From: Kashif Khan <361477+kashifkhan@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:59:15 -0500 Subject: [PATCH 46/46] Update sdk/core/azure-core/azure/core/polling/_poller.py --- sdk/core/azure-core/azure/core/polling/_poller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-core/azure/core/polling/_poller.py b/sdk/core/azure-core/azure/core/polling/_poller.py index 6cf4394c8d011..c8c3db07e810f 100644 --- a/sdk/core/azure-core/azure/core/polling/_poller.py +++ b/sdk/core/azure-core/azure/core/polling/_poller.py @@ -198,7 +198,7 @@ def _start(self): def polling_method(self) -> PollingMethod[PollingReturnType_co]: """Return the polling method associated to this poller. - :returns: The polling method + :return: The polling method :rtype: ~azure.core.polling.PollingMethod """ return self._polling_method