From 01f196125fc63e227536eed95a0ca743f43c891f Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Tue, 11 Jun 2024 15:33:34 -0700 Subject: [PATCH 1/4] Sample for sending image data --- sdk/ai/azure-ai-inference/README.md | 9 +- sdk/ai/azure-ai-inference/samples/README.md | 5 +- ...letions_from_input_json_with_image_url.py} | 8 +- ...sample_chat_completions_with_image_data.py | 91 +++++++++++++++++++ ...sample_chat_completions_with_image_url.py} | 8 +- 5 files changed, 109 insertions(+), 12 deletions(-) rename sdk/ai/azure-ai-inference/samples/{sample_chat_completions_from_input_json_with_images.py => sample_chat_completions_from_input_json_with_image_url.py} (92%) create mode 100644 sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py rename sdk/ai/azure-ai-inference/samples/{sample_chat_completions_with_images.py => sample_chat_completions_with_image_url.py} (93%) diff --git a/sdk/ai/azure-ai-inference/README.md b/sdk/ai/azure-ai-inference/README.md index 622d45ccc4f5..5d27ec3b1776 100644 --- a/sdk/ai/azure-ai-inference/README.md +++ b/sdk/ai/azure-ai-inference/README.md @@ -210,8 +210,13 @@ print(response.choices[0].message.content) -The following types or messages are supported: `SystemMessage`,`UserMessage`, `AssistantMessage`, `ToolMessage`. See sample [sample_chat_completions_with_tools.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_tools.py) for usage of `ToolMessage`. See [sample_chat_completions_with_images.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_images.py) for usage of `UserMessage` that -includes uploading an image. +The following types or messages are supported: `SystemMessage`,`UserMessage`, `AssistantMessage`, `ToolMessage`. See also samples: + +* [sample_chat_completions_with_tools.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_tools.py) for usage of `ToolMessage`. +* [sample_chat_completions_with_image_url.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_url.py) for usage of `UserMessage` that +includes sending an image URL. +* [sample_chat_completions_with_image_data.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py) for usage of `UserMessage` that +includes sending image data read from a local file. Alternatively, you can provide the messages as dictionary instead of using the strongly typed classes like `SystemMessage` and `UserMessage`: diff --git a/sdk/ai/azure-ai-inference/samples/README.md b/sdk/ai/azure-ai-inference/samples/README.md index 9d8e5cfe7aea..d83479d89098 100644 --- a/sdk/ai/azure-ai-inference/samples/README.md +++ b/sdk/ai/azure-ai-inference/samples/README.md @@ -92,11 +92,12 @@ similarly for the other samples. |[sample_chat_completions_streaming.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_streaming.py) | One chat completion operation using a synchronous client and streaming response. | |[sample_chat_completions_streaming_with_entra_id_auth.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_streaming_with_entra_id_auth.py) | One chat completion operation using a synchronous client and streaming response, using Entra ID authentication. This sample also shows setting the `azureml-model-deployment` HTTP request header, which may be required for some Managed Compute endpoint. | |[sample_chat_completions.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions.py) | One chat completion operation using a synchronous client. | -|[sample_chat_completions_with_images.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_images.py) | One chat completion operation using a synchronous client, which includes sending an input image. | +|[sample_chat_completions_with_image_url.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_url.py) | One chat completion operation using a synchronous client, which includes sending an input image URL. | +|[sample_chat_completions_with_image_data.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py) | One chat completion operation using a synchronous client, which includes sending input image data read from a local file. | |[sample_chat_completions_with_history.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_history.py) | Two chat completion operations using a synchronous client, with the second completion using chat history from the first. | |[sample_chat_completions_from_input_bytes.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_bytes.py) | One chat completion operation using a synchronous client, with input messages provided as `IO[bytes]`. | |[sample_chat_completions_from_input_json.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json.py) | One chat completion operation using a synchronous client, with input messages provided as a dictionary (type `MutableMapping[str, Any]`) | -|[sample_chat_completions_from_input_json_with_images.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_images.py) | One chat completion operation using a synchronous client, with input messages provided as a dictionary (type `MutableMapping[str, Any]`). Includes sending an input image. | +|[sample_chat_completions_from_input_json_with_image_url.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_image_url.py) | One chat completion operation using a synchronous client, with input messages provided as a dictionary (type `MutableMapping[str, Any]`). Includes sending an input image URL. | |[sample_chat_completions_with_tools.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_tools.py) | Shows how do use a tool (function) in chat completions, for an AI model that supports tools | |[sample_load_client.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_load_client.py) | Shows how do use the function `load_client` to create the appropriate synchronous client based on the provided endpoint URL. In this example, it creates a synchronous `ChatCompletionsClient`. | |[sample_get_model_info.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-inference/samples/sample_get_model_info.py) | Get AI model information using the chat completions client. Similarly can be done with all other clients. | diff --git a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_images.py b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_image_url.py similarity index 92% rename from sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_images.py rename to sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_image_url.py index 93abd4af8b33..bb55c7f772d6 100644 --- a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_images.py +++ b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_from_input_json_with_image_url.py @@ -7,11 +7,11 @@ This sample demonstrates how to get a chat completions response from the service using a synchronous client, and directly providing the JSON request body (containing input chat messages). The sample - shows how to include an image in the input chat messages. + shows how to include an image URL in the input chat messages. This sample will only work on AI models that support image input. USAGE: - python sample_chat_completions_from_input_json_with_image.py + python sample_chat_completions_from_input_json_with_image_url.py Set these two or three environment variables before running the sample: 1) CHAT_COMPLETIONS_ENDPOINT - Your endpoint URL, in the form @@ -26,7 +26,7 @@ # pyright: reportAttributeAccessIssue=false -def sample_chat_completions_from_input_json_with_image(): +def sample_chat_completions_from_input_json_with_image_url(): import os from azure.ai.inference import ChatCompletionsClient from azure.core.credentials import AzureKeyCredential @@ -83,4 +83,4 @@ def sample_chat_completions_from_input_json_with_image(): if __name__ == "__main__": - sample_chat_completions_from_input_json_with_image() + sample_chat_completions_from_input_json_with_image_url() diff --git a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py new file mode 100644 index 000000000000..9c6e55057c46 --- /dev/null +++ b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py @@ -0,0 +1,91 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +""" +DESCRIPTION: + This sample demonstrates how to get a chat completions response from + the service using a synchronous client. The sample shows how to load + an image from a file and include it in the input chat messages. + This sample will only work on AI models that support image input. + +USAGE: + python sample_chat_completions_with_image_data.py + + Set these two or three environment variables before running the sample: + 1) CHAT_COMPLETIONS_ENDPOINT - Your endpoint URL, in the form + https://..inference.ai.azure.com + where `your-deployment-name` is your unique AI Model deployment name, and + `your-azure-region` is the Azure region where your model is deployed. + 2) CHAT_COMPLETIONS_KEY - Your model key (a 32-character string). Keep it secret. + 3) CHAT_COMPLETIONS_DEPLOYMENT_NAME - Optional. The value for the HTTP + request header `azureml-model-deployment`. +""" + + +def sample_chat_completions_with_image_data(): + import os + from azure.ai.inference import ChatCompletionsClient + from azure.ai.inference.models import ( + SystemMessage, UserMessage, TextContentItem, + ImageContentItem, ImageUrl, ImageDetailLevel + ) + from azure.core.credentials import AzureKeyCredential + + try: + endpoint = os.environ["CHAT_COMPLETIONS_ENDPOINT"] + key = os.environ["CHAT_COMPLETIONS_KEY"] + except KeyError: + print("Missing environment variable 'CHAT_COMPLETIONS_ENDPOINT' or 'CHAT_COMPLETIONS_KEY'") + print("Set them before running this sample.") + exit() + + try: + model_deployment = os.environ["CHAT_COMPLETIONS_DEPLOYMENT_NAME"] + except KeyError: + print("Could not read optional environment variable `CHAT_COMPLETIONS_DEPLOYMENT_NAME`.") + print("HTTP request header `azureml-model-deployment` will not be set.") + model_deployment = None + + image_data_url = get_image_data_url("sample1.png", "png") + + client = ChatCompletionsClient( + endpoint=endpoint, + credential=AzureKeyCredential(key), + headers={"azureml-model-deployment": model_deployment}, + ) + + response = client.complete( + messages=[ + SystemMessage(content="You are an AI assistant that describes images in details."), + UserMessage( + content=[ + TextContentItem(text="What's in this image?"), + ImageContentItem( + image_url=ImageUrl( + url=image_data_url, + detail=ImageDetailLevel.HIGH, + ), + ), + ], + ), + ], + ) + + print(response.choices[0].message.content) + + +def get_image_data_url(image_file: str, image_format: str) -> str: + import base64 + try: + with open(image_file, "rb") as f: + image_data = base64.b64encode(f.read()).decode("utf-8") + except FileNotFoundError: + print(f"Could not read '{image_file}'.") + print("Set the correct path to the image file before running this sample.") + exit() + return f"data:image/{image_format};base64,{image_data}" + + +if __name__ == "__main__": + sample_chat_completions_with_image_data() diff --git a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_images.py b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_url.py similarity index 93% rename from sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_images.py rename to sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_url.py index 9e57692d8eec..d6c6a6e1bd47 100644 --- a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_images.py +++ b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_url.py @@ -6,11 +6,11 @@ DESCRIPTION: This sample demonstrates how to get a chat completions response from the service using a synchronous client. The sample - shows how to include an image in the input chat messages. + shows how to include an image URL in the input chat messages. This sample will only work on AI models that support image input. USAGE: - python sample_chat_completions_with_images.py + python sample_chat_completions_with_image_url.py Set these two or three environment variables before running the sample: 1) CHAT_COMPLETIONS_ENDPOINT - Your endpoint URL, in the form @@ -23,7 +23,7 @@ """ -def sample_chat_completions_with_images(): +def sample_chat_completions_with_image_url(): import os from azure.ai.inference import ChatCompletionsClient from azure.ai.inference.models import ( @@ -74,4 +74,4 @@ def sample_chat_completions_with_images(): if __name__ == "__main__": - sample_chat_completions_with_images() + sample_chat_completions_with_image_url() From 181439953d8b88cd87134f9e1fd2d82a287697ee Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:28:11 -0700 Subject: [PATCH 2/4] Option to construct ImageUrl directly from image file name and format --- .../azure/ai/inference/_patch.py | 2 + .../azure/ai/inference/models/__init__.py | 2 +- .../azure/ai/inference/models/_patch.py | 62 ++++++++++++++++++- ...sample_chat_completions_with_image_data.py | 7 +-- 4 files changed, 65 insertions(+), 8 deletions(-) diff --git a/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py b/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py index 05efb4031fc6..1c31cdf502e6 100644 --- a/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py +++ b/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py @@ -16,6 +16,8 @@ https://github.com/Azure/autorest.python/issues/2619 (all clients). Otherwise intellisense did not show the patched public methods on the client object, when the client is defined using context manager ("with" statement). +6. ImageUrl class is patched in /models/_patch.py, in order to support a constructor + that accepts image file and image format directly. """ import json diff --git a/sdk/ai/azure-ai-inference/azure/ai/inference/models/__init__.py b/sdk/ai/azure-ai-inference/azure/ai/inference/models/__init__.py index 662f068900e1..8ebb41cb3c4a 100644 --- a/sdk/ai/azure-ai-inference/azure/ai/inference/models/__init__.py +++ b/sdk/ai/azure-ai-inference/azure/ai/inference/models/__init__.py @@ -18,7 +18,7 @@ from ._models import ChatCompletionsToolDefinition from ._models import ContentItem from ._models import ImageContentItem -from ._models import ImageUrl +from ._patch import ImageUrl from ._models import TextContentItem from ._models import ChatRequestMessage from ._models import ChatResponseMessage diff --git a/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py b/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py index bbeda9d47f64..b31344a007f5 100644 --- a/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py +++ b/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py @@ -7,18 +7,73 @@ Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize """ import asyncio +import base64 import json import logging import queue import re -from typing import List, AsyncIterator, Iterator +from typing import List, AsyncIterator, Iterator, Optional, Union, Any, overload, Mapping from azure.core.rest import HttpResponse, AsyncHttpResponse +from ._models import ImageUrl as ImageUrlGenerated from .. import models as _models logger = logging.getLogger(__name__) +class ImageUrl(ImageUrlGenerated): + @overload + def __init__( + self, + *, + url: str, + detail: Optional[Union[str, "_models.ImageDetailLevel"]] = None, + ): + """An internet location from which the model may retrieve an image. + + :ivar url: The URL of the image. Required. + :vartype url: str + :ivar detail: The evaluation quality setting to use, which controls relative prioritization of speed, token consumption, and accuracy. Known values are: "auto", "low", and "high". + :vartype detail: str or ~azure.ai.inference.models.ImageDetailLevel + """ + ... + + @overload + def __init__( + self, + *, + image_file: str, + image_format: str, + detail: Optional[Union[str, "_models.ImageDetailLevel"]] = None, + ): + """A local file from which the model may retrieve an image. + + :ivar image_file: The local file name of the image. Required. + :vartype image_file: str + :ivar image_format: The format of image, such as "jpeg" or "png". Must be specified together with `image_file` Required. + :vartype image_format: str + :ivar detail: The evaluation quality setting to use, which controls relative prioritization of speed, token consumption, and accuracy. Known values are: "auto", "low", and "high". + :vartype detail: str or ~azure.ai.inference.models.ImageDetailLevel + """ + ... + + @overload + def __init__(self, mapping: Mapping[str, Any]): + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: # pylint: disable=useless-super-delegation + image_file = kwargs.pop("image_file", None) + image_format = kwargs.pop("image_format", None) + if image_file and image_format: + with open(image_file, "rb") as f: + image_data = base64.b64encode(f.read()).decode("utf-8") + kwargs["url"] = f"data:image/{image_format};base64,{image_data}" + super().__init__(*args, **kwargs) + + class BaseStreamingChatCompletions: """A base class for the sync and async streaming chat completions responses, holding any common code to deserializes the Server Sent Events (SSE) response stream into chat completions updates, each one @@ -106,7 +161,7 @@ def __init__(self, response: HttpResponse): def __iter__(self): return self - def __next__(self) -> _models.StreamingChatCompletionsUpdate: + def __next__(self) -> "_models.StreamingChatCompletionsUpdate": while self._queue.empty() and not self._done: self._done = self._read_next_block() if self._queue.empty(): @@ -145,7 +200,7 @@ def __init__(self, response: AsyncHttpResponse): def __aiter__(self): return self - async def __anext__(self) -> _models.StreamingChatCompletionsUpdate: + async def __anext__(self) -> "_models.StreamingChatCompletionsUpdate": while self._queue.empty() and not self._done: self._done = await self._read_next_block_async() if self._queue.empty(): @@ -170,6 +225,7 @@ async def aclose(self) -> None: __all__: List[str] = [ + "ImageUrl", "StreamingChatCompletions", "AsyncStreamingChatCompletions", ] # Add all objects you want publicly available to users at this package level diff --git a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py index 9c6e55057c46..8186df9e2bc3 100644 --- a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py +++ b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py @@ -47,8 +47,6 @@ def sample_chat_completions_with_image_data(): print("HTTP request header `azureml-model-deployment` will not be set.") model_deployment = None - image_data_url = get_image_data_url("sample1.png", "png") - client = ChatCompletionsClient( endpoint=endpoint, credential=AzureKeyCredential(key), @@ -63,8 +61,9 @@ def sample_chat_completions_with_image_data(): TextContentItem(text="What's in this image?"), ImageContentItem( image_url=ImageUrl( - url=image_data_url, - detail=ImageDetailLevel.HIGH, + image_file="sample1.png", + image_format="png", + detail=ImageDetailLevel.HIGH ), ), ], From 1b869f46220635e2f87d8269a922ca07bbf085d7 Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 13 Jun 2024 07:52:15 -0700 Subject: [PATCH 3/4] Add support for ImageUrl.load() --- .../azure/ai/inference/_patch.py | 3 +- .../azure/ai/inference/models/_patch.py | 69 ++++++++----------- ...sample_chat_completions_with_image_data.py | 4 +- 3 files changed, 30 insertions(+), 46 deletions(-) diff --git a/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py b/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py index 1c31cdf502e6..dcb98f187f29 100644 --- a/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py +++ b/sdk/ai/azure-ai-inference/azure/ai/inference/_patch.py @@ -16,8 +16,7 @@ https://github.com/Azure/autorest.python/issues/2619 (all clients). Otherwise intellisense did not show the patched public methods on the client object, when the client is defined using context manager ("with" statement). -6. ImageUrl class is patched in /models/_patch.py, in order to support a constructor - that accepts image file and image format directly. +6. Add support for load() method in ImageUrl class (see /models/_patch.py). """ import json diff --git a/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py b/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py index b31344a007f5..f2fb39cf1f3f 100644 --- a/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py +++ b/sdk/ai/azure-ai-inference/azure/ai/inference/models/_patch.py @@ -12,66 +12,51 @@ import logging import queue import re +import sys -from typing import List, AsyncIterator, Iterator, Optional, Union, Any, overload, Mapping +from typing import List, AsyncIterator, Iterator, Optional, Union from azure.core.rest import HttpResponse, AsyncHttpResponse from ._models import ImageUrl as ImageUrlGenerated from .. import models as _models +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + logger = logging.getLogger(__name__) class ImageUrl(ImageUrlGenerated): - @overload - def __init__( - self, - *, - url: str, - detail: Optional[Union[str, "_models.ImageDetailLevel"]] = None, - ): - """An internet location from which the model may retrieve an image. - - :ivar url: The URL of the image. Required. - :vartype url: str - :ivar detail: The evaluation quality setting to use, which controls relative prioritization of speed, token consumption, and accuracy. Known values are: "auto", "low", and "high". - :vartype detail: str or ~azure.ai.inference.models.ImageDetailLevel - """ - ... - @overload - def __init__( - self, + @classmethod + def load( + cls, *, image_file: str, image_format: str, - detail: Optional[Union[str, "_models.ImageDetailLevel"]] = None, - ): - """A local file from which the model may retrieve an image. + detail: Optional[Union[str, "_models.ImageDetailLevel"]] = None + ) -> Self: + """ + Create an ImageUrl object from a local image file. The method reads the image + file and encodes it as a base64 string, which together with the image format + is then used to format the JSON `url` value passed in the request payload. - :ivar image_file: The local file name of the image. Required. + :ivar image_file: The name of the local image file to load. Required. :vartype image_file: str - :ivar image_format: The format of image, such as "jpeg" or "png". Must be specified together with `image_file` Required. + :ivar image_format: The MIME type format of the image. For example: "jpeg", "png". Required. :vartype image_format: str - :ivar detail: The evaluation quality setting to use, which controls relative prioritization of speed, token consumption, and accuracy. Known values are: "auto", "low", and "high". + :ivar detail: The evaluation quality setting to use, which controls relative prioritization of + speed, token consumption, and accuracy. Known values are: "auto", "low", and "high". :vartype detail: str or ~azure.ai.inference.models.ImageDetailLevel + :return: An ImageUrl object with the image data encoded as a base64 string. + :rtype: ~azure.ai.inference.models.ImageUrl + :raises FileNotFoundError when the image file could not be opened. """ - ... - - @overload - def __init__(self, mapping: Mapping[str, Any]): - """ - :param mapping: raw JSON to initialize the model. - :type mapping: Mapping[str, Any] - """ - - def __init__(self, *args: Any, **kwargs: Any) -> None: # pylint: disable=useless-super-delegation - image_file = kwargs.pop("image_file", None) - image_format = kwargs.pop("image_format", None) - if image_file and image_format: - with open(image_file, "rb") as f: - image_data = base64.b64encode(f.read()).decode("utf-8") - kwargs["url"] = f"data:image/{image_format};base64,{image_data}" - super().__init__(*args, **kwargs) + with open(image_file, "rb") as f: + image_data = base64.b64encode(f.read()).decode("utf-8") + url = f"data:image/{image_format};base64,{image_data}" + return cls(url=url, detail=detail) class BaseStreamingChatCompletions: diff --git a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py index 8186df9e2bc3..6d78d6475aa7 100644 --- a/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py +++ b/sdk/ai/azure-ai-inference/samples/sample_chat_completions_with_image_data.py @@ -60,10 +60,10 @@ def sample_chat_completions_with_image_data(): content=[ TextContentItem(text="What's in this image?"), ImageContentItem( - image_url=ImageUrl( + image_url=ImageUrl.load( image_file="sample1.png", image_format="png", - detail=ImageDetailLevel.HIGH + detail=ImageDetailLevel.HIGH, ), ), ], From 0eb9d792967e7a4b2dc925d91fde4060e63a46eb Mon Sep 17 00:00:00 2001 From: Darren Cohen <39422044+dargilco@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:12:33 -0700 Subject: [PATCH 4/4] Add test for ImageUrl.load() --- .../tests/test_model_inference_client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sdk/ai/azure-ai-inference/tests/test_model_inference_client.py b/sdk/ai/azure-ai-inference/tests/test_model_inference_client.py index c8e722d32026..f1efcccaa353 100644 --- a/sdk/ai/azure-ai-inference/tests/test_model_inference_client.py +++ b/sdk/ai/azure-ai-inference/tests/test_model_inference_client.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. # ------------------------------------ import inspect +import os import azure.ai.inference as sdk from model_inference_test_base import ModelClientTestBase, ServicePreparerChatCompletions, ServicePreparerEmbeddings @@ -65,6 +66,18 @@ def test_embeddings(self, **kwargs): self._validate_embeddings_result(response) client.close() + def test_image_url_load(self, **kwargs): + local_folder = os.path.dirname(os.path.abspath(__file__)) + image_file = os.path.join(local_folder, "../samples/sample1.png") + image_url = sdk.models._patch.ImageUrl.load( + image_file=image_file, + image_format="png", + detail=sdk.models.ImageDetailLevel.AUTO, + ) + assert image_url + assert image_url.url.startswith("data:image/png;base64,iVBORw") + assert image_url.detail == sdk.models.ImageDetailLevel.AUTO + # ********************************************************************************** # # HAPPY PATH TESTS - CHAT COMPLETIONS