From ca675f5bab5de2aa927fe7d581c2959ba73c4cb6 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Fri, 31 Jan 2025 12:27:22 +0200 Subject: [PATCH 1/7] Add mistral vision code and cookbooks --- cookbook/models/mistral/.gitignore | 7 ++ .../models/mistral/image_bytes_input_agent.py | 34 ++++++ .../models/mistral/image_compare_agent.py | 17 +++ .../models/mistral/image_file_input_agent.py | 23 ++++ .../image_ocr_with_structured_output.py | 31 ++++++ .../mistral/image_transcribe_document_agent | 19 ++++ libs/agno/agno/models/anthropic/claude.py | 103 +++++++++--------- libs/agno/agno/models/mistral/mistral.py | 80 ++++++++++---- 8 files changed, 240 insertions(+), 74 deletions(-) create mode 100644 cookbook/models/mistral/.gitignore create mode 100644 cookbook/models/mistral/image_bytes_input_agent.py create mode 100644 cookbook/models/mistral/image_compare_agent.py create mode 100644 cookbook/models/mistral/image_file_input_agent.py create mode 100644 cookbook/models/mistral/image_ocr_with_structured_output.py create mode 100644 cookbook/models/mistral/image_transcribe_document_agent diff --git a/cookbook/models/mistral/.gitignore b/cookbook/models/mistral/.gitignore new file mode 100644 index 0000000000..5854ecb565 --- /dev/null +++ b/cookbook/models/mistral/.gitignore @@ -0,0 +1,7 @@ +*.jpg +*.jpeg +*.png +*.mp3 +*.wav +*.mp4 +*.mp3 diff --git a/cookbook/models/mistral/image_bytes_input_agent.py b/cookbook/models/mistral/image_bytes_input_agent.py new file mode 100644 index 0000000000..28694bae8f --- /dev/null +++ b/cookbook/models/mistral/image_bytes_input_agent.py @@ -0,0 +1,34 @@ +import requests + +from agno.agent import Agent +from agno.media import Image +from agno.tools.duckduckgo import DuckDuckGoTools +from agno.models.mistral.mistral import MistralChat + +agent = Agent( + model=MistralChat(id="pixtral-12b-2409"), + show_tool_calls=True, + markdown=True, +) + +image_url = "https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg" + +def fetch_image_bytes(url: str) -> bytes: + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + } + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.content + +image_bytes_from_url = fetch_image_bytes(image_url) + +agent.print_response( + "Tell me about this image.", + images=[ + Image(content=image_bytes_from_url), + ], +) + diff --git a/cookbook/models/mistral/image_compare_agent.py b/cookbook/models/mistral/image_compare_agent.py new file mode 100644 index 0000000000..6323eb6295 --- /dev/null +++ b/cookbook/models/mistral/image_compare_agent.py @@ -0,0 +1,17 @@ +from agno.agent import Agent +from agno.media import Image +from agno.models.mistral.mistral import MistralChat + +agent = Agent( + model=MistralChat(id="pixtral-12b-2409"), + markdown=True, +) + +agent.print_response( + "what are the differences between two images?", + images=[ + Image(url="https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg"), + Image(url="https://assets.visitorscoverage.com/production/wp-content/uploads/2024/04/AdobeStock_626542468-min-1024x683.jpeg"), + ], + stream=True, +) diff --git a/cookbook/models/mistral/image_file_input_agent.py b/cookbook/models/mistral/image_file_input_agent.py new file mode 100644 index 0000000000..3c558755c7 --- /dev/null +++ b/cookbook/models/mistral/image_file_input_agent.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from agno.agent import Agent +from agno.media import Image +from agno.models.mistral.mistral import MistralChat +from agno.tools.duckduckgo import DuckDuckGoTools + +agent = Agent( + model=MistralChat(id="pixtral-12b-2409"), + tools=[DuckDuckGoTools()], # pixtral-12b-2409 is not so great at tool calls, but it might work. + show_tool_calls=True, + markdown=True, +) + +image_path = Path(__file__).parent.joinpath("sample.jpeg") + +agent.print_response( + "Tell me about this image and give me the latest news about it from duckduckgo.", + images=[ + Image(filepath=image_path), + ], + stream=True, +) diff --git a/cookbook/models/mistral/image_ocr_with_structured_output.py b/cookbook/models/mistral/image_ocr_with_structured_output.py new file mode 100644 index 0000000000..439ffadb6d --- /dev/null +++ b/cookbook/models/mistral/image_ocr_with_structured_output.py @@ -0,0 +1,31 @@ +from typing import List + +from pydantic import BaseModel +from agno.agent import Agent +from agno.media import Image +from agno.models.mistral.mistral import MistralChat + +class GroceryItem(BaseModel): + item_name: str + price: float + +class GroceryListElements(BaseModel): + bill_number: str + items: List[GroceryItem] + total_price: float + +agent = Agent( + model=MistralChat(id="pixtral-12b-2409"), + instructions=[ + "Extract the text elements described by the user from the picture", + ], + response_model=GroceryListElements, + markdown=True, +) + +agent.print_response( + "From this restaurant bill, extract the bill number, item names and associated prices, and total price and return it as a string in a Json object", + images=[ + Image(url="https://i.imghippo.com/files/kgXi81726851246.jpg") + ], +) diff --git a/cookbook/models/mistral/image_transcribe_document_agent b/cookbook/models/mistral/image_transcribe_document_agent new file mode 100644 index 0000000000..1ccf27cb82 --- /dev/null +++ b/cookbook/models/mistral/image_transcribe_document_agent @@ -0,0 +1,19 @@ +""" +This agent transcribes an old written document from an image. +""" + +from agno.agent import Agent +from agno.media import Image +from agno.models.mistral.mistral import MistralChat + +agent = Agent( + model=MistralChat(id="pixtral-12b-2409"), + markdown=True, +) + +agent.print_response( + "Transcribe this document.", + images=[ + Image(url="https://ciir.cs.umass.edu/irdemo/hw-demo/page_example.jpg"), + ], +) diff --git a/libs/agno/agno/models/anthropic/claude.py b/libs/agno/agno/models/anthropic/claude.py index d6f2ac8446..d5fb889f6e 100644 --- a/libs/agno/agno/models/anthropic/claude.py +++ b/libs/agno/agno/models/anthropic/claude.py @@ -50,7 +50,7 @@ def format_image_for_message(image: Image) -> Optional[Dict[str, Any]]: elif image.filepath is not None: from pathlib import Path - path = Path(image.filepath) + path = Path(image.filepath) if isinstance(image.filepath, str) else image.filepath if path.exists() and path.is_file(): with open(image.filepath, "rb") as f: content_bytes = f.read() @@ -90,6 +90,55 @@ def format_image_for_message(image: Image) -> Optional[Dict[str, Any]]: return None +def _format_messages(messages: List[Message]) -> Tuple[List[Dict[str, str]], str]: + """ + Process the list of messages and separate them into API messages and system messages. + + Args: + messages (List[Message]): The list of messages to process. + + Returns: + Tuple[List[Dict[str, str]], str]: A tuple containing the list of API messages and the concatenated system messages. + """ + chat_messages: List[Dict[str, str]] = [] + system_messages: List[str] = [] + + for idx, message in enumerate(messages): + content = message.content or "" + if message.role == "system" or (message.role != "user" and idx in [0, 1]): + if content is not None: + system_messages.append(content) # type: ignore + continue + elif message.role == "user": + if isinstance(content, str): + content = [{"type": "text", "text": content}] + + if message.images is not None: + for image in message.images: + image_content = format_image_for_message(image) + if image_content: + content.append(image_content) + + # Handle tool calls from history + elif message.role == "assistant" and isinstance(message.content, str) and message.tool_calls: + if message.content: + content = [TextBlock(text=message.content, type="text")] + else: + content = [] + for tool_call in message.tool_calls: + content.append( + ToolUseBlock( + id=tool_call["id"], + input=json.loads(tool_call["function"]["arguments"]), + name=tool_call["function"]["name"], + type="tool_use", + ) + ) + + chat_messages.append({"role": message.role, "content": content}) # type: ignore + return chat_messages, " ".join(system_messages) + + @dataclass class Claude(Model): """ @@ -156,54 +205,6 @@ def request_kwargs(self) -> Dict[str, Any]: _request_params.update(self.request_params) return _request_params - def format_messages(self, messages: List[Message]) -> Tuple[List[Dict[str, str]], str]: - """ - Process the list of messages and separate them into API messages and system messages. - - Args: - messages (List[Message]): The list of messages to process. - - Returns: - Tuple[List[Dict[str, str]], str]: A tuple containing the list of API messages and the concatenated system messages. - """ - chat_messages: List[Dict[str, str]] = [] - system_messages: List[str] = [] - - for idx, message in enumerate(messages): - content = message.content or "" - if message.role == "system" or (message.role != "user" and idx in [0, 1]): - if content is not None: - system_messages.append(content) # type: ignore - continue - elif message.role == "user": - if isinstance(content, str): - content = [{"type": "text", "text": content}] - - if message.images is not None: - for image in message.images: - image_content = format_image_for_message(image) - if image_content: - content.append(image_content) - - # Handle tool calls from history - elif message.role == "assistant" and isinstance(message.content, str) and message.tool_calls: - if message.content: - content = [TextBlock(text=message.content, type="text")] - else: - content = [] - for tool_call in message.tool_calls: - content.append( - ToolUseBlock( - id=tool_call["id"], - input=json.loads(tool_call["function"]["arguments"]), - name=tool_call["function"]["name"], - type="tool_use", - ) - ) - - chat_messages.append({"role": message.role, "content": content}) # type: ignore - return chat_messages, " ".join(system_messages) - def prepare_request_kwargs(self, system_message: str) -> Dict[str, Any]: """ Prepare the request keyword arguments for the API call. @@ -274,7 +275,7 @@ def invoke(self, messages: List[Message]) -> AnthropicMessage: Returns: AnthropicMessage: The response from the model. """ - chat_messages, system_message = self.format_messages(messages) + chat_messages, system_message = _format_messages(messages) request_kwargs = self.prepare_request_kwargs(system_message) return self.get_client().messages.create( @@ -293,7 +294,7 @@ def invoke_stream(self, messages: List[Message]) -> Any: Returns: Any: The streamed response from the model. """ - chat_messages, system_message = self.format_messages(messages) + chat_messages, system_message = _format_messages(messages) request_kwargs = self.prepare_request_kwargs(system_message) return self.get_client().messages.stream( diff --git a/libs/agno/agno/models/mistral/mistral.py b/libs/agno/agno/models/mistral/mistral.py index 660ff60466..d16c6226b2 100644 --- a/libs/agno/agno/models/mistral/mistral.py +++ b/libs/agno/agno/models/mistral/mistral.py @@ -2,6 +2,7 @@ from os import getenv from typing import Any, Dict, Iterator, List, Optional, Union +from agno.media import Image from agno.models.base import Model, StreamData from agno.models.message import Message from agno.models.response import ModelResponse, ModelResponseEvent @@ -12,7 +13,7 @@ try: from mistralai import Mistral as MistralClient - from mistralai.models import AssistantMessage, SystemMessage, ToolMessage, UserMessage + from mistralai.models import AssistantMessage, SystemMessage, ToolMessage, UserMessage, TextChunk, ImageURLChunk from mistralai.models.chatcompletionresponse import ChatCompletionResponse from mistralai.models.deltamessage import DeltaMessage from mistralai.types.basemodel import Unset @@ -21,6 +22,59 @@ MistralMessage = Union[UserMessage, AssistantMessage, SystemMessage, ToolMessage] +def _format_image_for_message(image: Image) -> Optional[ImageURLChunk]: + # Case 1: Image is a URL + if image.url is not None: + return ImageURLChunk(image_url=image.url) + # Case 2: Image is a local file path + elif image.filepath is not None: + import base64 + from pathlib import Path + + path = Path(image.filepath) if isinstance(image.filepath, str) else image.filepath + if not path.exists() or not path.is_file(): + logger.error(f"Image file not found: {image}") + raise FileNotFoundError(f"Image file not found: {image}") + + with open(image.filepath, "rb") as image_file: + base64_image = base64.b64encode(image_file.read()).decode('utf-8') + return ImageURLChunk(image_url=f"data:image/jpeg;base64,{base64_image}") + + # Case 3: Image is a bytes object + elif image.content is not None: + import base64 + base64_image = base64.b64encode(image.content).decode('utf-8') + return ImageURLChunk(image_url=f"data:image/jpeg;base64,{base64_image}") + return None + +def _format_messages(messages: List[Message]) -> List[MistralMessage]: + mistral_messages: List[MistralMessage] = [] + for message in messages: + mistral_message: MistralMessage + if message.role == "user": + if message.images is not None: + content = [TextChunk(type="text", text=message.content)] + for image in message.images: + image_content = _format_image_for_message(image) + if image_content: + content.append(image_content) + mistral_message = UserMessage(role="user", content=content) + else: + mistral_message = UserMessage(role="user", content=message.content) + elif message.role == "assistant": + if message.tool_calls is not None: + mistral_message = AssistantMessage(role="assistant", content=message.content, tool_calls=message.tool_calls) + else: + mistral_message = AssistantMessage(role=message.role, content=message.content) + elif message.role == "system": + mistral_message = SystemMessage(role="system", content=message.content) + elif message.role == "tool": + mistral_message = ToolMessage(name="tool", content=message.content, tool_call_id=message.tool_call_id) + else: + raise ValueError(f"Unknown role: {message.role}") + mistral_messages.append(mistral_message) + return mistral_messages + @dataclass class MistralChat(Model): @@ -151,26 +205,6 @@ def to_dict(self) -> Dict[str, Any]: cleaned_dict = {k: v for k, v in _dict.items() if v is not None} return cleaned_dict - def _prepare_messages(self, messages: List[Message]) -> List[MistralMessage]: - mistral_messages: List[MistralMessage] = [] - for m in messages: - mistral_message: MistralMessage - if m.role == "user": - mistral_message = UserMessage(role=m.role, content=m.content) - elif m.role == "assistant": - if m.tool_calls is not None: - mistral_message = AssistantMessage(role=m.role, content=m.content, tool_calls=m.tool_calls) - else: - mistral_message = AssistantMessage(role=m.role, content=m.content) - elif m.role == "system": - mistral_message = SystemMessage(role=m.role, content=m.content) - elif m.role == "tool": - mistral_message = ToolMessage(name=m.name, content=m.content, tool_call_id=m.tool_call_id) - else: - raise ValueError(f"Unknown role: {m.role}") - mistral_messages.append(mistral_message) - return mistral_messages - def invoke(self, messages: List[Message]) -> ChatCompletionResponse: """ Send a chat completion request to the Mistral model. @@ -181,7 +215,7 @@ def invoke(self, messages: List[Message]) -> ChatCompletionResponse: Returns: ChatCompletionResponse: The response from the model. """ - mistral_messages = self._prepare_messages(messages) + mistral_messages = _format_messages(messages) response = self.client.chat.complete( model=self.id, messages=mistral_messages, @@ -201,7 +235,7 @@ def invoke_stream(self, messages: List[Message]) -> Iterator[Any]: Returns: Iterator[Any]: The streamed response. """ - mistral_messages = self._prepare_messages(messages) + mistral_messages = _format_messages(messages) response = self.client.chat.stream( model=self.id, messages=mistral_messages, From 3e73112d8388a18252f5f88d6ace403311cb9c0e Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Fri, 31 Jan 2025 12:37:17 +0200 Subject: [PATCH 2/7] Update --- .gitignore | 1 + agno.code-workspace | 15 --------------- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 agno.code-workspace diff --git a/.gitignore b/.gitignore index 83be36f0a5..dc58c5fb48 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ # Machine specific .idea .vscode +*.code-workspace # Ignore .env files .env diff --git a/agno.code-workspace b/agno.code-workspace deleted file mode 100644 index 1e3fd025ca..0000000000 --- a/agno.code-workspace +++ /dev/null @@ -1,15 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": { - "python.analysis.extraPaths": [ - "libs/agno", - "libs/infra/agno_docker", - "libs/infra/agno_aws" - ] - } -} - From e575d8e5e9f19bfaf860048b8bc311820c246be1 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Fri, 31 Jan 2025 12:53:11 +0200 Subject: [PATCH 3/7] Update cookbooks --- .../models/mistral/image_bytes_input_agent.py | 16 +++++++++------- cookbook/models/mistral/image_compare_agent.py | 8 ++++++-- .../models/mistral/image_file_input_agent.py | 4 +++- .../mistral/image_ocr_with_structured_output.py | 9 +++++---- libs/agno/agno/models/mistral/mistral.py | 15 ++++++++++----- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/cookbook/models/mistral/image_bytes_input_agent.py b/cookbook/models/mistral/image_bytes_input_agent.py index 28694bae8f..3af8ccad73 100644 --- a/cookbook/models/mistral/image_bytes_input_agent.py +++ b/cookbook/models/mistral/image_bytes_input_agent.py @@ -1,9 +1,8 @@ import requests - from agno.agent import Agent from agno.media import Image -from agno.tools.duckduckgo import DuckDuckGoTools from agno.models.mistral.mistral import MistralChat +from agno.tools.duckduckgo import DuckDuckGoTools agent = Agent( model=MistralChat(id="pixtral-12b-2409"), @@ -11,18 +10,22 @@ markdown=True, ) -image_url = "https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg" +image_url = ( + "https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg" +) + def fetch_image_bytes(url: str) -> bytes: headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", } response = requests.get(url, headers=headers) response.raise_for_status() return response.content + image_bytes_from_url = fetch_image_bytes(image_url) agent.print_response( @@ -31,4 +34,3 @@ def fetch_image_bytes(url: str) -> bytes: Image(content=image_bytes_from_url), ], ) - diff --git a/cookbook/models/mistral/image_compare_agent.py b/cookbook/models/mistral/image_compare_agent.py index 6323eb6295..2f99df9144 100644 --- a/cookbook/models/mistral/image_compare_agent.py +++ b/cookbook/models/mistral/image_compare_agent.py @@ -10,8 +10,12 @@ agent.print_response( "what are the differences between two images?", images=[ - Image(url="https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg"), - Image(url="https://assets.visitorscoverage.com/production/wp-content/uploads/2024/04/AdobeStock_626542468-min-1024x683.jpeg"), + Image( + url="https://tripfixers.com/wp-content/uploads/2019/11/eiffel-tower-with-snow.jpeg" + ), + Image( + url="https://assets.visitorscoverage.com/production/wp-content/uploads/2024/04/AdobeStock_626542468-min-1024x683.jpeg" + ), ], stream=True, ) diff --git a/cookbook/models/mistral/image_file_input_agent.py b/cookbook/models/mistral/image_file_input_agent.py index 3c558755c7..98c4b8a256 100644 --- a/cookbook/models/mistral/image_file_input_agent.py +++ b/cookbook/models/mistral/image_file_input_agent.py @@ -7,7 +7,9 @@ agent = Agent( model=MistralChat(id="pixtral-12b-2409"), - tools=[DuckDuckGoTools()], # pixtral-12b-2409 is not so great at tool calls, but it might work. + tools=[ + DuckDuckGoTools() + ], # pixtral-12b-2409 is not so great at tool calls, but it might work. show_tool_calls=True, markdown=True, ) diff --git a/cookbook/models/mistral/image_ocr_with_structured_output.py b/cookbook/models/mistral/image_ocr_with_structured_output.py index 439ffadb6d..b300d93451 100644 --- a/cookbook/models/mistral/image_ocr_with_structured_output.py +++ b/cookbook/models/mistral/image_ocr_with_structured_output.py @@ -1,19 +1,22 @@ from typing import List -from pydantic import BaseModel from agno.agent import Agent from agno.media import Image from agno.models.mistral.mistral import MistralChat +from pydantic import BaseModel + class GroceryItem(BaseModel): item_name: str price: float + class GroceryListElements(BaseModel): bill_number: str items: List[GroceryItem] total_price: float + agent = Agent( model=MistralChat(id="pixtral-12b-2409"), instructions=[ @@ -25,7 +28,5 @@ class GroceryListElements(BaseModel): agent.print_response( "From this restaurant bill, extract the bill number, item names and associated prices, and total price and return it as a string in a Json object", - images=[ - Image(url="https://i.imghippo.com/files/kgXi81726851246.jpg") - ], + images=[Image(url="https://i.imghippo.com/files/kgXi81726851246.jpg")], ) diff --git a/libs/agno/agno/models/mistral/mistral.py b/libs/agno/agno/models/mistral/mistral.py index d16c6226b2..86acc0b246 100644 --- a/libs/agno/agno/models/mistral/mistral.py +++ b/libs/agno/agno/models/mistral/mistral.py @@ -13,7 +13,7 @@ try: from mistralai import Mistral as MistralClient - from mistralai.models import AssistantMessage, SystemMessage, ToolMessage, UserMessage, TextChunk, ImageURLChunk + from mistralai.models import AssistantMessage, ImageURLChunk, SystemMessage, TextChunk, ToolMessage, UserMessage from mistralai.models.chatcompletionresponse import ChatCompletionResponse from mistralai.models.deltamessage import DeltaMessage from mistralai.types.basemodel import Unset @@ -22,6 +22,7 @@ MistralMessage = Union[UserMessage, AssistantMessage, SystemMessage, ToolMessage] + def _format_image_for_message(image: Image) -> Optional[ImageURLChunk]: # Case 1: Image is a URL if image.url is not None: @@ -35,18 +36,20 @@ def _format_image_for_message(image: Image) -> Optional[ImageURLChunk]: if not path.exists() or not path.is_file(): logger.error(f"Image file not found: {image}") raise FileNotFoundError(f"Image file not found: {image}") - + with open(image.filepath, "rb") as image_file: - base64_image = base64.b64encode(image_file.read()).decode('utf-8') + base64_image = base64.b64encode(image_file.read()).decode("utf-8") return ImageURLChunk(image_url=f"data:image/jpeg;base64,{base64_image}") # Case 3: Image is a bytes object elif image.content is not None: import base64 - base64_image = base64.b64encode(image.content).decode('utf-8') + + base64_image = base64.b64encode(image.content).decode("utf-8") return ImageURLChunk(image_url=f"data:image/jpeg;base64,{base64_image}") return None + def _format_messages(messages: List[Message]) -> List[MistralMessage]: mistral_messages: List[MistralMessage] = [] for message in messages: @@ -63,7 +66,9 @@ def _format_messages(messages: List[Message]) -> List[MistralMessage]: mistral_message = UserMessage(role="user", content=message.content) elif message.role == "assistant": if message.tool_calls is not None: - mistral_message = AssistantMessage(role="assistant", content=message.content, tool_calls=message.tool_calls) + mistral_message = AssistantMessage( + role="assistant", content=message.content, tool_calls=message.tool_calls + ) else: mistral_message = AssistantMessage(role=message.role, content=message.content) elif message.role == "system": From 9d6f6c07fcd008ea2fa3d9ed0eb6cbf7d3d71eea Mon Sep 17 00:00:00 2001 From: Manthan Gupta Date: Mon, 3 Feb 2025 17:01:49 +0530 Subject: [PATCH 4/7] update --- ...anscribe_document_agent => image_transcribe_document_agent.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cookbook/models/mistral/{image_transcribe_document_agent => image_transcribe_document_agent.py} (100%) diff --git a/cookbook/models/mistral/image_transcribe_document_agent b/cookbook/models/mistral/image_transcribe_document_agent.py similarity index 100% rename from cookbook/models/mistral/image_transcribe_document_agent rename to cookbook/models/mistral/image_transcribe_document_agent.py From 26907b14df495df344b33aa1e77765b091b2b59f Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Mon, 3 Feb 2025 14:01:51 +0200 Subject: [PATCH 5/7] update --- ...anscribe_document_agent => image_transcribe_document_agent.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cookbook/models/mistral/{image_transcribe_document_agent => image_transcribe_document_agent.py} (100%) diff --git a/cookbook/models/mistral/image_transcribe_document_agent b/cookbook/models/mistral/image_transcribe_document_agent.py similarity index 100% rename from cookbook/models/mistral/image_transcribe_document_agent rename to cookbook/models/mistral/image_transcribe_document_agent.py From 38e60f24911a2475a105c55e37130e5a3c3f6847 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Mon, 3 Feb 2025 14:04:59 +0200 Subject: [PATCH 6/7] update style --- cookbook/tools/exa_tools.py | 11 +++++++++-- libs/agno/agno/models/mistral/mistral.py | 2 +- libs/agno/agno/tools/exa.py | 1 - 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cookbook/tools/exa_tools.py b/cookbook/tools/exa_tools.py index c0adab6297..defca9feaf 100644 --- a/cookbook/tools/exa_tools.py +++ b/cookbook/tools/exa_tools.py @@ -2,13 +2,20 @@ from agno.tools.exa import ExaTools agent = Agent( - tools=[ExaTools(include_domains=["cnbc.com", "reuters.com", "bloomberg.com"], show_results=True)], + tools=[ + ExaTools( + include_domains=["cnbc.com", "reuters.com", "bloomberg.com"], + show_results=True, + ) + ], show_tool_calls=True, ) agent.print_response("Search for AAPL news", markdown=True) -agent.print_response("What is the paper at https://arxiv.org/pdf/2307.06435 about?", markdown=True) +agent.print_response( + "What is the paper at https://arxiv.org/pdf/2307.06435 about?", markdown=True +) agent.print_response( "Find me similar papers to https://arxiv.org/pdf/2307.06435 and provide a summary of what they contain", diff --git a/libs/agno/agno/models/mistral/mistral.py b/libs/agno/agno/models/mistral/mistral.py index 4976cf5da2..9a6caee291 100644 --- a/libs/agno/agno/models/mistral/mistral.py +++ b/libs/agno/agno/models/mistral/mistral.py @@ -56,7 +56,7 @@ def _format_messages(messages: List[Message]) -> List[MistralMessage]: mistral_message: MistralMessage if message.role == "user": if message.images is not None: - content = [TextChunk(type="text", text=message.content)] + content: List[Any] = [TextChunk(type="text", text=message.content)] for image in message.images: image_content = _format_image_for_message(image) if image_content: diff --git a/libs/agno/agno/tools/exa.py b/libs/agno/agno/tools/exa.py index 4ead6ccfbb..2159b7fce8 100644 --- a/libs/agno/agno/tools/exa.py +++ b/libs/agno/agno/tools/exa.py @@ -83,7 +83,6 @@ def __init__( self.include_domains: Optional[List[str]] = include_domains self.exclude_domains: Optional[List[str]] = exclude_domains - if search: self.register(self.search_exa) if get_contents: From 413226f7f0b61fee7e7d669265dc6ea7ba08a667 Mon Sep 17 00:00:00 2001 From: Dirk Brand Date: Mon, 3 Feb 2025 15:04:44 +0200 Subject: [PATCH 7/7] Merge --- libs/agno/agno/models/anthropic/claude.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/agno/agno/models/anthropic/claude.py b/libs/agno/agno/models/anthropic/claude.py index af16a11c6b..12d4b0d782 100644 --- a/libs/agno/agno/models/anthropic/claude.py +++ b/libs/agno/agno/models/anthropic/claude.py @@ -674,7 +674,7 @@ async def ainvoke(self, messages: List[Message]) -> AnthropicMessage: Returns: AnthropicMessage: The response from the model. """ - chat_messages, system_message = self.format_messages(messages) + chat_messages, system_message = _format_messages(messages) request_kwargs = self.prepare_request_kwargs(system_message) return await self.get_async_client().messages.create( @@ -693,7 +693,7 @@ async def ainvoke_stream(self, messages: List[Message]) -> Any: Returns: Any: The streamed response from the model. """ - chat_messages, system_message = self.format_messages(messages) + chat_messages, system_message = _format_messages(messages) request_kwargs = self.prepare_request_kwargs(system_message) return self.get_async_client().messages.stream(