Python: intro allowed content types in chat history channel receive. …

…Add mixed chat image sample. (#10347)

### Motivation and Context

During a group chat, any file reference content created by an assistant
agent, doesn't need to be communicated to a chat completion agent.
Filter these types out and only include other types, like text, if
available.

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

### Description

This PR:
- Adds a mixed chat image sample to have an assistant agent generate an
image, along with text, and call the chat completion agent successfully
with only the allowed types (like text).
- Adds a unit test to exercise the same behavior.
- Closes #10317

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [X] The code builds clean without any errors or warnings
- [X] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [X] All unit tests pass, and I have added new tests where possible
- [X] I didn't break anyone 😄

python/samples/concepts/README.md

@@ -14,6 +14,7 @@
 - [Mixed Chat Agents](./agents/mixed_chat_agents.py)
 - [Mixed Chat Agents Plugins](./agents/mixed_chat_agents_plugins.py)
 - [Mixed Chat Files](./agents/mixed_chat_files.py)
+- [Mixed Chat Images](./agents/mixed_chat_images.py)
 - [Mixed Chat Reset](./agents/mixed_chat_reset.py)
 - [Mixed Chat Streaming](./agents/mixed_chat_streaming.py)
 

python/samples/concepts/agents/mixed_chat_images.py

@@ -0,0 +1,96 @@
+# Copyright (c) Microsoft. All rights reserved.
+
+import asyncio
+
+from semantic_kernel.agents import AgentGroupChat, ChatCompletionAgent
+from semantic_kernel.agents.open_ai import OpenAIAssistantAgent
+from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent
+from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion
+from semantic_kernel.contents.annotation_content import AnnotationContent
+from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.utils.author_role import AuthorRole
+from semantic_kernel.kernel import Kernel
+
+#####################################################################
+# The following sample demonstrates how to create an OpenAI         #
+# assistant using either Azure OpenAI or OpenAI, a chat completion  #
+# agent and have them participate in a group chat working with      #
+# image content.                                                    #
+#####################################################################
+
+
+def _create_kernel_with_chat_completion(service_id: str) -> Kernel:
+    kernel = Kernel()
+    kernel.add_service(AzureChatCompletion(service_id=service_id))
+    return kernel
+
+
+async def invoke_agent(
+    chat: AgentGroupChat, agent: ChatCompletionAgent | OpenAIAssistantAgent, input: str | None = None
+) -> None:
+    """Invoke the agent with the user input."""
+    if input:
+        await chat.add_chat_message(message=ChatMessageContent(role=AuthorRole.USER, content=input))
+        print(f"# {AuthorRole.USER}: '{input}'")
+
+    async for content in chat.invoke(agent=agent):
+        print(f"# {content.role} - {content.name or '*'}: '{content.content}'")
+        if len(content.items) > 0:
+            for item in content.items:
+                if isinstance(item, AnnotationContent):
+                    print(f"\n`{item.quote}` => {item.file_id}")
+                    response_content = await agent.client.files.content(item.file_id)
+                    print(response_content.text)
+
+
+async def main():
+    try:
+        ANALYST_NAME = "Analyst"
+        ANALYST_INSTRUCTIONS = "Create charts as requested without explanation."
+        analyst_agent = await AzureAssistantAgent.create(
+            kernel=Kernel(),
+            enable_code_interpreter=True,
+            name=ANALYST_NAME,
+            instructions=ANALYST_INSTRUCTIONS,
+        )
+
+        SUMMARIZER_NAME = "Summarizer"
+        SUMMARIZER_INSTRUCTIONS = "Summarize the entire conversation for the user in natural language."
+        service_id = "summary"
+        summary_agent = ChatCompletionAgent(
+            service_id=service_id,
+            kernel=_create_kernel_with_chat_completion(service_id=service_id),
+            instructions=SUMMARIZER_INSTRUCTIONS,
+            name=SUMMARIZER_NAME,
+        )
+
+        chat = AgentGroupChat()
+
+        await invoke_agent(
+            chat=chat,
+            agent=analyst_agent,
+            input="""
+            Graph the percentage of storm events by state using a pie chart:
+
+                State, StormCount
+                TEXAS, 4701
+                KANSAS, 3166
+                IOWA, 2337
+                ILLINOIS, 2022
+                MISSOURI, 2016
+                GEORGIA, 1983
+                MINNESOTA, 1881
+                WISCONSIN, 1850
+                NEBRASKA, 1766
+                NEW YORK, 1750
+            """,
+        )
+        await invoke_agent(chat=chat, agent=summary_agent)
+    finally:
+        if analyst_agent is not None:
+            [await analyst_agent.delete_file(file_id=file_id) for file_id in analyst_agent.code_interpreter_file_ids]
+            await analyst_agent.delete()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

python/semantic_kernel/agents/channels/chat_history_channel.py

@@ -3,14 +3,19 @@
 import sys
 from collections import deque
 from collections.abc import AsyncIterable
+from copy import deepcopy
+
+from semantic_kernel.contents.image_content import ImageContent
+from semantic_kernel.contents.streaming_text_content import StreamingTextContent
+from semantic_kernel.contents.text_content import TextContent
 
 if sys.version_info >= (3, 12):
     from typing import override  # pragma: no cover
 else:
     from typing_extensions import override  # pragma: no cover
 
 from abc import abstractmethod
-from typing import TYPE_CHECKING, Deque, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, ClassVar, Deque, Protocol, runtime_checkable
 
 from semantic_kernel.agents.channels.agent_channel import AgentChannel
 from semantic_kernel.contents import ChatMessageContent
@@ -45,6 +50,14 @@ def invoke_stream(self, history: "ChatHistory") -> AsyncIterable["ChatMessageCon
 class ChatHistoryChannel(AgentChannel, ChatHistory):
     """An AgentChannel specialization for that acts upon a ChatHistoryHandler."""
 
+    ALLOWED_CONTENT_TYPES: ClassVar[tuple[type, ...]] = (
+        ImageContent,
+        FunctionCallContent,
+        FunctionResultContent,
+        StreamingTextContent,
+        TextContent,
+    )
+
     @override
     async def invoke(
         self,
@@ -142,10 +155,23 @@ async def receive(
     ) -> None:
         """Receive the conversation messages.
 
+        Do not include messages that only contain file references.
+
         Args:
             history: The history of messages in the conversation.
         """
-        self.messages.extend(history)
+        filtered_history: list[ChatMessageContent] = []
+        for message in history:
+            new_message = deepcopy(message)
+            if new_message.items is None:
+                new_message.items = []
+            allowed_items = [item for item in new_message.items if isinstance(item, self.ALLOWED_CONTENT_TYPES)]
+            if not allowed_items:
+                continue
+            new_message.items.clear()
+            new_message.items.extend(allowed_items)
+            filtered_history.append(new_message)
+        self.messages.extend(filtered_history)
 
     @override
     async def get_history(  # type: ignore

python/tests/unit/agents/test_chat_history_channel.py

@@ -7,7 +7,9 @@
 
 from semantic_kernel.agents.channels.chat_history_channel import ChatHistoryAgentProtocol, ChatHistoryChannel
 from semantic_kernel.contents.chat_message_content import ChatMessageContent
+from semantic_kernel.contents.file_reference_content import FileReferenceContent
 from semantic_kernel.contents.function_result_content import FunctionResultContent
+from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent
 from semantic_kernel.contents.utils.author_role import AuthorRole
 from semantic_kernel.exceptions import ServiceInvalidTypeError
 
@@ -200,3 +202,47 @@ async def test_reset_history():
     await channel.reset()
 
     assert len(channel.messages) == 0
+
+
+async def test_receive_skips_file_references():
+    channel = ChatHistoryChannel()
+
+    file_ref_item = FileReferenceContent()
+    streaming_file_ref_item = StreamingFileReferenceContent()
+    normal_item_1 = FunctionResultContent(id="test_id", result="normal content 1")
+    normal_item_2 = FunctionResultContent(id="test_id_2", result="normal content 2")
+
+    msg_with_file_only = ChatMessageContent(
+        role=AuthorRole.USER,
+        content="Normal message set as TextContent",
+        items=[file_ref_item],
+    )
+
+    msg_with_mixed = ChatMessageContent(
+        role=AuthorRole.USER,
+        content="Mixed content message",
+        items=[streaming_file_ref_item, normal_item_1],
+    )
+
+    msg_with_normal = ChatMessageContent(
+        role=AuthorRole.USER,
+        content="Normal message",
+        items=[normal_item_2],
+    )
+
+    history = [msg_with_file_only, msg_with_mixed, msg_with_normal]
+    await channel.receive(history)
+
+    assert len(channel.messages) == 3
+
+    assert channel.messages[0].content == "Normal message set as TextContent"
+    assert len(channel.messages[0].items) == 1
+
+    assert channel.messages[1].content == "Mixed content message"
+    assert len(channel.messages[0].items) == 1
+    assert channel.messages[1].items[0].result == "normal content 1"
+
+    assert channel.messages[2].content == "Normal message"
+    assert len(channel.messages[2].items) == 2
+    assert channel.messages[2].items[0].result == "normal content 2"
+    assert channel.messages[2].items[1].text == "Normal message"