Add support for returning multiple messages from gr.ChatInterface chat function

.changeset/famous-shoes-lose.md

@@ -0,0 +1,5 @@
+---
+"gradio": minor
+---
+
+feat:Add support for returning multiple messages from `gr.ChatInterface` chat function

demo/chatinterface_echo_multimodal/run.ipynb

@@ -0,0 +1 @@
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatinterface_echo_multimodal"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "def echo_multimodal(message, history):\n", "    response = []\n", "    response.append(\"You wrote: '\" + message[\"text\"] + \"' and uploaded:\")\n", "    for file in message.get(\"files\", []):\n", "        response.append((file, ))\n", "    return response\n", "\n", "demo = gr.ChatInterface(\n", "    echo_multimodal,\n", "    type=\"messages\",\n", "    multimodal=True,\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/chatinterface_echo_multimodal/run.py

@@ -0,0 +1,17 @@
+import gradio as gr
+
+def echo_multimodal(message, history):
+    response = []
+    response.append("You wrote: '" + message["text"] + "' and uploaded:")
+    for file in message.get("files", []):
+        response.append((file, ))
+    return response
+
+demo = gr.ChatInterface(
+    echo_multimodal,
+    type="messages",
+    multimodal=True,
+)
+
+if __name__ == "__main__":
+    demo.launch()

demo/chatinterface_options/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatinterface_options"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "\n", "example_code = \"\"\"\n", "Here's the code I generated:\n", "\n", "```python\n", "def greet(x):\n", "    return f\"Hello, {x}!\"\n", "```\n", "\n", "Is this correct?\n", "\"\"\"\n", "\n", "def chat(message, history):\n", "    if message == \"Yes, that's correct.\":\n", "        return \"Great!\"\n", "    else:\n", "        return {\n", "            \"role\": \"assistant\",\n", "            \"content\": example_code,\n", "            \"options\": [\n", "                {\"value\": \"Yes, that's correct.\", \"label\": \"Yes\"},\n", "                {\"value\": \"No\"}\n", "                ]\n", "            }\n", "\n", "demo = gr.ChatInterface(\n", "    chat,\n", "    type=\"messages\",\n", "    examples=[\"Write a Python function that takes a string and returns a greeting.\"]\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: chatinterface_options"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import random\n", "\n", "example_code = \"\"\"\n", "Here's an example Python lambda function:\n", "\n", "lambda x: x + {}\n", "\n", "Is this correct?\n", "\"\"\"\n", "\n", "def chat(message, history):\n", "    if message == \"Yes, that's correct.\":\n", "        return \"Great!\"\n", "    else:\n", "        return {\n", "            \"role\": \"assistant\",\n", "            \"content\": example_code.format(random.randint(1, 100)),\n", "            \"options\": [\n", "                {\"value\": \"Yes, that's correct.\", \"label\": \"Yes\"},\n", "                {\"value\": \"No\"}\n", "                ]\n", "            }\n", "\n", "demo = gr.ChatInterface(\n", "    chat,\n", "    type=\"messages\",\n", "    examples=[\"Write an example Python lambda function.\"]\n", ")\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/chatinterface_options/run.py

@@ -1,12 +1,10 @@
 import gradio as gr
+import random
 
 example_code = """
-Here's the code I generated:
+Here's an example Python lambda function:
 
-```python
-def greet(x):
-    return f"Hello, {x}!"
-```
+lambda x: x + {}
 
 Is this correct?
 """
@@ -17,7 +15,7 @@ def chat(message, history):
     else:
         return {
             "role": "assistant",
-            "content": example_code,
+            "content": example_code.format(random.randint(1, 100)),
             "options": [
                 {"value": "Yes, that's correct.", "label": "Yes"},
                 {"value": "No"}
@@ -27,7 +25,7 @@ def chat(message, history):
 demo = gr.ChatInterface(
     chat,
     type="messages",
-    examples=["Write a Python function that takes a string and returns a greeting."]
+    examples=["Write an example Python lambda function."]
 )
 
 if __name__ == "__main__":

gradio/chat_interface.py

@@ -103,7 +103,7 @@ def __init__(
     ):
         """
         Parameters:
-            fn: the function to wrap the chat interface around. In the default case (assuming `type` is set to "messages"), the function should accept two parameters: a `str` input message and `list` of openai-style dictionary {"role": "user" | "assistant", "content": `str` | {"path": `str`} | `gr.Component`} representing the chat history, and return/yield a `str` (if a simple message) or `dict` (for a complete openai-style message) response.
+            fn: the function to wrap the chat interface around. In the default case (assuming `type` is set to "messages"), the function should accept two parameters: a `str` input message and `list` of openai-style dictionary {"role": "user" | "assistant", "content": `str` | {"path": `str`} | `gr.Component`} representing the chat history, and the function should return/yield a `str` (if a simple message), a single-element tuple (for a file), a `dict` (for a complete openai-style message) response, or a `list` of such messages.
             multimodal: if True, the chat interface will use a `gr.MultimodalTextbox` component for the input, which allows for the uploading of multimedia files. If False, the chat interface will use a gr.Textbox component for the input. If this is True, the first argument of `fn` should accept not a `str` message but a `dict` message with keys "text" and "files"
             type: The format of the messages passed into the chat history parameter of `fn`. If "messages", passes the history as a list of dictionaries with openai-style "role" and "content" keys. The "content" key's value should be one of the following - (1) strings in valid Markdown (2) a dictionary with a "path" key and value corresponding to the file to display or (3) an instance of a Gradio component: at the moment gr.Image, gr.Plot, gr.Video, gr.Gallery, gr.Audio, and gr.HTML are supported. The "role" key should be one of 'user' or 'assistant'. Any other roles will not be displayed in the output. If this parameter is 'tuples' (deprecated), passes the chat history as a `list[list[str | None | tuple]]`, i.e. a list of lists. The inner list should have 2 elements: the user message and the response message.
             chatbot: an instance of the gr.Chatbot component to use for the chat interface, if you would like to customize the chatbot properties. If not provided, a default gr.Chatbot component will be created.
@@ -596,10 +596,11 @@ def response_as_dict(
     ) -> MessageDict:
         if isinstance(response, Message):
             new_response = response.model_dump()
-        elif isinstance(response, (str, Component)):
+        elif isinstance(response, (tuple, str, Component)):
             return {"role": "assistant", "content": response}
         else:
             new_response = response
+            new_response["role"] = "assistant"
         return cast(MessageDict, new_response)
 
     async def _submit_fn(
@@ -618,13 +619,16 @@ async def _submit_fn(
             response = await anyio.to_thread.run_sync(
                 self.fn, *inputs, limiter=self.limiter
             )
-        if isinstance(response, tuple):
+        if self.additional_outputs:
             response, *additional_outputs = response
         else:
             additional_outputs = None
         history = self._append_message_to_history(message, history, "user")
-        response_ = self.response_as_dict(response)
-        history = self._append_message_to_history(response_, history, "assistant")  # type: ignore
+        response_ = [response] if not isinstance(response, list) else response
+        for r in response_:
+            history = self._append_message_to_history(
+                self.response_as_dict(r), history, "assistant"
+            )
         if additional_outputs:
             return response, history, *additional_outputs
         return response, history
@@ -654,7 +658,7 @@ async def _stream_fn(
         additional_outputs = None
         try:
             first_response = await utils.async_iteration(generator)
-            if isinstance(first_response, tuple):
+            if self.additional_outputs:
                 first_response, *additional_outputs = first_response
             history_ = self._append_message_to_history(
                 first_response, history, "assistant"
@@ -666,7 +670,7 @@ async def _stream_fn(
         except StopIteration:
             yield None, history
         async for response in generator:
-            if isinstance(response, tuple):
+            if self.additional_outputs:
                 response, *additional_outputs = response
             history_ = self._append_message_to_history(response, history, "assistant")
             if not additional_outputs:
@@ -784,7 +788,7 @@ def _pop_last_user_message(
         history: list[MessageDict] | TupleFormat,
     ) -> tuple[list[MessageDict] | TupleFormat, str | MultimodalPostprocess]:
         """
-        Removes the last user message from the chat history and returns it.
+        Removes the message (or set of messages) that the user last sent from the chat history and returns them.
         If self.multimodal is True, returns a MultimodalPostprocess (dict) object with text and files.
         If self.multimodal is False, returns just the message text as a string.
         """
@@ -793,8 +797,9 @@ def _pop_last_user_message(
 
         if self.type == "tuples":
             history = self._tuples_to_messages(history)  # type: ignore
-        # Skip the last message as it's always an assistant message
-        i = len(history) - 2
+        i = len(history) - 1
+        while i >= 0 and history[i]["role"] == "assistant":  # type: ignore
+            i -= 1
         while i >= 0 and history[i]["role"] == "user":  # type: ignore
             i -= 1
         last_messages = history[i + 1 :]

guides/05_chatbots/01_creating-a-chatbot-fast.md

@@ -303,77 +303,43 @@ gr.ChatInterface(
 
 **Returning image, audio, video, or other files**:
 
-Sometimes, you don't want to return a complete Gradio component, but rather simply an image/audio/video/other file to be displayed inside the chatbot. You can do this by returning a complete openai-style dictionary from your chat function. The dictionary should consist of the following keys:
-
-* `role`: set to `"assistant"`
-* `content`: set to a dictionary with key `path` and value the filepath or URL you'd like to return
+Sometimes, you don't want to return a complete Gradio component, but rather simply an image/audio/video/other file to be displayed inside the chatbot. You can do this by returning a single-element tuple consisting of the string file path or URL.
 
 Here is an example:
 
 ```py
 import gradio as gr
 
-def fake(message, history):
+def artist(message, history):
     if message.strip():
-        return {
-            "role": "assistant", 
-            "content": {
-                "path": "https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav"
-                }
-            }
+        return ("https://github.com/gradio-app/gradio/raw/main/test/test_files/audio_sample.wav", )
     else:
         return "Please provide the name of an artist"
 
 gr.ChatInterface(
-    fake,
+    artist,
     type="messages",
     textbox=gr.Textbox(placeholder="Which artist's music do you want to listen to?", scale=7),
     chatbot=gr.Chatbot(placeholder="Play music by any artist!"),
 ).launch()
 ```
 
-
 **Providing preset responses**
 
-You may want to provide preset responses that a user can choose between when conversing with your chatbot. You can add the `options` key to the dictionary returned from your chat function to set these responses. The value corresponding to the `options` key should be a list of dictionaries, each with a `value` (a string that is the value that should be sent to the chat function when this response is clicked) and an optional `label` (if provided, is the text displayed as the preset response instead of the `value`). 
-
-This example illustrates how to use preset responses:
-
-```python
-import gradio as gr
+You may want to provide preset responses that a user can choose between when conversing with your chatbot. To do this, return a complete openai-style message dictionary from your chat function, and add the `options` key to the dictionary returned from your chat function to set these responses. 
 
-example_code = '''
-Here's the code I generated:
+The value corresponding to the `options` key should be a list of dictionaries, each with a `value` (a string that is the value that should be sent to the chat function when this response is clicked) and an optional `label` (if provided, is the text displayed as the preset response instead of the `value`). 
 
-def greet(x):
-    return f"Hello, {x}!"
+This example illustrates how to use preset responses:
 
-Is this correct?
-'''
+$code_chatinterface_options
 
-def chat(message, history):
-    if message == "Yes, that's correct.":
-        return "Great!"
-    else:
-        return {
-            "role": "assistant",
-            "content": example_code,
-            "options": [
-                {"value": "Yes, that's correct.", "label": "Yes"},
-                {"value": "No"}
-                ]
-            }
+**Returning Multiple Messages**
 
-demo = gr.ChatInterface(
-    chat,
-    type="messages",
-    examples=["Write a Python function that takes a string and returns a greeting."]
-)
+You can return multiple assistant messages from your chat function simply by returning a `list` of messages of any of the above types (you can even mix-and-match). This lets you, for example, send a message along with files, as in the following example:
 
-if __name__ == "__main__":
-    demo.launch()
+$code_chatinterface_echo_multimodal
 
-```
 ## Using Your Chatbot via API
 
 Once you've built your Gradio chat interface and are hosting it on [Hugging Face Spaces](https://hf.space) or somewhere else, then you can query it with a simple API at the `/chat` endpoint. The endpoint just expects the user's message (and potentially additional inputs if you have set any using the `additional_inputs` parameter), and will return the response, internally keeping track of the messages sent so far.

test/test_chat_interface.py

@@ -4,6 +4,7 @@
 from unittest.mock import patch
 
 import pytest
+from gradio_client import handle_file
 
 import gradio as gr
 
@@ -320,6 +321,60 @@ def double_multimodal(msg, history):
             result = client.predict({"text": "hello", "files": []}, api_name="/chat")
             assert result == "hello hello"
 
+    @pytest.mark.parametrize("type", ["tuples", "messages"])
+    def test_files_returned(self, type, connect):
+        def echo_first_file(msg, history):
+            return (msg["files"][0],)
+
+        chatbot = gr.ChatInterface(
+            echo_first_file,
+            type=type,
+            multimodal=True,
+        )
+        with connect(chatbot) as client:
+            result = client.predict(
+                {
+                    "text": "hello",
+                    "files": [handle_file("test/test_files/audio_sample.wav")],
+                },
+                api_name="/chat",
+            )
+            assert result[0].endswith("audio_sample.wav")
+
+    @pytest.mark.parametrize("type", ["tuples", "messages"])
+    def test_component_returned(self, type, connect):
+        def mock_chat_fn(msg, history):
+            return gr.Audio("test/test_files/audio_sample.wav")
+
+        chatbot = gr.ChatInterface(
+            mock_chat_fn,
+            type=type,
+            multimodal=True,
+        )
+        with connect(chatbot) as client:
+            result = client.predict(
+                {
+                    "text": "hello",
+                    "files": [handle_file("test/test_files/audio_sample.wav")],
+                },
+                api_name="/chat",
+            )
+            assert result["value"] == "test/test_files/audio_sample.wav"
+
+    @pytest.mark.parametrize("type", ["tuples", "messages"])
+    def test_multiple_messages(self, type, connect):
+        def multiple_messages(msg, history):
+            return [msg["text"], msg["text"]]
+
+        chatbot = gr.ChatInterface(
+            multiple_messages,
+            type=type,
+            multimodal=True,
+        )
+        with connect(chatbot) as client:
+            result = client.predict({"text": "hello", "files": []}, api_name="/chat")
+            assert result == ["hello", "hello"]
+
 
 class TestExampleMessages:
     def test_setup_example_messages_with_strings(self):