From 9bdec8c8edab9d88080d4dbd2eff65a5bf7b2628 Mon Sep 17 00:00:00 2001 From: Freddy Boulton Date: Mon, 23 Dec 2024 12:25:16 -0500 Subject: [PATCH] Update Transformers agent demo (#9885) * update code * modify imports * fix * update notebook * gr.load_chat: Allow loading any openai-compatible server immediately as a ChatInterface (#10222) * changes * add changeset * add changeset * Update gradio/external.py Co-authored-by: Abubakar Abid * changes * changes * Update guides/05_chatbots/01_creating-a-chatbot-fast.md Co-authored-by: Abubakar Abid * changes --------- Co-authored-by: Ali Abid Co-authored-by: gradio-pr-bot Co-authored-by: Abubakar Abid * Allow editing chatbot messages (#10203) * changes * add changeset * changes * changes * changes * changes * changes * changes * changes * Update gradio/events.py Co-authored-by: Abubakar Abid * Update gradio/components/chatbot.py Co-authored-by: Abubakar Abid * changes --------- Co-authored-by: Ali Abid Co-authored-by: gradio-pr-bot Co-authored-by: Abubakar Abid * Update Guides related to deploying Gradio chatbots to Discord, Slack, and website widgets (#10221) * changes * changes * update * chat history * add changeset * changes * add changeset * changes * guide * changes * changes * changes * guide * add images * slack guide * changes * format * add changeset * finish up slack * changes * improve js guide * update * changes * script * changes * changes * add changeset * Update guides/05_chatbots/01_creating-a-chatbot-fast.md Co-authored-by: Ali Abdalla * Update guides/05_chatbots/01_creating-a-chatbot-fast.md Co-authored-by: Ali Abdalla * Update guides/05_chatbots/06_creating-a-discord-bot-from-a-gradio-app.md Co-authored-by: Ali Abdalla * Update guides/05_chatbots/08_creating-a-website-widget-from-a-gradio-chatbot.md Co-authored-by: Ali Abdalla * Update guides/05_chatbots/08_creating-a-website-widget-from-a-gradio-chatbot.md Co-authored-by: Ali Abdalla * gradio * cta --------- Co-authored-by: gradio-pr-bot Co-authored-by: Ali Abdalla * Allow editable ChatInterface (#10229) * changes * add changeset * changes * changes * changes --------- Co-authored-by: Ali Abid Co-authored-by: gradio-pr-bot * Fix typing for components in `gr.Interface` and docstring in `image.py` (#10235) * changes * add changeset * changes * changes * change * add changeset * image.py * revert msg --------- Co-authored-by: gradio-pr-bot * Lite: Capture stdout and stderr from the main thread (#9984) * Add stdout and stderr events * add changeset * Refactoring * Format App.tsx * add changeset * Add python-error event to capture Python errors occurring in the running event loop after the initial app launch * Fix 's close button * Fix * Propagate python-error and initialization-error events to the controller * Add init-code|file-run-error events --------- Co-authored-by: gradio-pr-bot * use chat interface --------- Co-authored-by: Abubakar Abid Co-authored-by: aliabid94 Co-authored-by: Ali Abid Co-authored-by: gradio-pr-bot Co-authored-by: Ali Abdalla Co-authored-by: Yuichiro Tachibana (Tsuchiya) --- demo/agent_chatbot/requirements.txt | 2 +- demo/agent_chatbot/run.ipynb | 2 +- demo/agent_chatbot/run.py | 50 ++++++++++----- demo/agent_chatbot/utils.py | 64 ------------------- .../05_chatbots/03_agents-and-tool-usage.md | 49 ++++++++------ 5 files changed, 66 insertions(+), 101 deletions(-) delete mode 100644 demo/agent_chatbot/utils.py diff --git a/demo/agent_chatbot/requirements.txt b/demo/agent_chatbot/requirements.txt index 646219d1bb0ae..9ed70ee033a18 100644 --- a/demo/agent_chatbot/requirements.txt +++ b/demo/agent_chatbot/requirements.txt @@ -1 +1 @@ -git+https://github.com/huggingface/transformers.git#egg=transformers[agents] \ No newline at end of file +transformers>=4.47.0 \ No newline at end of file diff --git a/demo/agent_chatbot/run.ipynb b/demo/agent_chatbot/run.ipynb index edfb71b31c84e..ccf9c2fe27edc 100644 --- a/demo/agent_chatbot/run.ipynb +++ b/demo/agent_chatbot/run.ipynb @@ -1 +1 @@ -{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: agent_chatbot"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio git+https://github.com/huggingface/transformers.git#egg=transformers[agents]"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["# Downloading files from the demo repo\n", "import os\n", "!wget -q https://github.com/gradio-app/gradio/raw/main/demo/agent_chatbot/utils.py"]}, {"cell_type": "code", "execution_count": null, "id": "44380577570523278879349135829904343037", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from gradio import ChatMessage\n", "from transformers import load_tool, ReactCodeAgent, HfEngine # type: ignore\n", "from utils import stream_from_transformers_agent\n", "\n", "# Import tool from Hub\n", "image_generation_tool = load_tool(\"m-ric/text-to-image\")\n", "\n", "llm_engine = HfEngine(\"meta-llama/Meta-Llama-3-70B-Instruct\")\n", "# Initialize the agent with both tools\n", "agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine)\n", "\n", "def interact_with_agent(prompt, messages):\n", " messages.append(ChatMessage(role=\"user\", content=prompt))\n", " yield messages\n", " for msg in stream_from_transformers_agent(agent, prompt):\n", " messages.append(msg)\n", " yield messages\n", " yield messages\n", "\n", "with gr.Blocks() as demo:\n", " stored_message = gr.State([])\n", " chatbot = gr.Chatbot(label=\"Agent\",\n", " type=\"messages\",\n", " avatar_images=(None, \"https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png\"))\n", " text_input = gr.Textbox(lines=1, label=\"Chat Message\")\n", " text_input.submit(lambda s: (s, \"\"), [text_input], [stored_message, text_input]).then(interact_with_agent, [stored_message, chatbot], [chatbot])\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file +{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: agent_chatbot"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio transformers>=4.47.0"]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "from dataclasses import asdict\n", "from transformers import Tool, ReactCodeAgent # type: ignore\n", "from transformers.agents import stream_to_gradio, HfApiEngine # type: ignore\n", "\n", "# Import tool from Hub\n", "image_generation_tool = Tool.from_space(\n", " space_id=\"black-forest-labs/FLUX.1-schnell\",\n", " name=\"image_generator\",\n", " description=\"Generates an image following your prompt. Returns a PIL Image.\",\n", " api_name=\"/infer\",\n", ")\n", "\n", "llm_engine = HfApiEngine(\"Qwen/Qwen2.5-Coder-32B-Instruct\")\n", "# Initialize the agent with both tools and engine\n", "agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine)\n", "\n", "\n", "def interact_with_agent(prompt, history):\n", " messages = []\n", " yield messages\n", " for msg in stream_to_gradio(agent, prompt):\n", " messages.append(asdict(msg))\n", " yield messages\n", " yield messages\n", "\n", "\n", "demo = gr.ChatInterface(\n", " interact_with_agent,\n", " chatbot= gr.Chatbot(\n", " label=\"Agent\",\n", " type=\"messages\",\n", " avatar_images=(\n", " None,\n", " \"https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png\",\n", " ),\n", " ),\n", " examples=[\n", " [\"Generate an image of an astronaut riding an alligator\"],\n", " [\"I am writing a children's book for my daughter. Can you help me with some illustrations?\"],\n", " ],\n", " type=\"messages\",\n", ")\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file diff --git a/demo/agent_chatbot/run.py b/demo/agent_chatbot/run.py index c240d8d81460b..4e6cf36214536 100644 --- a/demo/agent_chatbot/run.py +++ b/demo/agent_chatbot/run.py @@ -1,30 +1,46 @@ import gradio as gr -from gradio import ChatMessage -from transformers import load_tool, ReactCodeAgent, HfEngine # type: ignore -from utils import stream_from_transformers_agent +from dataclasses import asdict +from transformers import Tool, ReactCodeAgent # type: ignore +from transformers.agents import stream_to_gradio, HfApiEngine # type: ignore # Import tool from Hub -image_generation_tool = load_tool("m-ric/text-to-image") +image_generation_tool = Tool.from_space( + space_id="black-forest-labs/FLUX.1-schnell", + name="image_generator", + description="Generates an image following your prompt. Returns a PIL Image.", + api_name="/infer", +) -llm_engine = HfEngine("meta-llama/Meta-Llama-3-70B-Instruct") -# Initialize the agent with both tools +llm_engine = HfApiEngine("Qwen/Qwen2.5-Coder-32B-Instruct") +# Initialize the agent with both tools and engine agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine) -def interact_with_agent(prompt, messages): - messages.append(ChatMessage(role="user", content=prompt)) + +def interact_with_agent(prompt, history): + messages = [] yield messages - for msg in stream_from_transformers_agent(agent, prompt): - messages.append(msg) + for msg in stream_to_gradio(agent, prompt): + messages.append(asdict(msg)) yield messages yield messages -with gr.Blocks() as demo: - stored_message = gr.State([]) - chatbot = gr.Chatbot(label="Agent", - type="messages", - avatar_images=(None, "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png")) - text_input = gr.Textbox(lines=1, label="Chat Message") - text_input.submit(lambda s: (s, ""), [text_input], [stored_message, text_input]).then(interact_with_agent, [stored_message, chatbot], [chatbot]) + +demo = gr.ChatInterface( + interact_with_agent, + chatbot= gr.Chatbot( + label="Agent", + type="messages", + avatar_images=( + None, + "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png", + ), + ), + examples=[ + ["Generate an image of an astronaut riding an alligator"], + ["I am writing a children's book for my daughter. Can you help me with some illustrations?"], + ], + type="messages", +) if __name__ == "__main__": demo.launch() diff --git a/demo/agent_chatbot/utils.py b/demo/agent_chatbot/utils.py deleted file mode 100644 index 6d4d5493488b7..0000000000000 --- a/demo/agent_chatbot/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# type: ignore -from __future__ import annotations - -from gradio import ChatMessage -from transformers.agents import ReactCodeAgent, agent_types -from typing import Generator - -def pull_message(step_log: dict): - if step_log.get("rationale"): - yield ChatMessage( - role="assistant", content=step_log["rationale"] - ) - if step_log.get("tool_call"): - used_code = step_log["tool_call"]["tool_name"] == "code interpreter" - content = step_log["tool_call"]["tool_arguments"] - if used_code: - content = f"```py\n{content}\n```" - yield ChatMessage( - role="assistant", - metadata={"title": f"🛠️ Used tool {step_log['tool_call']['tool_name']}"}, - content=content, - ) - if step_log.get("observation"): - yield ChatMessage( - role="assistant", content=f"```\n{step_log['observation']}\n```" - ) - if step_log.get("error"): - yield ChatMessage( - role="assistant", - content=str(step_log["error"]), - metadata={"title": "💥 Error"}, - ) - -def stream_from_transformers_agent( - agent: ReactCodeAgent, prompt: str -) -> Generator[ChatMessage, None, ChatMessage | None]: - """Runs an agent with the given prompt and streams the messages from the agent as ChatMessages.""" - - class Output: - output: agent_types.AgentType | str = None - - step_log = None - for step_log in agent.run(prompt, stream=True): - if isinstance(step_log, dict): - for message in pull_message(step_log): - print("message", message) - yield message - - Output.output = step_log - if isinstance(Output.output, agent_types.AgentText): - yield ChatMessage( - role="assistant", content=f"**Final answer:**\n```\n{Output.output.to_string()}\n```") # type: ignore - elif isinstance(Output.output, agent_types.AgentImage): - yield ChatMessage( - role="assistant", - content={"path": Output.output.to_string(), "mime_type": "image/png"}, # type: ignore - ) - elif isinstance(Output.output, agent_types.AgentAudio): - yield ChatMessage( - role="assistant", - content={"path": Output.output.to_string(), "mime_type": "audio/wav"}, # type: ignore - ) - else: - return ChatMessage(role="assistant", content=Output.output) diff --git a/guides/05_chatbots/03_agents-and-tool-usage.md b/guides/05_chatbots/03_agents-and-tool-usage.md index fa3be443b2317..66b2caee756a7 100644 --- a/guides/05_chatbots/03_agents-and-tool-usage.md +++ b/guides/05_chatbots/03_agents-and-tool-usage.md @@ -35,37 +35,50 @@ We'll start by importing the necessary classes from transformers and gradio. ```python import gradio as gr from gradio import ChatMessage -from transformers import load_tool, ReactCodeAgent, HfEngine -from utils import stream_from_transformers_agent +from transformers import Tool, ReactCodeAgent # type: ignore +from transformers.agents import stream_to_gradio, HfApiEngine # type: ignore # Import tool from Hub -image_generation_tool = load_tool("m-ric/text-to-image") - +image_generation_tool = Tool.from_space( + space_id="black-forest-labs/FLUX.1-schnell", + name="image_generator", + description="Generates an image following your prompt. Returns a PIL Image.", + api_name="/infer", +) -llm_engine = HfEngine("meta-llama/Meta-Llama-3-70B-Instruct") -# Initialize the agent with both tools +llm_engine = HfApiEngine("Qwen/Qwen2.5-Coder-32B-Instruct") +# Initialize the agent with both tools and engine agent = ReactCodeAgent(tools=[image_generation_tool], llm_engine=llm_engine) ``` -Then we'll build the UI. The bulk of the logic is handled by `stream_from_transformers_agent`. We won't cover it in this guide because it will soon be merged to transformers but you can see its source code [here](https://huggingface.co/spaces/gradio/agent_chatbot/blob/main/utils.py). +Then we'll build the UI: ```python -def interact_with_agent(prompt, messages): - messages.append(ChatMessage(role="user", content=prompt)) +def interact_with_agent(prompt, history): + messages = [] yield messages - for msg in stream_from_transformers_agent(agent, prompt): - messages.append(msg) + for msg in stream_to_gradio(agent, prompt): + messages.append(asdict(msg)) yield messages yield messages -with gr.Blocks() as demo: - stored_message = gr.State([]) - chatbot = gr.Chatbot(label="Agent", - type="messages", - avatar_images=(None, "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png")) - text_input = gr.Textbox(lines=1, label="Chat Message") - text_input.submit(lambda s: (s, ""), [text_input], [stored_message, text_input]).then(interact_with_agent, [stored_message, chatbot], [chatbot]) +demo = gr.ChatInterface( + interact_with_agent, + chatbot= gr.Chatbot( + label="Agent", + type="messages", + avatar_images=( + None, + "https://em-content.zobj.net/source/twitter/53/robot-face_1f916.png", + ), + ), + examples=[ + ["Generate an image of an astronaut riding an alligator"], + ["I am writing a children's book for my daughter. Can you help me with some illustrations?"], + ], + type="messages", +) ``` You can see the full demo code [here](https://huggingface.co/spaces/gradio/agent_chatbot/blob/main/app.py).