Merge pull request #1 from tsunghan-wu/handle_function_native_multiturn

Merge handle function native multiturn

README.md

@@ -1,4 +1,12 @@
 # FastChat
+
+## Latest Update and TODOs
+
+- [ ] Enable google serach function call (by 10/28/2024)
+    - [x] Modify the fastchat codebase to support function calling during the chat for OpenAI GPT-4. Please see more at (./docs/agent.md)
+    - [ ] Complete the google search function. Currently, it's a prototpye function at [`fastchat/tools/search.py`](./fastchat/tools/search.py)
+    - [ ] Make the agent call scalable for all LLMs.
+
 | [**Demo**](https://lmarena.ai/) | [**Discord**](https://discord.gg/6GXcFg3TH8) | [**X**](https://x.com/lmsysorg) |
 
 FastChat is an open platform for training, serving, and evaluating large language model based chatbots.

agent_api_endpoints.json

@@ -0,0 +1,17 @@
+{
+  "react-agent": {
+    "model_name": "gpt-4o-2024-08-06",
+    "api_type": "openai",
+    "api_base": "https://api.openai.com/v1",
+    "api_key": "YOUR API KEY",
+    "anony_only": false,
+    "recommended_config": {
+      "temperature": 0.7,
+      "top_p": 1.0
+    },
+    "text-arena": true,
+    "vision-arena": false,
+    "agent-mode": true,
+    "custom_system_prompt": true
+  }
+}

docs/agent.md

@@ -0,0 +1,76 @@
+# Agent Arena Working Area
+
+**Note**: Please install packages and ensure you can successfully execute [Launch a WebUI with an API Model](https://github.com/tsunghan-wu/Agent_FastChat/blob/main/docs/model_support.md#api-based-models).
+
+## Launch agent-enabled Chatbot Arena (for OpenAI APIs currently)
+
+1. Specify the endpoint information in a JSON configuration file. For instance, create a file named `agent_api_endpoints.json`:
+
+```
+{
+  "gpt4o": {
+    "model_name": "gpt-4o-2024-08-06",
+    "api_type": "openai",
+    "api_base": "https://api.openai.com/v1",
+    "api_key": "sk-******",
+    "anony_only": false,
+    "recommended_config": {
+      "temperature": 0.7,
+      "top_p": 1.0
+    },
+    "text-arena": true,
+    "vision-arena": false,
+    "agent-mode": true
+  }
+}
+```
+
+2. Launch the Gradio web server with the argument `--register api_endpoints.json`:
+
+```bash
+python3 -m fastchat.serve.gradio_web_server_agent --controller "" --share --register agent_api_endpoints.json
+```
+
+Now, you can open a browser and interact with the model.
+
+## Examples:
+
+1. Using Agents
+
+```
+User: What's the weather today?
+
+GPT-4: 
+{
+    "thought": "The query asks about the current weather, which is dynamic and location-specific information. Since I don't have direct access to real-time weather data, I should use the available tool to search for the current weather conditions. The 'google_search' tool can help find this information by querying with relevant keywords.",
+    "action": {
+        "name": "google_search",
+        "reason": "To obtain the current weather information from online sources.",
+        "arguments": {
+            "key_words": ["current weather"],
+            "topk": 1
+        }
+    }
+}
+
+Then, we'll execute the google search function in the fastchat/tools/search.py where it currently only return a pre-defined string. You can also conduct follow-up (multi-round) chatting, it won't be a problem.
+```
+
+2. Without using agents
+
+```
+User: How are you?
+GPT-4:
+{
+    "thought": "The query is asking for a status update or well-being check on myself as an assistant. This is a common conversational question and doesn't require additional information from external sources. I can answer this directly based on my designed functionality.",
+    "answer": "I'm just a virtual assistant, so I don't have feelings or states of being, but I'm here and ready to help you with any questions or tasks you have!"
+}
+```
+
+TODOs:
+
+- [ ] Complete the function in `fastchat/tools/search.py`
+- [ ] Discuss the format of "history message" in the prompt. Also, we should probably refine the frontend interface. Reference: https://github.com/reworkd/AgentGPT
+- [ ] Discuss how to scale the searching functions to all LLMs and the code structure.
+
+- Note: Please run `./format.sh` before merging into the main branch.

fastchat/conversation.py

@@ -55,6 +55,47 @@ class Conversation:
     # The system message
     system_message: str = ""
     system_message_vision: str = ""
+    # Agent template
+    agent_prompt: str = """You are a helpful assistant. Your goal is to reason about the user query and decide on the best course of action to answer it accurately.
+
+Available Tools (function call)
+- Tools are invoked using the format: tool_name(**kwargs) -> Executes the specified tool with provided arguments.
+
+- web_search(key_words: list[str], topk: int = 3) -> Returns search results from Google.
+
+Instructions:
+1. Analyze the query, previous reasoning steps, and observations.
+2. Decide on the next action: use a tool or provide a final answer.
+3. Respond using the following JSON format:
+
+If you need to use a tool:
+{
+    "thought": "Detailed reasoning explaining your next steps",
+    "action": {
+        "name": "Tool name (choose from available tools)",
+        "reason": "Why you chose this tool",
+        "arguments": {{
+            "arg_name_1": "value_1",
+            "arg_name_2": "value_2",
+            ...
+        }
+    }
+}
+
+If you have enough information to answer the query:
+{
+    "thought": "Your reasoning process leading to the conclusion",
+    "answer": "A thorough and well-supported answer"
+}
+
+Key Points to Remember:
+- Be thorough in your reasoning.
+- Use tools when you need more information.
+- Always base your reasoning on the actual observations from tool use.
+- If a tool returns no results or fails, acknowledge this and consider using a different tool or approach.
+- Provide a final answer only when you're confident you have sufficient information.
+- If you cannot find the necessary information after using available tools, admit that you don't have enough information to answer the query confidently."""
+
     # The names of two roles
     roles: Tuple[str] = ("USER", "ASSISTANT")
     # All messages. Each item is (role, message).
@@ -437,6 +478,26 @@ def to_openai_api_messages(self):
                     ret.append({"role": "assistant", "content": msg})
         return ret
 
+    def to_openai_agent_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        if self.system_message == "":
+            ret = [{"role": "system", "content": self.agent_prompt}]
+        else:
+            ret = [
+                {
+                    "role": "system",
+                    "content": self.system_message + "\n\n" + self.agent_prompt,
+                }
+            ]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({"role": "user", "content": msg})
+            else:
+                if msg is not None:
+                    ret.append({"role": "assistant", "content": msg})
+        return ret
+
     def to_gemini_api_messages(self):
         from fastchat.utils import load_image
 
@@ -721,6 +782,18 @@ def get_conv_template(name: str) -> Conversation:
     )
 )
 
+# A template for ReAct agent.
+register_conv_template(
+    Conversation(
+        name="react-agent",
+        system_message=Conversation.agent_prompt,
+        roles=("user", "assistant"),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n### ",
+        stop_str="###",
+    )
+)
+
 # A template with a one-shot conversation example
 register_conv_template(
     Conversation(

fastchat/model/model_adapter.py

@@ -144,6 +144,14 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
         return get_conv_template("one_shot")
 
 
+class ReActAgentAdapter(BaseModelAdapter):
+    def match(self, model_path: str):
+        return "react-agent" in model_path.lower()
+
+    def get_default_conv_template(self, model_path: str) -> Conversation:
+        return get_conv_template("react-agent")
+
+
 # A global registry for all model adapters
 # TODO (lmzheng): make it a priority queue.
 model_adapters: List[BaseModelAdapter] = []
@@ -2602,6 +2610,7 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
 register_model_adapter(Llama31Adapter)
 register_model_adapter(GrokAdapter)
 register_model_adapter(NoSystemAdapter)
+register_model_adapter(ReActAgentAdapter)
 
 # After all adapters, try the default base adapter.
 register_model_adapter(BaseModelAdapter)

fastchat/serve/api_provider.py

@@ -27,6 +27,8 @@ def get_api_provider_stream_iter(
     if model_api_dict["api_type"] == "openai":
         if model_api_dict.get("vision-arena", False):
             prompt = conv.to_openai_vision_api_messages()
+        elif model_api_dict.get("agent-mode", False):
+            prompt = conv.to_openai_agent_api_messages()
         else:
             prompt = conv.to_openai_api_messages()
         stream_iter = openai_api_stream_iter(
@@ -281,6 +283,7 @@ def openai_api_stream_iter(
 
     # Make requests for logging
     text_messages = []
+    print(messages)
     for message in messages:
         if type(message["content"]) == str:  # text-only model
             text_messages.append(message)