[BFCL] Multi Turn Dataset and Possible Answer Fix (Base Category) (Sh…

…ishirPatil#719) This PR addresses a few issues with the base multi turn entries: 1. Some initial config values are in the wrong format, and would result in execution error even when running the ground truth function calls in order. 2. Some entries in the initial config are not used by the `_load_scenario` method. 3. Some ground truth function calls have wrong parameter values, and would result in `{"error": "xxx"}` when executed. 4. Some functions are unreasonable in a real-life setting with its parameter or its functionality. After this PR, all info in the initial config are used, and the ground truth function calls will not results in any error when executed in order. This is part of the effort to thoroughly bug fix the multi turn categories. We will have more PR coming in the next few days. --------- Co-authored-by: Charlie Cheng-Jie Ji <[email protected]> Co-authored-by: Fanjia-Yan <[email protected]> Co-authored-by: VishnuSuresh27 <[email protected]>
VishnuSuresh27 · Nov 11, 2024 · e7e0fdd · e7e0fdd
1 parent 1ed7d87
commit e7e0fdd
Show file tree

Hide file tree

Showing 10 changed files with 552 additions and 400 deletions.
diff --git a/berkeley-function-call-leaderboard/CHANGELOG.md b/berkeley-function-call-leaderboard/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 All notable changes to the Berkeley Function Calling Leaderboard will be documented in this file.
 
+- [Oct 24, 2024] [#719](https://github.com/ShishirPatil/gorilla/pull/719): Bug fix in the dataset and ground truth for the multi-turn categories.
 - [Oct 17, 2024] [#683](https://github.com/ShishirPatil/gorilla/pull/683): Bug fix for the multi turn categories for ambiguity in action intention and function parameters.
 - [Oct 17, 2024] [#709](https://github.com/ShishirPatil/gorilla/pull/709): Rephrase question prompt for Java and JavaScript categories to improve clarity and action intent.
 - [Oct 17, 2024] [#708](https://github.com/ShishirPatil/gorilla/pull/708): Update the ground truth for the REST category to be up-to-date with the latest API response structure.

diff --git a/...all-leaderboard/bfcl/eval_checker/multi_turn_eval/func_source_code/gorilla_file_system.py b/...all-leaderboard/bfcl/eval_checker/multi_turn_eval/func_source_code/gorilla_file_system.py
@@ -1,5 +1,6 @@
 import datetime
 import subprocess
+from copy import deepcopy
 from typing import Dict, List, Optional, Union
 
 from .long_context import FILE_CONTENT_EXTENSION, FILES_TAIL_USED
@@ -129,6 +130,10 @@ def __eq__(self, other: object) -> bool:
         return self.name == other.name and self.contents == other.contents
 
 
+DEFAULT_STATE = {
+    "root": Directory("/", None)
+}
+
 class GorillaFileSystem:
 
     def __init__(self) -> None:
@@ -183,16 +188,16 @@ def _load_scenario(self, scenario: dict, long_context: bool = False) -> None:
             }
         }
         """
+        DEFAULT_STATE_COPY = deepcopy(DEFAULT_STATE)
         self.long_context = long_context
-        self.root = Directory("/")
-        self.long_context = long_context
+        self.root = DEFAULT_STATE_COPY["root"]
         if "root" in scenario:
             root_dir = Directory(list(scenario["root"].keys())[0], None)
             self.root = self._load_directory(
                 scenario["root"][list(scenario["root"].keys())[0]]["contents"],
                 root_dir
             )
-            self._current_dir = self.root
+        self._current_dir = self.root
 
     def _load_directory(
         self, current: dict, parent: Optional[Directory] = None

diff --git a/...nction-call-leaderboard/bfcl/eval_checker/multi_turn_eval/func_source_code/message_api.py b/...nction-call-leaderboard/bfcl/eval_checker/multi_turn_eval/func_source_code/message_api.py
@@ -1,6 +1,7 @@
 import random
-from typing import Dict, List, Optional, Union
 from copy import deepcopy
+from typing import Dict, List, Optional, Union
+
 DEFAULT_STATE = {
     "generated_ids": set(),
     "user_count": 4,
@@ -10,29 +11,22 @@
         "Catherine": "USR003",
         "Daniel": "USR004",
     },
-    "inbox": {
-        1: {
-            "sender_id": "USR001",
-            "receiver_id": "USR002",
-            "message": "My name is Alice. I want to connect.",
+    "inbox": [
+        {
+            "USR002": "My name is Alice. I want to connect.",
         },
-        2: {
-            "sender_id": "USR001",
-            "receiver_id": "USR003",
-            "message": "Could you upload the file?",
+        {
+            "USR003": "Could you upload the file?",
         },
-        3: {
-            "sender_id": "USR001",
-            "receiver_id": "USR004",
-            "message": "Could you upload the file?",
+        {
+            "USR004": "Could you upload the file?",
         },
-        4: {"sender_id": "USR001", "receiver_id": "USR002", "message": "I am busy."},
-        5: {"sender_id": "USR001", "receiver_id": "USR002", "message": "I am on leave."},
-    },
-    "message_count": 0,
+    ],
+    "message_count": 3,
     "current_user": None,
 }
-
+
+
 class MessageAPI:
     """
     A class representing a Message API for managing user interactions in a workspace.
@@ -53,7 +47,6 @@ class MessageAPI:
         get_user_id(user: str): Get the user ID for a given username.
         login(user_id: str): Log in a user.
         send_message(receiver_id: str, message: str): Send a message to another user.
-        view_messages_received(): View messages received by the current user.
         view_messages_sent(): View messages sent by the current user.
         delete_message(receiver_id: str, message_index: int): Delete a sent message.
         add_contact(name: str, user_id: str): Add a new contact to the workspace.
@@ -65,13 +58,12 @@ def __init__(self):
         """
         Initialize the MessageAPI with a workspace ID.
         """
-        DEFAULT_STATE_COPY = deepcopy(DEFAULT_STATE)
-        self.generated_ids = DEFAULT_STATE_COPY["generated_ids"]
-        self.user_count: int = DEFAULT_STATE_COPY["user_count"]
-        self.user_map: Dict[str, str] = DEFAULT_STATE_COPY["user_map"]
-        self.inbox: Dict[int, Dict[str, Union[str, int]]] = DEFAULT_STATE_COPY["inbox"]
-        self.message_count: int = DEFAULT_STATE_COPY["message_count"]
-        self.current_user: Optional[str] = DEFAULT_STATE_COPY["current_user"]
+        self.generated_ids: set
+        self.user_count: int
+        self.user_map: Dict[str, str]
+        self.inbox: List[Dict[str, str]]
+        self.message_count: int
+        self.current_user: Optional[str]
 
     def _load_scenario(self, scenario: dict, long_context=False) -> None:
         """
@@ -82,9 +74,16 @@ def _load_scenario(self, scenario: dict, long_context=False) -> None:
         """
         DEFAULT_STATE_COPY = deepcopy(DEFAULT_STATE)
         self._random = random.Random((scenario.get("random_seed", 200191)))
+        self.generated_ids = scenario.get(
+            "generated_ids", DEFAULT_STATE_COPY["generated_ids"]
+        )
         self.user_count = scenario.get("user_count", DEFAULT_STATE_COPY["user_count"])
-        self.current_user = scenario.get("current_user", DEFAULT_STATE_COPY["current_user"])
         self.user_map = scenario.get("user_map", DEFAULT_STATE_COPY["user_map"])
+        self.inbox = scenario.get("inbox", DEFAULT_STATE_COPY["inbox"])
+        self.message_count = scenario.get(
+            "message_count", DEFAULT_STATE_COPY["message_count"]
+        )
+        self.current_user = scenario.get("current_user", DEFAULT_STATE_COPY["current_user"])
 
     def __eq__(self, value: object) -> bool:
         if not isinstance(value, MessageAPI):
@@ -155,9 +154,16 @@ def message_login(self, user_id: str) -> Dict[str, Union[str, bool]]:
             "message": f"User '{user_id}' logged in successfully.",
         }
 
-    def send_message(
-        self, receiver_id: str, message: str
-    ) -> Dict[str, Union[str, bool]]:
+    def message_get_login_status(self) -> Dict[str, bool]:
+        """
+        Get the login status of the current user.
+
+        Returns:
+            login_status (bool): True if the current user is logged in, False otherwise.
+        """
+        return {"login_status": bool(self.current_user)}
+
+    def send_message(self, receiver_id: str, message: str) -> Dict[str, Union[str, bool]]:
         """
         Send a message to a user.
         Args:
@@ -177,23 +183,17 @@ def send_message(
         # Generate a unique message ID
         message_id = self.generate_id()
         # Store the message in the inbox
-        self.inbox[message_id] = {
-            "sender_id": self.current_user,
-            "receiver_id": receiver_id,
-            "message": message,
-        }
+        self.inbox.append({receiver_id: message})
         self.message_count += 1
         return {
             "sent_status": True,
             "message_id": message_id,
             "message": f"Message sent to '{receiver_id}' successfully.",
         }
 
-    def delete_message(
-        self, receiver_id: str, message_id: int
-    ) -> Dict[str, Union[bool, str]]:
+    def delete_message(self, receiver_id: str) -> Dict[str, Union[bool, str]]:
         """
-        Delete a message sent to a user.
+        Delete the latest message sent to a receiver.
         Args:
             receiver_id (str): User ID of the user to send the message to.
             message_id (int): ID of the message to be deleted.
@@ -204,51 +204,18 @@ def delete_message(
         """
         if not self.current_user:
             return {"error": "No user is currently logged in."}
-        # Check if the message exists in the inbox
-        if message_id not in self.inbox:
-            return {"error": f"Message ID {message_id} not found."}
-        # Ensure the current user is either the sender or receiver of the message
-        message_data = self.inbox[message_id]
-        if (
-            message_data["sender_id"] != self.current_user
-            and message_data["receiver_id"] != self.current_user
-        ):
-            return {"error": "You do not have permission to delete this message."}
-        # Check if the sender and receiver match the input arguments
-        if (
-            message_data["sender_id"] != self.current_user
-            or message_data["receiver_id"] != receiver_id
-        ):
-            return {
-                "error": f"Message ID {message_id} does not match the provided sender and receiver."
-            }
-        # If everything checks out, delete the message
-        del self.inbox[message_id]
-        return {
-            "deleted_status": True,
-            "message_id": message_id,
-            "message": f"Message ID {message_id} deleted successfully.",
-        }
 
-    def view_messages_received(self) -> Dict[str, Union[Dict[str, List[str]], str]]:
-        """
-        View all messages sent to the current user.
-
-        Returns:
-            messages (Dict[str, List[str]]): Dictionary of senders and their messages sent to the current user.
-        """
-        if not self.current_user:
-            return {"error": "No user is currently logged in."}
-        # Dictionary to collect messages grouped by sender
-        received_messages = {}
-        # Loop through the inbox and collect messages sent to the current user
-        for message_id, message_data in self.inbox.items():
-            if message_data["receiver_id"] == self.current_user:
-                sender = message_data["sender_id"]
-                if sender not in received_messages:
-                    received_messages[sender] = []
-                received_messages[sender].append(message_data["message"])
-        return {"messages": received_messages}
+        # Loop through the inbox in reverse order to find the first message sent to the receiver
+        for message in self.inbox[::-1]:
+            receiver, _ = list(message.items())[0]
+            if receiver == receiver_id:
+                self.inbox.remove(message)
+                return {
+                    "deleted_status": True,
+                    "message_id": receiver,
+                    "message": f"Receiver {receiver_id}'s first message deleted successfully.",
+                }
+        return {"error": f"Receiver ID {receiver_id} not found."}
 
     def view_messages_sent(self) -> Dict[str, Union[Dict[str, List[str]], str]]:
         """
@@ -262,12 +229,12 @@ def view_messages_sent(self) -> Dict[str, Union[Dict[str, List[str]], str]]:
         # Dictionary to collect messages grouped by receiver
         sent_messages = {}
         # Loop through the inbox and collect messages sent by the current user
-        for message_id, message_data in self.inbox.items():
-            if message_data["sender_id"] == self.current_user:
-                receiver = message_data["receiver_id"]
-                if receiver not in sent_messages:
-                    sent_messages[receiver] = []
-                sent_messages[receiver].append(message_data["message"])
+        for message in self.inbox:
+            receiver, message_content = list(message.items())[0]
+            if receiver not in sent_messages:
+                sent_messages[receiver] = [message_content]
+            else:
+                sent_messages[receiver].append(message_content)
         return {"messages": sent_messages}
 
     def add_contact(self, user_name: str) -> Dict[str, Union[bool, str]]:
@@ -302,7 +269,6 @@ def search_messages(
             keyword (str): The keyword to search for in messages.
         Returns:
             results (List[Dict[str, Union[str, List[str]]]]): List of dictionaries containing matching messages.
-                Each dictionary has keys:
                 - sender_id (str): The ID of the user who sent the message.
                 - receiver_id (str): The ID of the user who received the message.
                 - messages (List[str]): List of messages containing the keyword.
@@ -312,21 +278,16 @@ def search_messages(
         keyword_lower = keyword.lower()
         results = []
         # Iterate through the inbox to search for the keyword in messages
-        for message_id, message_data in self.inbox.items():
-            # Check if the current user is either the sender or receiver
-            if (
-                message_data["sender_id"] == self.current_user
-                or message_data["receiver_id"] == self.current_user
-            ):
-                # Check if the message contains the keyword (case-insensitive)
-                if keyword_lower in message_data["message"].lower():
-                    results.append(
-                        {
-                            "sender_id": message_data["sender_id"],
-                            "receiver_id": message_data["receiver_id"],
-                            "message": message_data["message"],
-                        }
-                    )
+        # for message_id, message_data in self.inbox.items():
+        for message_data in self.inbox:
+            receiver_id, message_content = list(message_data.items())[0]
+            if keyword_lower in message_content.lower():
+                results.append(
+                    {
+                        "receiver_id": receiver_id,
+                        "message": message_content,
+                    }
+                )
         return {"results": results}
 
     def get_message_stats(self) -> Dict[str, Union[Dict[str, int], str]]:
@@ -344,17 +305,13 @@ def get_message_stats(self) -> Dict[str, Union[Dict[str, int], str]]:
         received_count = 0
         contacts = set()
         # Loop through the inbox to calculate stats
-        for message_id, message_data in self.inbox.items():
-            if message_data["sender_id"] == self.current_user:
-                sent_count += 1
-                contacts.add(message_data["receiver_id"])
-            if message_data["receiver_id"] == self.current_user:
-                received_count += 1
-                contacts.add(message_data["sender_id"])
+        for message_data in self.inbox:
+            receiver_id, message_content = list(message_data.items())[0]
+            received_count += 1
+            contacts.add(receiver_id)
         total_contacts = len(contacts)
         return {
             "stats": {
-                "sent_count": sent_count,
                 "received_count": received_count,
                 "total_contacts": total_contacts,
             }