Merge remote-tracking branch 'upstream/main' into claude-3.5

berkeley-function-call-leaderboard/.env.example

@@ -6,6 +6,7 @@ FIREWORKS_API_KEY=
 ANTHROPIC_API_KEY=
 NVIDIA_API_KEY=nvapi-XXXXXX
 YI_API_KEY=
+GOGOAGENT_API_KEY=
 
 # We use Vertex AI to inference Google Gemini models
 VERTEX_AI_PROJECT_ID=

berkeley-function-call-leaderboard/CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to the Berkeley Function Calling Leaderboard will be documen
   - `claude-3-5-haiku-20241022-FC`
   - `claude-3-5-sonnet-20241022`
   - `claude-3-5-sonnet-20241022-FC`
+- [Nov 8, 2024] [#720](https://github.com/ShishirPatil/gorilla/pull/720): Add new model `BitAgent/GoGoAgent` to the leaderboard.
 - [Oct 30, 2024] [#725](https://github.com/ShishirPatil/gorilla/pull/725), [#733](https://github.com/ShishirPatil/gorilla/pull/733): Update evaluation metric for multi-turn categories:
   - Introduce a new response-based checker, which works alongside with the existing state-based checker.
     - The new checker compares the model’s execution result against the ground truth execution result, ensuring that the model’s result encompasses the ground truth (i.e., ground truth must be a strict subset of the model result).

berkeley-function-call-leaderboard/README.md

@@ -99,6 +99,7 @@ FIREWORKS_API_KEY=
 ANTHROPIC_API_KEY=
 NVIDIA_API_KEY=nvapi-XXXXXX
 YI_API_KEY=
+GOGOAGENT_API_KEY=
 
 VERTEX_AI_PROJECT_ID=
 VERTEX_AI_LOCATION=
@@ -215,6 +216,7 @@ Below is _a table of models we support_ to run our leaderboard evaluation agains
 |Qwen/Qwen2-{1.5B,7B}-Instruct 💻| Prompt|
 |Team-ACE/ToolACE-8B 💻| Function Calling|
 |openbmb/MiniCPM3-4B 💻| Function Calling|
+|BitAgent/GoGoAgent 💻| Prompt|
 
 Here {MODEL} 💻 means the model needs to be hosted locally and called by vllm, {MODEL} means the models that are called API calls. For models with a trailing `-FC`, it means that the model supports function-calling feature. You can check out the table summarizing feature supports among different models [here](https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html#prompt).
 
@@ -341,7 +343,7 @@ We welcome additions to the Function Calling Leaderboard! To add a new model, pl
 3. **Update the Handler Map and Model Metadata:**
 
    - Modify `bfcl/model_handler/handler_map.py`. This is a mapping of the model name to their handler class.
-   - Modify `bfcl/val_checker/model_metadata.py`:
+   - Modify `bfcl/eval_checker/model_metadata.py`:
      - Update the `MODEL_METADATA_MAPPING` with the model's display name, URL, license, and company information. The key should match the one in `bfcl/model_handler/handler_map.py`.
      - If your model is price-based, update the `INPUT_PRICE_PER_MILLION_TOKEN` and `OUTPUT_PRICE_PER_MILLION_TOKEN`.
      - If your model doesn't have a cost, add it to the `NO_COST_MODELS` list.

berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py

@@ -673,6 +673,12 @@
         "openbmb",
         "Apache-2.0",
     ],
+    "BitAgent/GoGoAgent": [
+        "GoGoAgent",
+        "https://gogoagent.ai",
+        "BitAgent",
+        "Proprietary",
+    ],
 }
 
 INPUT_PRICE_PER_MILLION_TOKEN = {
@@ -828,4 +834,5 @@
     "MadeAgents/Hammer2.0-3b",
     "MadeAgents/Hammer2.0-1.5b",
     "MadeAgents/Hammer2.0-0.5b",
+    "BitAgent/GoGoAgent",
 ]

berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py

@@ -23,6 +23,7 @@
 from bfcl.model_handler.proprietary_model.nvidia import NvidiaHandler
 from bfcl.model_handler.proprietary_model.openai import OpenAIHandler
 from bfcl.model_handler.proprietary_model.yi import YiHandler
+from bfcl.model_handler.proprietary_model.gogoagent import GoGoAgentHandler
 
 # TODO: Add Deepseek V2, meta-llama/Llama-3.1-405B-Instruct
 
@@ -83,6 +84,7 @@
     "command-r-plus-optimized": CohereHandler,
     "snowflake/arctic": NvidiaHandler,
     "nvidia/nemotron-4-340b-instruct": NvidiaHandler,
+    "BitAgent/GoGoAgent": GoGoAgentHandler,
     # "yi-large-fc": YiHandler,  #  Their API is under maintenance, and will not be back online in the near future
 }
 

berkeley-function-call-leaderboard/bfcl/model_handler/proprietary_model/gogoagent.py

@@ -0,0 +1,14 @@
+import os
+
+from bfcl.model_handler.proprietary_model.openai import OpenAIHandler
+from openai import OpenAI
+
+
+class GoGoAgentHandler(OpenAIHandler):
+    def __init__(self, model_name, temperature) -> None:
+        super().__init__(model_name, temperature)
+        self.is_fc_model = False
+
+        self.client = OpenAI(
+            base_url="https://api.gogoagent.ai", api_key=os.getenv("GOGOAGENT_API_KEY")
+        )