diff --git a/berkeley-function-call-leaderboard/.env.example b/berkeley-function-call-leaderboard/.env.example index 18b72b1ef..290c3517f 100644 --- a/berkeley-function-call-leaderboard/.env.example +++ b/berkeley-function-call-leaderboard/.env.example @@ -6,6 +6,7 @@ FIREWORKS_API_KEY= ANTHROPIC_API_KEY= NVIDIA_API_KEY=nvapi-XXXXXX YI_API_KEY= +GOGOAGENT_API_KEY= # We use Vertex AI to inference Google Gemini models VERTEX_AI_PROJECT_ID= diff --git a/berkeley-function-call-leaderboard/CHANGELOG.md b/berkeley-function-call-leaderboard/CHANGELOG.md index fed6560e5..2fb8c540d 100644 --- a/berkeley-function-call-leaderboard/CHANGELOG.md +++ b/berkeley-function-call-leaderboard/CHANGELOG.md @@ -2,6 +2,7 @@ All notable changes to the Berkeley Function Calling Leaderboard will be documented in this file. +- [Nov 8, 2024] [#720](https://github.com/ShishirPatil/gorilla/pull/720): Add new model `BitAgent/GoGoAgent` to the leaderboard. - [Oct 30, 2024] [#725](https://github.com/ShishirPatil/gorilla/pull/725), [#733](https://github.com/ShishirPatil/gorilla/pull/733): Update evaluation metric for multi-turn categories: - Introduce a new response-based checker, which works alongside with the existing state-based checker. - The new checker compares the model’s execution result against the ground truth execution result, ensuring that the model’s result encompasses the ground truth (i.e., ground truth must be a strict subset of the model result). diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md index 4acbf926a..1bfd4b6d3 100644 --- a/berkeley-function-call-leaderboard/README.md +++ b/berkeley-function-call-leaderboard/README.md @@ -99,6 +99,7 @@ FIREWORKS_API_KEY= ANTHROPIC_API_KEY= NVIDIA_API_KEY=nvapi-XXXXXX YI_API_KEY= +GOGOAGENT_API_KEY= VERTEX_AI_PROJECT_ID= VERTEX_AI_LOCATION= @@ -213,6 +214,7 @@ Below is _a table of models we support_ to run our leaderboard evaluation agains |Qwen/Qwen2-{1.5B,7B}-Instruct 💻| Prompt| |Team-ACE/ToolACE-8B 💻| Function Calling| |openbmb/MiniCPM3-4B 💻| Function Calling| +|BitAgent/GoGoAgent 💻| Prompt| Here {MODEL} 💻 means the model needs to be hosted locally and called by vllm, {MODEL} means the models that are called API calls. For models with a trailing `-FC`, it means that the model supports function-calling feature. You can check out the table summarizing feature supports among different models [here](https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html#prompt). @@ -339,7 +341,7 @@ We welcome additions to the Function Calling Leaderboard! To add a new model, pl 3. **Update the Handler Map and Model Metadata:** - Modify `bfcl/model_handler/handler_map.py`. This is a mapping of the model name to their handler class. - - Modify `bfcl/val_checker/model_metadata.py`: + - Modify `bfcl/eval_checker/model_metadata.py`: - Update the `MODEL_METADATA_MAPPING` with the model's display name, URL, license, and company information. The key should match the one in `bfcl/model_handler/handler_map.py`. - If your model is price-based, update the `INPUT_PRICE_PER_MILLION_TOKEN` and `OUTPUT_PRICE_PER_MILLION_TOKEN`. - If your model doesn't have a cost, add it to the `NO_COST_MODELS` list. diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py index 8a17ded10..ab2f38709 100644 --- a/berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py +++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/model_metadata.py @@ -649,6 +649,12 @@ "openbmb", "Apache-2.0", ], + "BitAgent/GoGoAgent": [ + "GoGoAgent", + "https://gogoagent.ai", + "BitAgent", + "Proprietary", + ], } INPUT_PRICE_PER_MILLION_TOKEN = { @@ -796,4 +802,5 @@ "MadeAgents/Hammer2.0-3b", "MadeAgents/Hammer2.0-1.5b", "MadeAgents/Hammer2.0-0.5b", + "BitAgent/GoGoAgent", ] diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py index eae669af0..668177ac3 100644 --- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py @@ -23,6 +23,7 @@ from bfcl.model_handler.proprietary_model.nvidia import NvidiaHandler from bfcl.model_handler.proprietary_model.openai import OpenAIHandler from bfcl.model_handler.proprietary_model.yi import YiHandler +from bfcl.model_handler.proprietary_model.gogoagent import GoGoAgentHandler # TODO: Add Deepseek V2, meta-llama/Llama-3.1-405B-Instruct @@ -79,6 +80,7 @@ "command-r-plus-optimized": CohereHandler, "snowflake/arctic": NvidiaHandler, "nvidia/nemotron-4-340b-instruct": NvidiaHandler, + "BitAgent/GoGoAgent": GoGoAgentHandler, # "yi-large-fc": YiHandler, # Their API is under maintenance, and will not be back online in the near future } diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/proprietary_model/gogoagent.py b/berkeley-function-call-leaderboard/bfcl/model_handler/proprietary_model/gogoagent.py new file mode 100644 index 000000000..dd58eb7a9 --- /dev/null +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/proprietary_model/gogoagent.py @@ -0,0 +1,14 @@ +import os + +from bfcl.model_handler.proprietary_model.openai import OpenAIHandler +from openai import OpenAI + + +class GoGoAgentHandler(OpenAIHandler): + def __init__(self, model_name, temperature) -> None: + super().__init__(model_name, temperature) + self.is_fc_model = False + + self.client = OpenAI( + base_url="https://api.gogoagent.ai", api_key=os.getenv("GOGOAGENT_API_KEY") + )