diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md index f7f300107..02fc5fa05 100644 --- a/berkeley-function-call-leaderboard/README.md +++ b/berkeley-function-call-leaderboard/README.md @@ -111,7 +111,9 @@ Below is *a table of models we support* to run our leaderboard evaluation agains |mistral-small-2402 | Prompt| |mistral-tiny-2312 | Prompt| |Nexusflow-Raven-v2 | Function Calling| -|NousResearch/Hermes-2-{Pro-Llama-3-8B,Pro-Mistral-7B,Theta-Llama-3-8B} 💻| Function Calling| +|NousResearch/Hermes-2-Pro-Llama-3-{8B,70B} 💻| Function Calling| +|NousResearch/Hermes-2-Pro-Mistral-7B 💻| Function Calling| +|NousResearch/Hermes-2-Theta-Llama-3-{8B,70B} 💻| Function Calling| |snowflake/arctic | Prompt| |nvidia/nemotron-4-340b-instruct| Prompt| |THUDM/glm-4-9b-chat 💻| Function Calling| diff --git a/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py b/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py index c8e6863ec..da642f7d7 100644 --- a/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py +++ b/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py @@ -324,6 +324,12 @@ "NousResearch", "apache-2.0", ], + "NousResearch/Hermes-2-Pro-Llama-3-70B": [ + "Hermes-2-Pro-Llama-3-70B (FC)", + "https://huggingface.co/NousResearch/Hermes-2-Pro-Llama-3-70B", + "NousResearch", + "apache-2.0", + ], "NousResearch/Hermes-2-Pro-Mistral-7B": [ "Hermes-2-Pro-Mistral-7B (FC)", "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B", @@ -336,6 +342,12 @@ "NousResearch", "apache-2.0", ], + "NousResearch/Hermes-2-Theta-Llama-3-70B": [ + "Hermes-2-Theta-Llama-3-70B (FC)", + "https://huggingface.co/NousResearch/Hermes-2-Theta-Llama-3-70B", + "NousResearch", + "apache-2.0", + ], "meta-llama/Meta-Llama-3-8B-Instruct": [ "Meta-Llama-3-8B-Instruct (Prompt)", "https://llama.meta.com/llama3", @@ -489,6 +501,8 @@ "NousResearch/Hermes-2-Pro-Mistral-7B": 135, "NousResearch/Hermes-2-Pro-Llama-3-8B": 77, "NousResearch/Hermes-2-Theta-Llama-3-8B": 73, + "NousResearch/Hermes-2-Theta-Llama-3-70B": 716, + "NousResearch/Hermes-2-Pro-Llama-3-70B": 674, "meta-llama/Meta-Llama-3-8B-Instruct": 73, "meta-llama/Meta-Llama-3-70B-Instruct": 307, "gorilla-openfunctions-v2": 83, diff --git a/berkeley-function-call-leaderboard/model_handler/constant.py b/berkeley-function-call-leaderboard/model_handler/constant.py index 6c7349533..13d46c5c8 100644 --- a/berkeley-function-call-leaderboard/model_handler/constant.py +++ b/berkeley-function-call-leaderboard/model_handler/constant.py @@ -141,8 +141,10 @@ "meetkai/functionary-small-v2.4-FC", "meetkai/functionary-medium-v2.4-FC", "NousResearch/Hermes-2-Pro-Llama-3-8B", + "NousResearch/Hermes-2-Pro-Llama-3-70B", "NousResearch/Hermes-2-Pro-Mistral-7B", "NousResearch/Hermes-2-Theta-Llama-3-8B", + "NousResearch/Hermes-2-Theta-Llama-3-70B", "command-r-plus-FC", "command-r-plus-FC-optimized", "THUDM/glm-4-9b-chat", diff --git a/berkeley-function-call-leaderboard/model_handler/handler_map.py b/berkeley-function-call-leaderboard/model_handler/handler_map.py index 15fea6486..5eb77c889 100644 --- a/berkeley-function-call-leaderboard/model_handler/handler_map.py +++ b/berkeley-function-call-leaderboard/model_handler/handler_map.py @@ -69,8 +69,10 @@ "meetkai/functionary-medium-v2.4-FC": FunctionaryHandler, "databricks-dbrx-instruct": DatabricksHandler, "NousResearch/Hermes-2-Pro-Llama-3-8B": HermesHandler, + "NousResearch/Hermes-2-Pro-Llama-3-70B": HermesHandler, "NousResearch/Hermes-2-Pro-Mistral-7B": HermesHandler, "NousResearch/Hermes-2-Theta-Llama-3-8B": HermesHandler, + "NousResearch/Hermes-2-Theta-Llama-3-70B": HermesHandler, "meta-llama/Meta-Llama-3-8B-Instruct": LlamaHandler, "meta-llama/Meta-Llama-3-70B-Instruct": LlamaHandler, "command-r-plus-FC": CohereHandler,