[BFCL] Add New Model open-mistral-nemo-2407, open-mixtral-8x22b, open…

…-mixtral-8x7b (#570) 1. Add the following new models: - `open-mistral-nemo-2407` - `open-mistral-nemo-2407-FC-Any` - `open-mistral-nemo-2407-FC-Auto` - `open-mixtral-8x22b` - `open-mixtral-8x22b-FC-Any` - `open-mixtral-8x22b-FC-Auto` - `open-mixtral-8x7b` 2. Upgrade `mistral-large-2402` endpoint to `mistral-large-2407` for both the prompt and function calling variants. We will update the leaderboard shortly, in a different PR.
ShishirPatil · Aug 7, 2024 · 453f71c · 453f71c
1 parent 3c009e8
commit 453f71c
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 21 deletions.
diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md
@@ -107,8 +107,12 @@ Below is *a table of models we support* to run our leaderboard evaluation agains
 |meetkai/functionary-{small,medium}-v2.4-FC| Function Calling|
 |meetkai/functionary-small-v2.2-FC| Function Calling|
 |meta-llama/Meta-Llama-3-{8B,70B}-Instruct | Prompt|
-|mistral-large-2402-FC-{Any,Auto} | Function Calling|
-|mistral-large-2402 | Prompt|
+|open-mixtral-{8x7b,8x22b} | Prompt|
+|open-mixtral-8x22b-FC-{Any,Auto} | Function Calling|
+|open-mistral-nemo-2407 | Prompt|
+|open-mistral-nemo-2407-FC-{Any,Auto} | Function Calling|
+|mistral-large-2407-FC-{Any,Auto} | Function Calling|
+|mistral-large-2407 | Prompt|
 |mistral-medium-2312 | Prompt|
 |mistral-small-2402-FC-{Any,Auto} | Function Calling|
 |mistral-small-2402 | Prompt|

diff --git a/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py b/berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py
@@ -132,6 +132,48 @@
         "Anthropic",
         "Proprietary",
     ],
+    "open-mistral-nemo-2407": [
+        "Open-Mistral-Nemo-2407 (Prompt)",
+        "https://mistral.ai/news/mistral-nemo/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mistral-nemo-2407-FC-Any": [
+        "Open-Mistral-Nemo-2407 (FC Any)",
+        "https://mistral.ai/news/mistral-nemo/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mistral-nemo-2407-FC-Auto": [
+        "Open-Mistral-Nemo-2407 (FC Auto)",
+        "https://mistral.ai/news/mistral-nemo/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mixtral-8x22b": [
+        "Open-Mixtral-8x22b (Prompt)",
+        "https://mistral.ai/news/mixtral-8x22b/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mixtral-8x22b-FC-Any": [
+        "Open-Mixtral-8x22b (FC Any)",
+        "https://mistral.ai/news/mixtral-8x22b/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mixtral-8x22b-FC-Auto": [
+        "Open-Mixtral-8x22b (FC Auto)",
+        "https://mistral.ai/news/mixtral-8x22b/",
+        "Mistral AI",
+        "Proprietary",
+    ],
+    "open-mixtral-8x7b": [
+        "Open-Mixtral-8x7b (Prompt)",
+        "https://mistral.ai/news/mixtral-of-experts/",
+        "Mistral AI",
+        "Proprietary",
+    ],
     "mistral-medium-2312": [
         "Mistral-Medium-2312 (Prompt)",
         "https://docs.mistral.ai/guides/model-selection/",
@@ -144,9 +186,9 @@
         "Mistral AI",
         "Proprietary",
     ],
-    "mistral-large-2402": [
-        "Mistral-Large-2402 (Prompt)",
-        "https://docs.mistral.ai/guides/model-selection/",
+    "mistral-large-2407": [
+        "mistral-large-2407 (Prompt)",
+        "https://mistral.ai/news/mistral-large-2407/",
         "Mistral AI",
         "Proprietary",
     ],
@@ -246,9 +288,9 @@
         "Mistral AI",
         "Proprietary",
     ],
-    "mistral-large-2402-FC-Any": [
-        "Mistral-large-2402 (FC Any)",
-        "https://docs.mistral.ai/guides/model-selection/",
+    "mistral-large-2407-FC-Any": [
+        "mistral-large-2407 (FC Any)",
+        "https://mistral.ai/news/mistral-large-2407/",
         "Mistral AI",
         "Proprietary",
     ],
@@ -258,9 +300,9 @@
         "Mistral AI",
         "Proprietary",
     ],
-    "mistral-large-2402-FC-Auto": [
-        "Mistral-large-2402 (FC Auto)",
-        "https://docs.mistral.ai/guides/model-selection/",
+    "mistral-large-2407-FC-Auto": [
+        "mistral-large-2407 (FC Auto)",
+        "https://mistral.ai/news/mistral-large-2407/",
         "Mistral AI",
         "Proprietary",
     ],
@@ -463,8 +505,8 @@
     "claude-3-5-sonnet-20240620": 3,
     "claude-2.1": 8,
     "claude-instant-1.2": 0.8,
-    "mistral-large-2402-FC-Any": 4,
-    "mistral-large-2402-FC-Auto": 4,
+    "mistral-large-2407-FC-Any": 4,
+    "mistral-large-2407-FC-Auto": 4,
     "mistral-medium-2312": 2.7,
     "mistral-small-2402-FC-Any": 1,
     "mistral-small-2402-FC-Auto": 1,
@@ -505,8 +547,8 @@
     "claude-3-haiku-20240307": 1.25,
     "claude-2.1": 24,
     "claude-instant-1.2": 2.4,
-    "mistral-large-2402-FC-Any": 12,
-    "mistral-large-2402-FC-Auto": 12,
+    "mistral-large-2407-FC-Any": 12,
+    "mistral-large-2407-FC-Auto": 12,
     "mistral-small-2402": 3,
     "mistral-medium-2312": 8.1,
     "mistral-small-2402-FC-Any": 3,

diff --git a/berkeley-function-call-leaderboard/model_handler/constant.py b/berkeley-function-call-leaderboard/model_handler/constant.py
@@ -128,9 +128,13 @@
     "claude-3-sonnet-20240229-FC",
     "claude-3-haiku-20240307-FC",
     "claude-3-5-sonnet-20240620-FC",
-    "mistral-large-2402-FC",
-    "mistral-large-2402-FC-Any",
-    "mistral-large-2402-FC-Auto",
+    "open-mistral-nemo-2407-FC-Any",
+    "open-mistral-nemo-2407-FC-Auto",
+    "open-mixtral-8x22b-FC-Any",
+    "open-mixtral-8x22b-FC-Auto",
+    "mistral-large-2407-FC",
+    "mistral-large-2407-FC-Any",
+    "mistral-large-2407-FC-Auto",
     "mistral-small-2402-FC-Any",
     "mistral-small-2402-FC-Auto",
     "mistral-small-2402-FC",

diff --git a/berkeley-function-call-leaderboard/model_handler/handler_map.py b/berkeley-function-call-leaderboard/model_handler/handler_map.py
@@ -51,9 +51,16 @@
     "claude-3-haiku-20240307-FC": ClaudeFCHandler,
     "claude-3-5-sonnet-20240620": ClaudePromptingHandler,
     "claude-3-5-sonnet-20240620-FC": ClaudeFCHandler,
-    "mistral-large-2402": MistralHandler,
-    "mistral-large-2402-FC-Any": MistralHandler,
-    "mistral-large-2402-FC-Auto": MistralHandler,
+    "open-mistral-nemo-2407": MistralHandler,
+    "open-mistral-nemo-2407-FC-Any": MistralHandler,
+    "open-mistral-nemo-2407-FC-Auto": MistralHandler,
+    "open-mixtral-8x22b": MistralHandler,
+    "open-mixtral-8x22b-FC-Any": MistralHandler,
+    "open-mixtral-8x22b-FC-Auto": MistralHandler,
+    "open-mixtral-8x7b": MistralHandler,
+    "mistral-large-2407": MistralHandler,
+    "mistral-large-2407-FC-Any": MistralHandler,
+    "mistral-large-2407-FC-Auto": MistralHandler,
     "mistral-medium-2312": MistralHandler,
     "mistral-small-2402": MistralHandler,
     "mistral-small-2402-FC-Any": MistralHandler,