BFCL April 11th Patch. Add Latency Statistics.

berkeley-function-call-leaderboard/README.md

@@ -222,6 +222,7 @@ For inferencing `Databrick-DBRX-instruct`, you need to create a Databrick Azure
 
 ## Changelog
 
+* [April 11, 2024] [#347](https://github.com/ShishirPatil/gorilla/pull/347): Add the 95th percentile latency to the leaderboard statistics. This metric is useful for understanding the latency distribution of the models, especially the worst-case scenario.
 * [April 10, 2024] [#339](https://github.com/ShishirPatil/gorilla/pull/339): Introduce REST API sanity check for the executable test category. It ensures that all the API endpoints involved during the execution evaluation process are working properly. If any of them are not behaving as expected, the evaluation process will be stopped by default as the result will be inaccurate. Users can choose to bypass this check by setting the `--skip-api-sanity-check` flag.
 * [April 9, 2024] [#338](https://github.com/ShishirPatil/gorilla/pull/338): Bug fix in the evaluation datasets (including both prompts and function docs). Bug fix for possible answers as well.
 * [April 8, 2024] [#330](https://github.com/ShishirPatil/gorilla/pull/330): Fixed an oversight that was introduced in [#299](https://github.com/ShishirPatil/gorilla/pull/299). For function-calling (FC) models that cannot take `float` type in input, when the parameter type is a `float`, the evaluation procedure will convert that type to `number` in the model input and mention in the parameter description that `This is a float type value.`. An additional field `format: float` will also be included in the model input to make it clear about the type. Updated the model handler for Claude, Mistral, and OSS to better parse the model output.

berkeley-function-call-leaderboard/eval_checker/eval_runner_helper.py

@@ -6,6 +6,7 @@
 import subprocess
 import statistics
 from tqdm import tqdm
+import numpy as np
 
 REST_API_GROUND_TRUTH_FILE_PATH = "api_status_ground_truth_function_calls.json"
 
@@ -35,6 +36,7 @@
     "Cost ($ Per 1k Function Calls)",
     "Latency Mean (s)",
     "Latency Standard Deviation (s)",
+    "Latency 95th Percentile (s)",
 ]
 
 # Note that we don't need to substitute `_` with `/` in the model name here.
@@ -254,7 +256,7 @@
         "https://huggingface.co/NousResearch/Hermes-2-Pro-Mistral-7B",
         "NousResearch",
         "apache-2.0",
-    ]
+    ],
 }
 
 INPUT_PRICE_PER_MILLION_TOKEN = {
@@ -319,6 +321,7 @@
     "deepseek-ai/deepseek-coder-6.7b-instruct": 2040,
     "google/gemma-7b-it": 161,
     "glaiveai/glaive-function-calling-v1": 99,
+    "NousResearch/Hermes-2-Pro-Mistral-7B": 666,
 }
 
 OSS_INPUT_TOKEN = {
@@ -419,7 +422,7 @@ def write_list_of_dicts_to_file(filename, data, subdir=None):
 
         # Construct the full path to the file
         filename = os.path.join(subdir, filename)
-        
+
     # Write the list of dictionaries to the file in JSON format
     with open(filename, "w") as f:
         for i, entry in enumerate(data):
@@ -472,29 +475,31 @@ def is_empty_output(decoded_output):
 
 
 def api_status_sanity_check():
-    
+
     # We only need to import the executable_checker_rest in this function. So a local import is used.
     from checker import executable_checker_rest
-    
+
     ground_truth_dummy = load_file(REST_API_GROUND_TRUTH_FILE_PATH)
 
     # Use the ground truth data to make sure the API is working correctly
-    command = F"cd .. ; python apply_function_credential_config.py --input_file ./eval_checker/{REST_API_GROUND_TRUTH_FILE_PATH};"
+    command = f"cd .. ; python apply_function_credential_config.py --input_file ./eval_checker/{REST_API_GROUND_TRUTH_FILE_PATH};"
     try:
         subprocess.run(command, shell=True, capture_output=True, text=True, check=True)
     except subprocess.CalledProcessError as e:
         write_list_of_dicts_to_file(REST_API_GROUND_TRUTH_FILE_PATH, ground_truth_dummy)
         raise RuntimeError(e.stderr) from e
-    
+
     ground_truth_replaced = load_file(REST_API_GROUND_TRUTH_FILE_PATH)
     write_list_of_dicts_to_file(REST_API_GROUND_TRUTH_FILE_PATH, ground_truth_dummy)
-    
+
     correct_count = 0
-    for idx, data in tqdm(enumerate(ground_truth_replaced), total=len(ground_truth_replaced)):
+    for idx, data in tqdm(
+        enumerate(ground_truth_replaced), total=len(ground_truth_replaced)
+    ):
         status = executable_checker_rest(data["ground_truth"], idx)
         if status["valid"]:
             correct_count += 1
-            
+
     if correct_count != len(ground_truth_replaced):
         error_msg = f"API Status Test Failed. {len(ground_truth_replaced) - correct_count} out of {len(ground_truth_replaced)} API behaviors are not as expected. Be careful with executable test category results; they may be inaccurate."
         raise BadAPIStatusError(error_msg)
@@ -563,7 +568,7 @@ def record_cost_latency(leaderboard_table, model_name, model_output_data):
 
 def get_metric(model_name, cost_data, latency_data):
 
-    cost, mean_latency, std_latency = "N/A", "N/A", "N/A"
+    cost, mean_latency, std_latency, percentile_95_latency = "N/A", "N/A", "N/A", "N/A"
 
     if (
         model_name in INPUT_PRICE_PER_MILLION_TOKEN
@@ -580,16 +585,22 @@ def get_metric(model_name, cost_data, latency_data):
         cost = round(cost, 2)
 
     if model_name in OSS_LATENCY:
-        mean_latency, std_latency = OSS_LATENCY[model_name] / 1700, "N/A"
+        mean_latency, std_latency, percentile_95_latency = (
+            OSS_LATENCY[model_name] / 1700,
+            "N/A",
+            "N/A",
+        )
         mean_latency = round(mean_latency, 2)
         cost = mean_latency * 1000 * A100_PRICE_PER_HOUR / 3600
         cost = round(cost, 2)
 
     elif len(latency_data["data"]) != 0:
         mean_latency = statistics.mean(latency_data["data"])
         std_latency = statistics.stdev(latency_data["data"])
+        percentile_95_latency = np.percentile(latency_data["data"], 95)
         mean_latency = round(mean_latency, 2)
         std_latency = round(std_latency, 2)
+        percentile_95_latency = round(percentile_95_latency, 2)
 
         if model_name not in INPUT_PRICE_PER_MILLION_TOKEN:
             cost = sum(latency_data["data"]) * A100_PRICE_PER_HOUR / 3600
@@ -598,7 +609,7 @@ def get_metric(model_name, cost_data, latency_data):
     if model_name in NO_COST_MODELS:
         cost = "N/A"
 
-    return cost, mean_latency, std_latency
+    return cost, mean_latency, std_latency, percentile_95_latency
 
 
 def generate_leaderboard_csv(leaderboard_table, output_path):
@@ -670,7 +681,7 @@ def generate_leaderboard_csv(leaderboard_table, output_path):
             ]
         )
 
-        cost, latency_mean, latency_std = get_metric(
+        cost, latency_mean, latency_std, percentile_95_latency = get_metric(
             model_name_escaped, cost_data, latency_data
         )
 
@@ -707,16 +718,17 @@ def generate_leaderboard_csv(leaderboard_table, output_path):
                 cost,
                 latency_mean,
                 latency_std,
+                percentile_95_latency,
             ]
         )
 
     data.sort(key=lambda x: x[1], reverse=True)
     for i in range(len(data)):
         data[i][0] = str(i + 1)
         data[i][1] = "{:.2f}%".format(data[i][1] * 100)
-        for j in range(6, len(data[i]) - 3):
+        for j in range(6, len(data[i]) - 4):
             data[i][j] = "{:.2f}%".format(data[i][j] * 100)
-        for j in range(len(data[i]) - 3, len(data[i])):
+        for j in range(len(data[i]) - 4, len(data[i])):
             data[i][j] = str(data[i][j])
 
     data.insert(0, COLUMNS)
@@ -760,7 +772,6 @@ def oss_file_formatter(input_file_path, output_dir):
     data = load_file(input_file_path)
     assert len(data) == 2000, "OSS result.json file should have 2000 entries."
 
-
     for key, value in FILENAME_INDEX_MAPPING.items():
         start, end = value
         output_file = os.path.join(

berkeley-function-call-leaderboard/requirements.txt

@@ -6,4 +6,5 @@ ray
 shortuuid
 mistralai
 anthropic
-openai
+openai
+numpy