From bf71896a9886bcb9e95f13c6575d9961abb8e7b1 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sat, 6 Jul 2024 21:49:26 -0700 Subject: [PATCH 1/7] overhaul apply_function_credential_config.py to improve usability --- .../apply_function_credential_config.py | 84 ++++++++++++------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index 7b6124896..b549eb00c 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -1,10 +1,12 @@ +import glob import json import argparse - +import os +from eval_checker import custom_exception parser = argparse.ArgumentParser(description="Replace placeholders in the function credential config file.") -parser.add_argument("--input-file", help="Path to the function credential config file.", required=True) -parser.add_argument("--output-file", help="Path to the output file.", default="") +parser.add_argument("--input-dir", help="Path to the input directory containing the files.") +parser.add_argument("--output-dir", help="Path to the output directory where the files will be placed.") args = parser.parse_args() # Load the configuration with actual API keys @@ -41,37 +43,55 @@ def replace_placeholders(data): data[idx] = item.replace(placeholder, actual_value) return data -def main(): - # Verify all values are provided - for key, value in PLACEHOLDERS.items(): - if value == "": - print(f"Please provide a value for the placeholder {key}.") - return - print("All API keys are present.") - - modified_data = [] - with open(f"{args.input_file}", 'r') as f: - lines = f.readlines() - for line in lines: - try: - data = json.loads(line) # Parse each line as a JSON object - data = replace_placeholders(data) # Replace placeholders - modified_data.append(json.dumps(data)) # Convert back to string and store - except json.JSONDecodeError: - # Handle the case where a line is not a valid JSON object - print("Invalid JSON line skipped.") - continue - if args.output_file == "": - with open(f"{args.input_file}", 'w') as f: - for modified_line in modified_data: - f.write(modified_line + '\n') # Write each modified JSON object back to the input file - print(f"All placeholders have been replaced in {args.input_file} 🦍.") - else: - with open(f"{args.output_file}", 'w') as f: +def main(input_dir, output_dir): + print(f"Input directory: {input_dir}") + # Get a list of all entries in the folder + entries = os.scandir(input_dir) + + json_files_pattern = os.path.join(input_dir, "*.json") + for input_file_path in glob.glob(json_files_pattern): + file_name = os.path.basename(input_file_path) + output_file_path = os.path.join(output_dir, file_name) + + modified_data = [] + with open(input_file_path, "r") as f: + lines = f.readlines() + for line in lines: + try: + data = json.loads(line) # Parse each line as a JSON object + data = replace_placeholders(data) # Replace placeholders + modified_data.append(json.dumps(data)) # Convert back to string and store + except json.JSONDecodeError: + # Handle the case where a line is not a valid JSON object + print("Invalid JSON line skipped.") + continue + + with open(output_file_path, 'w') as f: for modified_line in modified_data: f.write(modified_line + '\n') # Write each modified JSON object overwrite the output file - print(f"All placeholders have been replaced in {args.output_file} 🦍.") + print(f"All placeholders have been replaced in {args.output_file} 🦍.") + + # Process each subdirectory + subdirs = [entry.path for entry in entries if entry.is_dir()] + for subdir in subdirs: + main(subdir, subdir) + + if __name__ == "__main__": - main() \ No newline at end of file + # Verify all values are provided + for key, value in PLACEHOLDERS.items(): + if value == "": + raise custom_exception.NoAPIKeyError() + print("All API keys are present.") + + input_dir = args.input_dir + if input_dir is None: + input_dir = "./data/" + + output_dir = args.output_dir + if output_dir is None: + output_dir = input_dir + + main(input_dir, output_dir) \ No newline at end of file From cb50b1d109c82337559bc21df05eba78614edd9a Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sat, 6 Jul 2024 22:23:16 -0700 Subject: [PATCH 2/7] keep support for single file as well --- .../apply_function_credential_config.py | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index b549eb00c..fe3751623 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -5,8 +5,8 @@ from eval_checker import custom_exception parser = argparse.ArgumentParser(description="Replace placeholders in the function credential config file.") -parser.add_argument("--input-dir", help="Path to the input directory containing the files.") -parser.add_argument("--output-dir", help="Path to the output directory where the files will be placed.") +parser.add_argument("--input-path", help="Path to the function credential config file. Can be a file or a directory.") +parser.add_argument("--output-path", help="Path to the output file.") args = parser.parse_args() # Load the configuration with actual API keys @@ -44,7 +44,28 @@ def replace_placeholders(data): return data -def main(input_dir, output_dir): +def process_file(input_file_path, output_file_path): + modified_data = [] + with open(input_file_path, "r") as f: + lines = f.readlines() + for line in lines: + try: + data = json.loads(line) # Parse each line as a JSON object + data = replace_placeholders(data) # Replace placeholders + modified_data.append(json.dumps(data)) # Convert back to string and store + except json.JSONDecodeError: + # Handle the case where a line is not a valid JSON object + print("Invalid JSON line skipped.") + continue + + with open(output_file_path, 'w') as f: + for modified_line in modified_data: + f.write(modified_line + '\n') # Write each modified JSON object overwrite the output file + + print(f"All placeholders have been replaced in {args.output_file} 🦍.") + + +def process_dir(input_dir, output_dir): print(f"Input directory: {input_dir}") # Get a list of all entries in the folder entries = os.scandir(input_dir) @@ -53,30 +74,12 @@ def main(input_dir, output_dir): for input_file_path in glob.glob(json_files_pattern): file_name = os.path.basename(input_file_path) output_file_path = os.path.join(output_dir, file_name) - - modified_data = [] - with open(input_file_path, "r") as f: - lines = f.readlines() - for line in lines: - try: - data = json.loads(line) # Parse each line as a JSON object - data = replace_placeholders(data) # Replace placeholders - modified_data.append(json.dumps(data)) # Convert back to string and store - except json.JSONDecodeError: - # Handle the case where a line is not a valid JSON object - print("Invalid JSON line skipped.") - continue - - with open(output_file_path, 'w') as f: - for modified_line in modified_data: - f.write(modified_line + '\n') # Write each modified JSON object overwrite the output file - - print(f"All placeholders have been replaced in {args.output_file} 🦍.") + process_file(input_file_path, output_file_path) # Process each subdirectory subdirs = [entry.path for entry in entries if entry.is_dir()] for subdir in subdirs: - main(subdir, subdir) + process_dir(subdir, subdir) if __name__ == "__main__": @@ -86,12 +89,15 @@ def main(input_dir, output_dir): raise custom_exception.NoAPIKeyError() print("All API keys are present.") - input_dir = args.input_dir - if input_dir is None: - input_dir = "./data/" + input_path = args.input_path + if input_path is None: + input_path = "./data/" - output_dir = args.output_dir - if output_dir is None: - output_dir = input_dir - - main(input_dir, output_dir) \ No newline at end of file + output_path = args.output_path + if output_path is None: + output_path = input_path + + if os.path.isdir(input_path): + process_dir(input_path, output_path) + else: + process_file(input_path, output_path) \ No newline at end of file From 4756e66f38e5e15afe3da81777a91029842d02a2 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sat, 6 Jul 2024 22:38:37 -0700 Subject: [PATCH 3/7] fix typo --- .../apply_function_credential_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index fe3751623..70c32d54d 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -62,7 +62,7 @@ def process_file(input_file_path, output_file_path): for modified_line in modified_data: f.write(modified_line + '\n') # Write each modified JSON object overwrite the output file - print(f"All placeholders have been replaced in {args.output_file} 🦍.") + print(f"All placeholders have been replaced for {input_file_path} 🦍.") def process_dir(input_dir, output_dir): From 8b1e35590e5bce3bd52a7c6405775b1ce4a64945 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sat, 6 Jul 2024 22:46:09 -0700 Subject: [PATCH 4/7] comment out support for subdirectory --- .../apply_function_credential_config.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index 70c32d54d..bb3b2724f 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -58,9 +58,12 @@ def process_file(input_file_path, output_file_path): print("Invalid JSON line skipped.") continue - with open(output_file_path, 'w') as f: - for modified_line in modified_data: - f.write(modified_line + '\n') # Write each modified JSON object overwrite the output file + # Write each modified JSON object overwrite the output file + with open(output_file_path, "w") as f: + for i, modified_line in enumerate(modified_data): + f.write(modified_line) + if i < len(data) - 1: + f.write("\n") print(f"All placeholders have been replaced for {input_file_path} 🦍.") @@ -77,9 +80,9 @@ def process_dir(input_dir, output_dir): process_file(input_file_path, output_file_path) # Process each subdirectory - subdirs = [entry.path for entry in entries if entry.is_dir()] - for subdir in subdirs: - process_dir(subdir, subdir) + # subdirs = [entry.path for entry in entries if entry.is_dir()] + # for subdir in subdirs: + # process_dir(subdir, subdir) if __name__ == "__main__": From aa7ec5008d0092784d9a2a534c4b278a9611b316 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sun, 7 Jul 2024 15:39:20 -0700 Subject: [PATCH 5/7] update setup instruction --- berkeley-function-call-leaderboard/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/berkeley-function-call-leaderboard/README.md b/berkeley-function-call-leaderboard/README.md index 28969bf4c..2492d93b8 100644 --- a/berkeley-function-call-leaderboard/README.md +++ b/berkeley-function-call-leaderboard/README.md @@ -52,10 +52,10 @@ To run the executable test categories, there are 4 API keys to include: 3. OMDB API: http://www.omdbapi.com/apikey.aspx 4. Geocode API: https://geocode.maps.co/ -The `apply_function_credential_config.py` takes an input and optionally an output file. If the output file is not given as an argument, it will overwrite your original file with the data reset. +The `apply_function_credential_config.py` will automatically search for dataset files in the default `./data/` directory and replace the placeholder values with the actual API keys. ```bash -python apply_function_credential_config.py --input-file data/gorilla_openfunctions_v1_test_rest.json +python apply_function_credential_config.py ``` Then, use `eval_data_compilation.py` to compile all files by From 3f58509a9f6acc07edd974127a612494706e8f50 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Sun, 7 Jul 2024 22:50:24 -0700 Subject: [PATCH 6/7] remove commented block --- .../apply_function_credential_config.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index bb3b2724f..9299e73b6 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -69,6 +69,9 @@ def process_file(input_file_path, output_file_path): def process_dir(input_dir, output_dir): + # This function does not support nested directories + # To support nested directories, refer to this commit: + # https://github.com/ShishirPatil/gorilla/pull/508/commits/8b1e35590e5bce3bd52a7c6405775b1ce4a64945 print(f"Input directory: {input_dir}") # Get a list of all entries in the folder entries = os.scandir(input_dir) @@ -79,11 +82,6 @@ def process_dir(input_dir, output_dir): output_file_path = os.path.join(output_dir, file_name) process_file(input_file_path, output_file_path) - # Process each subdirectory - # subdirs = [entry.path for entry in entries if entry.is_dir()] - # for subdir in subdirs: - # process_dir(subdir, subdir) - if __name__ == "__main__": # Verify all values are provided From c0aaf766a2556610d25f86a902f297626032a869 Mon Sep 17 00:00:00 2001 From: Huanzhi Mao Date: Wed, 10 Jul 2024 23:43:35 -0700 Subject: [PATCH 7/7] fix wording --- .../apply_function_credential_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/berkeley-function-call-leaderboard/apply_function_credential_config.py b/berkeley-function-call-leaderboard/apply_function_credential_config.py index 9299e73b6..38196481f 100644 --- a/berkeley-function-call-leaderboard/apply_function_credential_config.py +++ b/berkeley-function-call-leaderboard/apply_function_credential_config.py @@ -58,7 +58,7 @@ def process_file(input_file_path, output_file_path): print("Invalid JSON line skipped.") continue - # Write each modified JSON object overwrite the output file + # Write the modified data to the output file with open(output_file_path, "w") as f: for i, modified_line in enumerate(modified_data): f.write(modified_line)