Skip to content

Commit

Permalink
[BFCL] Remove Workaround Patch for Vertex AI Package (#761)
Browse files Browse the repository at this point in the history 
Since the typecasting bugs in Gemini Function Calling has been fixed in
the upstream `google-cloud-aiplatform` library and the version has been
updated in #760 , the workaround patch for the Vertex AI package
introduced in #658 is no longer needed and should be removed for code
readability.
  • Loading branch information
@HuanzhiMao
HuanzhiMao authored Nov 16, 2024
1 parent 4b6b7d0 commit cb8ff39
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 147 deletions.
1 change: 1 addition & 0 deletions berkeley-function-call-leaderboard/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
All notable changes to the Berkeley Function Calling Leaderboard will be documented in this file.

- [Nov 15, 2024] [#762](https://github.com/ShishirPatil/gorilla/pull/762): Supply `data_multi_turn.csv` for multi-turn evaluation results
- [Nov 14, 2024] [#760](https://github.com/ShishirPatil/gorilla/pull/760), [#761](https://github.com/ShishirPatil/gorilla/pull/761): Upstream `google-cloud-aiplatform` library fixed typecasting bugs in Function Calling. Updated to version `1.72.0` and remove the workaround patch introduced in [#648](https://github.com/ShishirPatil/gorilla/pull/648).
- [Nov 14, 2024] [#747](https://github.com/ShishirPatil/gorilla/pull/747): Minor Grammatical Corrections to `DEFAULT_SYSTEM_PROMPT` that is supplied to all prompting models.
- [Nov 13, 2024] [#737](https://github.com/ShishirPatil/gorilla/pull/737), [#739](https://github.com/ShishirPatil/gorilla/pull/739), [#740](https://github.com/ShishirPatil/gorilla/pull/740): Bug fix in the dataset and possible answers for the live and multi-turn categories.
- [Nov 9, 2024] [#749](https://github.com/ShishirPatil/gorilla/pull/749): Remove `Llama-3.2-3B-Instruct-FC` and `Llama-3.2-1B-Instruct-FC` from the leaderboard. According to the [official Llama documentation](https://www.llama.com/docs/model-cards-and-prompt-formats/llama3_2#-tool-calling-(1b/3b)-), these models perform function calling using the prompt-style chat template rather than the specialized function-calling format.
Expand Down
150 changes: 3 additions & 147 deletions berkeley-function-call-leaderboard/bfcl/model_handler/proprietary_model/gemini.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import json
import os

import vertexai
Expand All @@ -14,9 +13,6 @@
func_doc_language_specific_pre_processing,
system_prompt_pre_processing_chat_model,
)

# This import from struct_pb2 should eventually be removed. See comment in the `_handle_struct_values` and `_handle_list_values` method below
from google.protobuf.struct_pb2 import ListValue, Struct
from vertexai.generative_models import (
Content,
FunctionDeclaration,
Expand Down Expand Up @@ -51,128 +47,6 @@ def _substitute_prompt_role(prompts: list[dict]) -> list[dict]:

return prompts

def _handle_struct_values(self, input):
# Note: Below is a workaround for a bug in the Vertex AI library
# Accoding to the Vertex AI documentation https://ai.google.dev/gemini-api/docs/function-calling/tutorial?lang=python, cited below:
"""
# Set the model up with tools.
house_fns = [power_disco_ball, start_music, dim_lights]
model = genai.GenerativeModel(model_name="gemini-1.5-flash", tools=house_fns)
# Call the API.
chat = model.start_chat()
response = chat.send_message("Turn this place into a party!")
# Print out each of the function calls requested from this single call.
for part in response.parts:
if fn := part.function_call:
args = ", ".join(f"{key}={val}" for key, val in fn.args.items())
print(f"{fn.name}({args})")
"""
# ", ".join(f"{key}={val}" for key, val in fn.args.items()) should get the function call arguments in a ready-to-execute format
# However, the above code snippet will not work as expected when `val` is a ListValue object, and it would further cause the json serialization error when writing the result to a file
"""
# This is a typical ListValue object that is causing the issue. It is a list of 4 string values
values {
string_value: "driver"
}
values {
string_value: "passenger"
}
values {
string_value: "rear_left"
}
values {
string_value: "rear_right"
}
"""
# To fix this, we need to unpack the ListValue object to a list of string values before joining them
# So the above example gets converted to:
"""
["driver", "passenger", "rear_left", "rear_right"]
"""
# This will be the temporary fix until the bug in the Vertex AI library is fixed
# Convert it to dictionary for easier manipulation
input = {k: v for k, v in input.items()}
for k, v in input.items():
if type(v) == ListValue:
input[k] = self._handle_list_values(v)
elif type(v) == Struct:
input[k] = self._handle_struct_values(v)
return input

def _handle_list_values(self, input):
"""
@typing.final
class Value(google.protobuf.message.Message):
"`Value` represents a dynamically typed value which can be either
null, a number, a string, a boolean, a recursive struct value, or a
list of values. A producer of value is expected to set one of these
variants. Absence of any variant indicates an error.
The JSON representation for `Value` is JSON value.
"
DESCRIPTOR: google.protobuf.descriptor.Descriptor
NULL_VALUE_FIELD_NUMBER: builtins.int
NUMBER_VALUE_FIELD_NUMBER: builtins.int
STRING_VALUE_FIELD_NUMBER: builtins.int
BOOL_VALUE_FIELD_NUMBER: builtins.int
STRUCT_VALUE_FIELD_NUMBER: builtins.int
LIST_VALUE_FIELD_NUMBER: builtins.int
null_value: global___NullValue.ValueType
"Represents a null value."
number_value: builtins.float
"Represents a double value."
string_value: builtins.str
"Represents a string value."
bool_value: builtins.bool
"Represents a boolean value."
@property
def struct_value(self) -> global___Struct:
"Represents a structured value."
@property
def list_value(self) -> global___ListValue:
"Represents a repeated `Value`."
def __init__(
self,
*,
null_value: global___NullValue.ValueType | None = ...,
number_value: builtins.float | None = ...,
string_value: builtins.str | None = ...,
bool_value: builtins.bool | None = ...,
struct_value: global___Struct | None = ...,
list_value: global___ListValue | None = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["bool_value", b"bool_value", "kind", b"kind", "list_value", b"list_value", "null_value", b"null_value", "number_value", b"number_value", "string_value", b"string_value", "struct_value", b"struct_value"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["bool_value", b"bool_value", "kind", b"kind", "list_value", b"list_value", "null_value", b"null_value", "number_value", b"number_value", "string_value", b"string_value", "struct_value", b"struct_value"]) -> None: ...
def WhichOneof(self, oneof_group: typing.Literal["kind", b"kind"]) -> typing.Literal["null_value", "number_value", "string_value", "bool_value", "struct_value", "list_value"] | None: ...
"""
parsed_list_result = []
for item in input.values:
field_name = item.WhichOneof("kind")
if field_name == "null_value":
value = item.null_value
elif field_name == "number_value":
value = item.number_value
elif field_name == "string_value":
value = item.string_value
elif field_name == "bool_value":
value = item.bool_value
elif field_name == "struct_value":
value = self._handle_struct_values(item.struct_value)
elif field_name == "list_value":
value = self._handle_list_values(item.list_value)
else:
value = None
parsed_list_result.append(value)

return parsed_list_result

def decode_ast(self, result, language="Python"):
if "FC" not in self.model_name:
result = result.replace("```tool_code\n", "").replace("\n```", "")
Expand Down Expand Up @@ -279,30 +153,12 @@ def _parse_query_response_FC(self, api_response: any) -> dict:
if part.function_call and part.function_call.name:
part_func_name = part.function_call.name
part_func_args = part.function_call.args
# Bug patch for the Vertex AI library
part_func_args_dict = self._handle_struct_values(part_func_args)
part_func_args_dict = {k: v for k, v in part_func_args.items()}

fc_parts.append({part_func_name: part_func_args_dict})
tool_call_func_names.append(part_func_name)
else:
# Note: Below is a workaround for a bug in the Vertex AI library
# We should use `part.text` to get the text content of the part, per the Vertex AI documentation
# However, calling part.text will yield the error
"""TypeError: argument of type 'Part' is not iterable"""
# This is a bug in the Vertex AI library, specifically from line `if "text" not in self._raw_part:` in the `text` method in `Part` class in vertexai.generative_models._generative_models
"""
class Part:
@property
def text(self) -> str:
if "text" not in self._raw_part:
raise AttributeError(
"Response candidate content part has no text.\n"
"Part:\n" + _dict_to_pretty_string(self.to_dict())
)
return self._raw_part.text
"""
# Directly accessing the `part._raw_part.text` attribute is a workaround for this bug, until the bug is fixed
text_parts.append(part._raw_part.text)
text_parts.append(part.text)

model_responses = fc_parts if fc_parts else text_parts

Expand Down Expand Up @@ -427,7 +283,7 @@ def _parse_query_response_prompting(self, api_response: any) -> dict:
# This is a workaround for this bug, until the bug is fixed

if len(api_response.candidates[0].content.parts) > 0:
model_responses = api_response.candidates[0].content.parts[0]._raw_part.text
model_responses = api_response.text
else:
model_responses = "The model did not return any response."
return {
Expand Down

0 comments on commit cb8ff39

Please sign in to comment.