better functionary v3.2 parsing & test validation (ignore comments)

ochafik · Feb 10, 2025 · 5c7bfd4 · 5c7bfd4
1 parent c2a2933
commit 5c7bfd4
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 9 deletions.
diff --git a/common/chat.cpp b/common/chat.cpp
@@ -84,7 +84,8 @@ static common_chat_msg parse_json_tool_calls(
     const std::string& input,
     const std::optional<std::regex> & trigger_opt,
     const std::regex & function_regex,
-    const std::regex & close_regex) {
+    const std::regex & close_regex,
+    bool allow_raw_python = false) {
     std::smatch match;
 
     common_chat_msg result;
@@ -116,6 +117,10 @@ static common_chat_msg parse_json_tool_calls(
 
         json arguments;
         if (!parse_json(it, end, arguments)) {
+            if (allow_raw_python && name == "python") {
+                result.tool_calls.push_back({name, json({{"code", std::string(it, end)}}).dump(), /* id= */ ""});
+                break;
+            }
             throw std::runtime_error("Failed to parse json tool call arguments: " + input);
         }
         if (!std::regex_search(it, end, match, close_regex)) {
@@ -730,10 +735,10 @@ static common_chat_params common_chat_params_init_functionary_v3_2(const common_
                 auto args_rule = builder.add_schema(name + "-args", parameters);
                 first_tool_rules.push_back(builder.add_rule(name + "-call", "( \"assistant<|end_header_id|>\\n\" )? \"" + name + "\\n\" " + args_rule));
                 subsequent_tool_rules.push_back(builder.add_rule(name + "-call2", "\">>>" + name + "\\n\" " + args_rule));
-                data.grammar_triggers.push_back({name + "\n", /* .at_start = */ true});
-                data.grammar_triggers.push_back({"assistant<|end_header_id|>\n" + name + "\n", /* .at_start = */ true});
-                data.grammar_triggers.push_back({">>>" + name + "\n", /* .at_start = */ false});
-                data.grammar_triggers.push_back({">>>assistant<|end_header_id|>\n" + name + "\n", /* .at_start = */ false});
+                data.grammar_triggers.push_back({name, /* .at_start = */ true});
+                data.grammar_triggers.push_back({"assistant<|end_header_id|>\n" + name, /* .at_start = */ true});
+                data.grammar_triggers.push_back({">>>" + name, /* .at_start = */ false});
+                data.grammar_triggers.push_back({">>>assistant<|end_header_id|>\n" + name, /* .at_start = */ false});
             });
             data.preserved_tokens = {
                 "<|end_header_id|>",
@@ -788,7 +793,7 @@ static common_chat_msg common_chat_parse_functionary_v3_2(const std::string & in
     }
     // TODO: tighten & simplify.
     try {
-        auto res = parse_json_tool_calls(std::string(it, end), std::nullopt, function_regex, close_regex);
+        auto res = parse_json_tool_calls(std::string(it, end), std::nullopt, function_regex, close_regex, /* allow_raw_python= */ true);
         res.content = content + res.content;
         return res;
     } catch (const std::exception & e) {

diff --git a/examples/server/tests/unit/test_tool_call.py b/examples/server/tests/unit/test_tool_call.py
@@ -523,7 +523,7 @@ def do_test_hello_world(expected_arguments_override: str | None, **kwargs):
     res = server.make_request("POST", "/v1/chat/completions", data={
         "max_tokens": 256,
         "messages": [
-            {"role": "system", "content": "You are a coding assistant."},
+            {"role": "system", "content": "You are a tool-calling agent."},
             {"role": "user", "content": "say hello world with python"},
         ],
         "tools": [PYTHON_TOOL],
@@ -544,7 +544,7 @@ def do_test_hello_world(expected_arguments_override: str | None, **kwargs):
         assert 'code' in actual_arguments, f"code not found in {json.dumps(actual_arguments)}"
         code = actual_arguments["code"]
         assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}"
-        assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', code), f'Expected hello world, got {code}'
+        assert re.match(r'''((#.*)?\n)*print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', code), f'Expected hello world, got {code}'
 
 
 @pytest.mark.slow
@@ -606,7 +606,7 @@ def test_hello_world(expected_arguments_override: str | None, hf_repo: str, temp
         export LLAMA_CACHE=$HOME/Library/Caches/llama.cpp
         export LLAMA_SERVER_BIN_PATH=$PWD/build/bin/llama-server
     
-        ( for temp in "" "--temp 0.0" "--temp 0.5" "--temp 0.75" "--temp 1.0" ; do
+        ( for temp in "" "--temp=0.0" "--temp=0.5" "--temp=0.75" "--temp=1.0" ; do
             for ignore_grammar in 0 1 ; do (
                 export LLAMA_IGNORE_CHAT_GRAMMAR=$ignore_grammar ;
                 python examples/server/tests/unit/test_tool_call.py --n 5 --seed 124 $temp --model "Qwen 2.5 Coder 7B Q4_K_M" --hf bartowski/Qwen2.5-Coder-7B-Instruct-GGUF ;