From 3c9b0c9dde6718b23ef5b0f4960355f0d494bdfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Hynek=20Kydl=C3=AD=C4=8Dek?= <kydlicek.hynek@gmail.com>
Date: Thu, 6 Feb 2025 13:35:34 +0100
Subject: [PATCH] Bug fix extractive match (#540)

* update extraction match to reflect newest math-verify

* revert symbols, improve sets handling

* rm todo

* fmt + remove empty excepts + bump l2s

* fmt

* docstring

* fixed boxed, bump broken latex2sympy

* allow more separators
---
 pyproject.toml                                       |  2 +-
 .../metrics/utils/extractive_match_utils.py          |  9 +++------
 tests/metrics/test_extractive_match.py               | 12 +++++++++++-
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 9ff7050f..ba5ff998 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -109,7 +109,7 @@ multilingual = [
     "jieba", # for chinese tokenizer
     "pyvi", # for vietnamese tokenizer
 ]
-math = ["latex2sympy2_extended==1.0.4"]
+math = ["latex2sympy2_extended==1.0.6"]
 
 [project.urls]
 Homepage = "https://github.com/huggingface/lighteval"
diff --git a/src/lighteval/metrics/utils/extractive_match_utils.py b/src/lighteval/metrics/utils/extractive_match_utils.py
index 44470410..d931d44c 100644
--- a/src/lighteval/metrics/utils/extractive_match_utils.py
+++ b/src/lighteval/metrics/utils/extractive_match_utils.py
@@ -215,10 +215,7 @@ def make_latex_env_pattern(prefix: str = "", context: Literal["boxed", "plain"]
         rf"(?<!\\)\\\((?P<{prefix}latexInlineParenthesis>{inline_content_parenthesis})(?<!\\)\\\)",
         rf"\s\[(?P<{prefix}latexInlineBracket>{inline_content_bracket})\]\s",
     ]
-    if context == "boxed":
-        # allow also matching plain boxed
-        patterns.append(rf"(?P<{prefix}latexBoxed>\\boxed{{.+}})")
-    elif context == "plain":
+    if context == "plain":
         simple_number = r"-?\d+(?:[.,]\d+)?"
         patterns.append(rf"(?P<{prefix}latexFraction>-?\\frac{{{simple_number}}}{{{simple_number}}})")
 
@@ -237,7 +234,7 @@ def lazy_latex_regex(latex_config: LatexExtractionConfig, language: Language) ->
     and_word = translation_literal.and_word
     or_word = translation_literal.or_word
     next_groups = "".join(
-        [rf"(?:\s*(?:{and_word}|{or_word})\s*{make_latex_env_pattern(f'next{i}_')})?" for i in range(1, 6)]
+        [rf"(?:\s*(?:{and_word}|{or_word}|,)\s*{make_latex_env_pattern(f'next{i}_')})?" for i in range(1, 6)]
     )
 
     latex_envs_re = rf"(?:{first_latex_group}{next_groups})"
@@ -269,7 +266,7 @@ def lazy_latex_regex(latex_config: LatexExtractionConfig, language: Language) ->
         latex_re_boxed = make_latex_env_pattern(prefix="first_", context="boxed")
         next_groups = "".join(
             [
-                rf"(?:\s*(?:{and_word}|{or_word})\s*{make_latex_env_pattern(f'next{i}_', context='boxed')})?"
+                rf"(?:\s*(?:{and_word}|{or_word}|,)\s*{make_latex_env_pattern(f'next{i}_', context='boxed')})?"
                 for i in range(1, 6)
             ]
         )
diff --git a/tests/metrics/test_extractive_match.py b/tests/metrics/test_extractive_match.py
index fb163bf3..6b53d3a7 100644
--- a/tests/metrics/test_extractive_match.py
+++ b/tests/metrics/test_extractive_match.py
@@ -1035,7 +1035,7 @@ def test_math_extraction_additional_cases(gold, pred, expected):
         ),
         (
             r"$(2,1),(1,2),(-1,-20),(-20,-1)$",
-            r"solutions are:\n\n\\[\n\\boxed{(1, 2)}, \\boxed{(2, 1)}, \\boxed{(-1, -20)}, \\boxed{(-20, -1)}\n\\]",
+            "solutions are:\n\n\\[\n\\boxed{(1, 2)}, \\boxed{(2, 1)}, \\boxed{(-1, -20)}, \\boxed{(-20, -1)}\n\\]",
             1,
         ),
         (
@@ -1121,6 +1121,16 @@ def test_math_extraction_additional_cases(gold, pred, expected):
             r"$\boxed{10^{\frac{\sqrt{13} - 5}{6}}} \quad \text{and} \quad \boxed{10^{-\frac{5 + \sqrt{13}}{6}}}$",
             1,
         ),
+        (
+            r"\boxed{1} and and and or thus but \boxed{2} and \boxed{3}",
+            r"$\boxed{2,3}$",
+            1,
+        ),
+        (
+            r"\boxed{1} and and and or thus but \boxed{2} and \boxed{3}",
+            r"$\boxed{1,2,3}$",
+            0,
+        ),
     ],
 )
 def test_math_numina_cases(gold, pred, expected):