From 18e78999c5517fd2d6c97629d2ead28cd085803c Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Mon, 26 Oct 2026 00:17:25 +0530 Subject: [PATCH 1/8] fix(assistantbench): disambiguate thousands-separator commas from decimal commas in fix_number Closes #404 The old code called .replace(',', '.') unconditionally in fix_number(), causing two classes of silent scoring errors: - '3,080,000' -> '3.080.000' -> float() raises -> correct answer scores 0.0 - '1,000' -> '1.000' = 1.0 -> matched gold 1 -> wrong answer scores 1.0 Fix: extract disambiguation logic into _fix_comma() in evaluate_utils/utils.py and call it from both fix_number() implementations (evaluator.py and evaluate_dicts.py). Disambiguation rules: - Multiple commas -> US grouping, strip all ('3,080,000' -> '3080000') - Single comma + 3 digits -> thousands separator ('1,000' -> '1000') - Single comma + 1-2 digs -> European decimal comma ('14,2' -> '14.2') - Anything else -> leave unchanged (falls back to string comparison) Also narrows bare except: clauses to explicit exception types. Tests added (23 new cases): - test_fix_comma - 12-case parametrized unit test for _fix_comma - test_evaluate_comma_numbers - 9-case end-to-end question_scorer test - test_evaluate_comma_numbers_in_dicts - dict evaluation path - test_evaluate_comma_numbers_negative - negative thousands separator --- .../evaluate_utils/evaluate_dicts.py | 4 +- .../evaluation/evaluate_utils/utils.py | 44 +++++++++ .../assistantbench/evaluation/evaluator.py | 8 +- tests/assistantbench/test_evaluation.py | 89 +++++++++++++++++++ 4 files changed, 140 insertions(+), 5 deletions(-) diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py index 76136c06a..b5778dc11 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py @@ -2,7 +2,7 @@ import numpy as np -from .utils import _align_bags +from .utils import _align_bags, _fix_comma def calculate_f1_score(precision, recall): @@ -43,7 +43,7 @@ def fix_number(number): " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft") ).strip() copy_ans = copy_ans.strip() - copy_ans = copy_ans.replace(",", ".") + copy_ans = _fix_comma(copy_ans) try: return float(copy_ans) except: diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py index 82be9e4ad..1db8cd40c 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py @@ -1,3 +1,4 @@ +import re from typing import Callable, List, Set import numpy as np @@ -23,3 +24,46 @@ def _align_bags( for row, column in zip(row_ind, col_ind): max_scores[row] = max(max_scores[row], scores[row, column]) return max_scores + + +def _fix_comma(number: str) -> str: + """ + Disambiguates US thousands-separator commas from European decimal commas + before passing a string to ``float()``. + + Rules (applied in order): + + * **Multiple commas** → US grouping style; strip all commas. + ``"3,080,000"`` → ``"3080000"``, ``"12,34,567"`` → ``"1234567"`` + * **Single comma, group of exactly 3 digits after** → thousands separator; + strip the comma. ``"1,000"`` → ``"1000"``, ``"1,234"`` → ``"1234"`` + * **Single comma, 1–2 digits after** → European decimal comma; replace + with a period. ``"14,2"`` → ``"14.2"``, ``"3,14"`` → ``"3.14"`` + * **Anything else** → return unchanged (non-numeric strings fall back to + string comparison upstream). + + The function intentionally does *not* handle a single comma followed by + 4 or more digits (e.g. ``"1,0000"``), leaving those strings unchanged so + that ``float()`` will reject them and the caller falls back gracefully. + + Args: + number: a stripped string that may represent a numeric value. + + Returns: + The string with commas adjusted, ready to pass to ``float()``. + """ + if number.count(",") > 1: + # Multiple commas → US grouping (e.g. "3,080,000", "12,34,567") + return number.replace(",", "") + + match = re.fullmatch(r"(-?\d+),(\d+)", number) + if match: + decimal_digits = match.group(2) + if len(decimal_digits) == 3: + # Single comma + exactly 3 digits → thousands separator + return number.replace(",", "") + if len(decimal_digits) <= 2: + # Single comma + 1–2 digits → European decimal comma + return number.replace(",", ".") + + return number diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py index 7910eadf8..85e7c64dd 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py @@ -5,6 +5,7 @@ import numpy as np from .evaluate_utils.evaluate_factory import get_evaluator +from .evaluate_utils.utils import _fix_comma def find_isnan(samp): @@ -60,10 +61,11 @@ def fix_number(number): " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft") ).strip() copy_ans = copy_ans.strip() - copy_ans = copy_ans.replace(",", ".").replace(" square kilometers", "") + copy_ans = copy_ans.replace(" square kilometers", "") + copy_ans = _fix_comma(copy_ans) try: return float(copy_ans), True - except: + except (ValueError, TypeError): return number, False elif type(number) == int: return float(number), True @@ -102,7 +104,7 @@ def fix_prediction(prediction, gold_answer, evaluator): def question_scorer(prediction, gold_answer): try: prediction = json.loads(prediction) - except: + except (json.JSONDecodeError, TypeError, ValueError): prediction = prediction answer_list = ( diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index 4973d7158..d60cee7b8 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -4,6 +4,7 @@ import gymnasium as gym import pytest +from browsergym.assistantbench.evaluation.evaluate_utils.utils import _fix_comma from browsergym.assistantbench.evaluation.evaluator import question_scorer from browsergym.experiments.benchmark.metadata.utils import ( task_list_from_metadata, @@ -52,6 +53,93 @@ def test_evaluate(original_id: str): assert has_ans == expected_has_ans +# --------------------------------------------------------------------------- +# Comma-disambiguation tests — issue #404 +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "number_str, expected", + [ + # Multiple commas → US thousands grouping, all commas stripped + ("3,080,000", "3080000"), + ("1,010,000", "1010000"), + ("12,34,567", "1234567"), # non-standard grouping still stripped + # Single comma + exactly 3 digits → thousands separator + ("1,000", "1000"), + ("1,234", "1234"), + ("-1,000", "-1000"), # negative thousands + # Single comma + 1–2 digits → European decimal comma + ("14,2", "14.2"), + ("3,14", "3.14"), + # Single comma + 4+ digits → left unchanged (ambiguous; float() will reject) + ("1,0000", "1,0000"), + # No comma → untouched + ("3080000", "3080000"), + ("14.2", "14.2"), + ("hello", "hello"), + ], +) +def test_fix_comma(number_str: str, expected: str): + """Unit-test _fix_comma directly to cover every branch.""" + assert _fix_comma(number_str) == expected + + +@pytest.mark.parametrize( + "prediction, gold_answer, expected_score", + [ + # ── Previously broken: correct answers scored 0 ───────────────────── + # dev-split validation.26: house price with US thousands separator + ("3,080,000", "3080000", 1.0), + # dev-split validation.3: apartment price + ("1,010,000", "1010000", 1.0), + # Simple thousands + ("1,000", "1000", 1.0), + # ── Previously broken: wrong answers scored 1.0 ────────────────────── + # "1,000" → 1.0 which matched gold 1 — 1000x error scored as perfect + ("1,000", "1", 0.0), + ("1,010", "1.01", 0.0), + # ── Must stay correct: European decimal comma ──────────────────────── + # dev-split validation.2 + ("14,2", "14.2", 1.0), + ("3,14", "3.14", 1.0), + # ── Control cases: no commas, must be unaffected ───────────────────── + ("1000", "1000", 1.0), + ("3080000", "3080000", 1.0), + ], +) +def test_evaluate_comma_numbers(prediction: str, gold_answer: str, expected_score: float): + """End-to-end question_scorer test for comma-formatted numeric answers.""" + score, _ = question_scorer(prediction, gold_answer) + assert score == expected_score, ( + f"question_scorer({prediction!r}, {gold_answer!r}) " + f"returned {score}, expected {expected_score}" + ) + + +def test_evaluate_comma_numbers_in_dicts(): + """ + Validates the dict-evaluation path (evaluate_dicts.py) also applies the + comma fix. A JSON value "1,000" against a gold numeric 1000 should score + 1.0, not 0.5 (which the old code returned due to type mismatch after + misparse). + """ + prediction = '[{"sender": "DHL", "price": "1,000"}]' + gold_answer = '{"sender": "DHL", "price": 1000}' + + score, _ = question_scorer(prediction, gold_answer) + assert score == 1.0, f"Dict scorer returned {score}, expected 1.0" + + +def test_evaluate_comma_numbers_negative(): + """Negative numbers with thousands separators should be handled.""" + score, _ = question_scorer("-1,000", "-1000") + assert score == 1.0 + + +# --------------------------------------------------------------------------- + + @pytest.mark.parametrize( "original_id", [id for id in data_points.keys() if isinstance(data_points[id]["answer"], (str, float, int))], @@ -75,3 +163,4 @@ def test_evaluate_within_env(original_id: str): assert reward == expected_score env.close() + From afae07b284a40dd561e32dd250193f7ac41f225b Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Wed, 28 Oct 2026 21:55:02 +0530 Subject: [PATCH 2/8] fix: remove trailing blank line in test_evaluation.py (black formatting) --- tests/assistantbench/test_evaluation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index d60cee7b8..b81b985d5 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -163,4 +163,3 @@ def test_evaluate_within_env(original_id: str): assert reward == expected_score env.close() - From ee1c15ebe8244a5b4ed8ca2d6284fb3f6b087cfa Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Mon, 26 Oct 2026 00:17:25 +0530 Subject: [PATCH 3/8] fix(assistantbench): disambiguate thousands-separator commas from decimal commas in fix_number Closes #404 The old code called .replace(',', '.') unconditionally in fix_number(), causing two classes of silent scoring errors: - '3,080,000' -> '3.080.000' -> float() raises -> correct answer scores 0.0 - '1,000' -> '1.000' = 1.0 -> matched gold 1 -> wrong answer scores 1.0 Fix: extract disambiguation logic into _fix_comma() in evaluate_utils/utils.py and call it from both fix_number() implementations (evaluator.py and evaluate_dicts.py). Disambiguation rules: - Multiple commas -> US grouping, strip all ('3,080,000' -> '3080000') - Single comma + 3 digits -> thousands separator ('1,000' -> '1000') - Single comma + 1-2 digs -> European decimal comma ('14,2' -> '14.2') - Anything else -> leave unchanged (falls back to string comparison) Also narrows bare except: clauses to explicit exception types. Tests added (23 new cases): - test_fix_comma - 12-case parametrized unit test for _fix_comma - test_evaluate_comma_numbers - 9-case end-to-end question_scorer test - test_evaluate_comma_numbers_in_dicts - dict evaluation path - test_evaluate_comma_numbers_negative - negative thousands separator --- .../evaluate_utils/evaluate_dicts.py | 4 +- .../evaluation/evaluate_utils/utils.py | 44 +++++++++ .../assistantbench/evaluation/evaluator.py | 8 +- tests/assistantbench/test_evaluation.py | 89 +++++++++++++++++++ 4 files changed, 140 insertions(+), 5 deletions(-) diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py index 76136c06a..b5778dc11 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py @@ -2,7 +2,7 @@ import numpy as np -from .utils import _align_bags +from .utils import _align_bags, _fix_comma def calculate_f1_score(precision, recall): @@ -43,7 +43,7 @@ def fix_number(number): " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft") ).strip() copy_ans = copy_ans.strip() - copy_ans = copy_ans.replace(",", ".") + copy_ans = _fix_comma(copy_ans) try: return float(copy_ans) except: diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py index 82be9e4ad..1db8cd40c 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py @@ -1,3 +1,4 @@ +import re from typing import Callable, List, Set import numpy as np @@ -23,3 +24,46 @@ def _align_bags( for row, column in zip(row_ind, col_ind): max_scores[row] = max(max_scores[row], scores[row, column]) return max_scores + + +def _fix_comma(number: str) -> str: + """ + Disambiguates US thousands-separator commas from European decimal commas + before passing a string to ``float()``. + + Rules (applied in order): + + * **Multiple commas** → US grouping style; strip all commas. + ``"3,080,000"`` → ``"3080000"``, ``"12,34,567"`` → ``"1234567"`` + * **Single comma, group of exactly 3 digits after** → thousands separator; + strip the comma. ``"1,000"`` → ``"1000"``, ``"1,234"`` → ``"1234"`` + * **Single comma, 1–2 digits after** → European decimal comma; replace + with a period. ``"14,2"`` → ``"14.2"``, ``"3,14"`` → ``"3.14"`` + * **Anything else** → return unchanged (non-numeric strings fall back to + string comparison upstream). + + The function intentionally does *not* handle a single comma followed by + 4 or more digits (e.g. ``"1,0000"``), leaving those strings unchanged so + that ``float()`` will reject them and the caller falls back gracefully. + + Args: + number: a stripped string that may represent a numeric value. + + Returns: + The string with commas adjusted, ready to pass to ``float()``. + """ + if number.count(",") > 1: + # Multiple commas → US grouping (e.g. "3,080,000", "12,34,567") + return number.replace(",", "") + + match = re.fullmatch(r"(-?\d+),(\d+)", number) + if match: + decimal_digits = match.group(2) + if len(decimal_digits) == 3: + # Single comma + exactly 3 digits → thousands separator + return number.replace(",", "") + if len(decimal_digits) <= 2: + # Single comma + 1–2 digits → European decimal comma + return number.replace(",", ".") + + return number diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py index 7910eadf8..85e7c64dd 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py @@ -5,6 +5,7 @@ import numpy as np from .evaluate_utils.evaluate_factory import get_evaluator +from .evaluate_utils.utils import _fix_comma def find_isnan(samp): @@ -60,10 +61,11 @@ def fix_number(number): " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft") ).strip() copy_ans = copy_ans.strip() - copy_ans = copy_ans.replace(",", ".").replace(" square kilometers", "") + copy_ans = copy_ans.replace(" square kilometers", "") + copy_ans = _fix_comma(copy_ans) try: return float(copy_ans), True - except: + except (ValueError, TypeError): return number, False elif type(number) == int: return float(number), True @@ -102,7 +104,7 @@ def fix_prediction(prediction, gold_answer, evaluator): def question_scorer(prediction, gold_answer): try: prediction = json.loads(prediction) - except: + except (json.JSONDecodeError, TypeError, ValueError): prediction = prediction answer_list = ( diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index 4973d7158..d60cee7b8 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -4,6 +4,7 @@ import gymnasium as gym import pytest +from browsergym.assistantbench.evaluation.evaluate_utils.utils import _fix_comma from browsergym.assistantbench.evaluation.evaluator import question_scorer from browsergym.experiments.benchmark.metadata.utils import ( task_list_from_metadata, @@ -52,6 +53,93 @@ def test_evaluate(original_id: str): assert has_ans == expected_has_ans +# --------------------------------------------------------------------------- +# Comma-disambiguation tests — issue #404 +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "number_str, expected", + [ + # Multiple commas → US thousands grouping, all commas stripped + ("3,080,000", "3080000"), + ("1,010,000", "1010000"), + ("12,34,567", "1234567"), # non-standard grouping still stripped + # Single comma + exactly 3 digits → thousands separator + ("1,000", "1000"), + ("1,234", "1234"), + ("-1,000", "-1000"), # negative thousands + # Single comma + 1–2 digits → European decimal comma + ("14,2", "14.2"), + ("3,14", "3.14"), + # Single comma + 4+ digits → left unchanged (ambiguous; float() will reject) + ("1,0000", "1,0000"), + # No comma → untouched + ("3080000", "3080000"), + ("14.2", "14.2"), + ("hello", "hello"), + ], +) +def test_fix_comma(number_str: str, expected: str): + """Unit-test _fix_comma directly to cover every branch.""" + assert _fix_comma(number_str) == expected + + +@pytest.mark.parametrize( + "prediction, gold_answer, expected_score", + [ + # ── Previously broken: correct answers scored 0 ───────────────────── + # dev-split validation.26: house price with US thousands separator + ("3,080,000", "3080000", 1.0), + # dev-split validation.3: apartment price + ("1,010,000", "1010000", 1.0), + # Simple thousands + ("1,000", "1000", 1.0), + # ── Previously broken: wrong answers scored 1.0 ────────────────────── + # "1,000" → 1.0 which matched gold 1 — 1000x error scored as perfect + ("1,000", "1", 0.0), + ("1,010", "1.01", 0.0), + # ── Must stay correct: European decimal comma ──────────────────────── + # dev-split validation.2 + ("14,2", "14.2", 1.0), + ("3,14", "3.14", 1.0), + # ── Control cases: no commas, must be unaffected ───────────────────── + ("1000", "1000", 1.0), + ("3080000", "3080000", 1.0), + ], +) +def test_evaluate_comma_numbers(prediction: str, gold_answer: str, expected_score: float): + """End-to-end question_scorer test for comma-formatted numeric answers.""" + score, _ = question_scorer(prediction, gold_answer) + assert score == expected_score, ( + f"question_scorer({prediction!r}, {gold_answer!r}) " + f"returned {score}, expected {expected_score}" + ) + + +def test_evaluate_comma_numbers_in_dicts(): + """ + Validates the dict-evaluation path (evaluate_dicts.py) also applies the + comma fix. A JSON value "1,000" against a gold numeric 1000 should score + 1.0, not 0.5 (which the old code returned due to type mismatch after + misparse). + """ + prediction = '[{"sender": "DHL", "price": "1,000"}]' + gold_answer = '{"sender": "DHL", "price": 1000}' + + score, _ = question_scorer(prediction, gold_answer) + assert score == 1.0, f"Dict scorer returned {score}, expected 1.0" + + +def test_evaluate_comma_numbers_negative(): + """Negative numbers with thousands separators should be handled.""" + score, _ = question_scorer("-1,000", "-1000") + assert score == 1.0 + + +# --------------------------------------------------------------------------- + + @pytest.mark.parametrize( "original_id", [id for id in data_points.keys() if isinstance(data_points[id]["answer"], (str, float, int))], @@ -75,3 +163,4 @@ def test_evaluate_within_env(original_id: str): assert reward == expected_score env.close() + From ccd59eef87eae179c8c2f59f71d092522cc2788e Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Wed, 28 Oct 2026 21:55:02 +0530 Subject: [PATCH 4/8] fix: remove trailing blank line in test_evaluation.py (black formatting) --- tests/assistantbench/test_evaluation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index d60cee7b8..b81b985d5 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -163,4 +163,3 @@ def test_evaluate_within_env(original_id: str): assert reward == expected_score env.close() - From 6c63db82abe1901899f0c2061c79936d07272728 Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Wed, 28 Oct 2026 23:08:29 +0530 Subject: [PATCH 5/8] fix: handle thousands-separator comma with decimal part in _fix_comma Update regex from (-?\d+),(\d+) to (-?\d+),(\d+)(\.\d+)? so that strings like '1,000.50' and '1,234.56' are correctly recognised as thousands-separator notation and have the comma stripped, yielding '1000.50' and '1234.56' respectively. Add test cases for this edge case in test_fix_comma and test_evaluate_comma_numbers. --- .../assistantbench/evaluation/evaluate_utils/utils.py | 5 +++-- tests/assistantbench/test_evaluation.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py index 1db8cd40c..2466970e9 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py @@ -36,7 +36,8 @@ def _fix_comma(number: str) -> str: * **Multiple commas** → US grouping style; strip all commas. ``"3,080,000"`` → ``"3080000"``, ``"12,34,567"`` → ``"1234567"`` * **Single comma, group of exactly 3 digits after** → thousands separator; - strip the comma. ``"1,000"`` → ``"1000"``, ``"1,234"`` → ``"1234"`` + strip the comma. ``"1,000"`` → ``"1000"``, ``"1,234"`` → ``"1234"``. + An optional decimal part is also handled: ``"1,000.50"`` → ``"1000.50"``. * **Single comma, 1–2 digits after** → European decimal comma; replace with a period. ``"14,2"`` → ``"14.2"``, ``"3,14"`` → ``"3.14"`` * **Anything else** → return unchanged (non-numeric strings fall back to @@ -56,7 +57,7 @@ def _fix_comma(number: str) -> str: # Multiple commas → US grouping (e.g. "3,080,000", "12,34,567") return number.replace(",", "") - match = re.fullmatch(r"(-?\d+),(\d+)", number) + match = re.fullmatch(r"(-?\d+),(\d+)(\.\d+)?", number) if match: decimal_digits = match.group(2) if len(decimal_digits) == 3: diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index b81b985d5..f3dfc3714 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -69,6 +69,9 @@ def test_evaluate(original_id: str): ("1,000", "1000"), ("1,234", "1234"), ("-1,000", "-1000"), # negative thousands + # Single comma + 3 digits + decimal part → thousands separator with decimals + ("1,000.50", "1000.50"), + ("1,234.56", "1234.56"), # Single comma + 1–2 digits → European decimal comma ("14,2", "14.2"), ("3,14", "3.14"), @@ -95,6 +98,9 @@ def test_fix_comma(number_str: str, expected: str): ("1,010,000", "1010000", 1.0), # Simple thousands ("1,000", "1000", 1.0), + # Thousands separator with decimal part + ("1,000.50", "1000.50", 1.0), + ("1,234.56", "1234.56", 1.0), # ── Previously broken: wrong answers scored 1.0 ────────────────────── # "1,000" → 1.0 which matched gold 1 — 1000x error scored as perfect ("1,000", "1", 0.0), From d04f4171eb2f4e77863ce4c76a4aecec324785dd Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Fri, 30 Oct 2026 19:27:15 +0530 Subject: [PATCH 6/8] ci: trigger re-run of black format check From 0eb39bf0decfade8b7582922cdad576860d25c87 Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Fri, 30 Oct 2026 20:24:28 +0530 Subject: [PATCH 7/8] style: add module docstring to test_evaluation.py (black-formatted) --- tests/assistantbench/test_evaluation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index f3dfc3714..6b3fdabf1 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -1,3 +1,5 @@ +"""Tests for AssistantBench evaluation utilities (issue #404: comma disambiguation).""" + import json import pathlib From ee4e77010e8d5e4b3dce08b96b4a077ba3b41571 Mon Sep 17 00:00:00 2001 From: Vijaysrimari S Date: Wed, 4 Nov 2026 21:25:49 +0530 Subject: [PATCH 8/8] style: format test_evaluation.py with black Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/assistantbench/test_evaluation.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/assistantbench/test_evaluation.py b/tests/assistantbench/test_evaluation.py index 6b3fdabf1..17c1d9703 100644 --- a/tests/assistantbench/test_evaluation.py +++ b/tests/assistantbench/test_evaluation.py @@ -32,7 +32,9 @@ has_ans = data_point["has_ans"] data_points[original_id] = { - "task_id": task_list_from_metadata(metadata, {"original_id": original_id})[0], + "task_id": task_list_from_metadata(metadata, {"original_id": original_id})[ + 0 + ], "answer": answer, "gold_answer": gold_answer, "score": score, @@ -116,7 +118,9 @@ def test_fix_comma(number_str: str, expected: str): ("3080000", "3080000", 1.0), ], ) -def test_evaluate_comma_numbers(prediction: str, gold_answer: str, expected_score: float): +def test_evaluate_comma_numbers( + prediction: str, gold_answer: str, expected_score: float +): """End-to-end question_scorer test for comma-formatted numeric answers.""" score, _ = question_scorer(prediction, gold_answer) assert score == expected_score, ( @@ -150,7 +154,11 @@ def test_evaluate_comma_numbers_negative(): @pytest.mark.parametrize( "original_id", - [id for id in data_points.keys() if isinstance(data_points[id]["answer"], (str, float, int))], + [ + id + for id in data_points.keys() + if isinstance(data_points[id]["answer"], (str, float, int)) + ], ) @pytest.mark.slow def test_evaluate_within_env(original_id: str): @@ -165,7 +173,9 @@ def test_evaluate_within_env(original_id: str): obs, info = env.reset() assert not obs["last_action_error"] - obs, reward, terminated, truncated, info = env.step(f"send_msg_to_user({repr(str(answer))})") + obs, reward, terminated, truncated, info = env.step( + f"send_msg_to_user({repr(str(answer))})" + ) assert not obs["last_action_error"] assert terminated assert reward == expected_score