fix(assistantbench): disambiguate thousands-separator commas from decimal commas in fix_number

browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from .utils import _align_bags
+from .utils import _align_bags, _fix_comma
 
 
 def calculate_f1_score(precision, recall):
@@ -43,7 +43,7 @@ def fix_number(number):
             " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft")
         ).strip()
         copy_ans = copy_ans.strip()
-        copy_ans = copy_ans.replace(",", ".")
+        copy_ans = _fix_comma(copy_ans)
         try:
             return float(copy_ans)
         except:

browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/utils.py

@@ -1,3 +1,4 @@
+import re
 from typing import Callable, List, Set
 
 import numpy as np
@@ -23,3 +24,47 @@ def _align_bags(
     for row, column in zip(row_ind, col_ind):
         max_scores[row] = max(max_scores[row], scores[row, column])
     return max_scores
+
+
+def _fix_comma(number: str) -> str:
+    """
+    Disambiguates US thousands-separator commas from European decimal commas
+    before passing a string to ``float()``.
+
+    Rules (applied in order):
+
+    * **Multiple commas** → US grouping style; strip all commas.
+      ``"3,080,000"`` → ``"3080000"``, ``"12,34,567"`` → ``"1234567"``
+    * **Single comma, group of exactly 3 digits after** → thousands separator;
+      strip the comma.  ``"1,000"`` → ``"1000"``, ``"1,234"`` → ``"1234"``.
+      An optional decimal part is also handled: ``"1,000.50"`` → ``"1000.50"``.
+    * **Single comma, 1–2 digits after** → European decimal comma; replace
+      with a period.  ``"14,2"`` → ``"14.2"``, ``"3,14"`` → ``"3.14"``
+    * **Anything else** → return unchanged (non-numeric strings fall back to
+      string comparison upstream).
+
+    The function intentionally does *not* handle a single comma followed by
+    4 or more digits (e.g. ``"1,0000"``), leaving those strings unchanged so
+    that ``float()`` will reject them and the caller falls back gracefully.
+
+    Args:
+        number: a stripped string that may represent a numeric value.
+
+    Returns:
+        The string with commas adjusted, ready to pass to ``float()``.
+    """
+    if number.count(",") > 1:
+        # Multiple commas → US grouping (e.g. "3,080,000", "12,34,567")
+        return number.replace(",", "")
+
+    match = re.fullmatch(r"(-?\d+),(\d+)(\.\d+)?", number)
+    if match:
+        decimal_digits = match.group(2)
+        if len(decimal_digits) == 3:
+            # Single comma + exactly 3 digits → thousands separator
+            return number.replace(",", "")
+        if len(decimal_digits) <= 2:
+            # Single comma + 1–2 digits → European decimal comma
+            return number.replace(",", ".")
+
+    return number

browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from .evaluate_utils.evaluate_factory import get_evaluator
+from .evaluate_utils.utils import _fix_comma
 
 
 def find_isnan(samp):
@@ -60,10 +61,11 @@ def fix_number(number):
             " ".join(" ".join(copy_ans.split("$")).split("%")).split("sqft")
         ).strip()
         copy_ans = copy_ans.strip()
-        copy_ans = copy_ans.replace(",", ".").replace(" square kilometers", "")
+        copy_ans = copy_ans.replace(" square kilometers", "")
+        copy_ans = _fix_comma(copy_ans)
         try:
             return float(copy_ans), True
-        except:
+        except (ValueError, TypeError):
             return number, False
     elif type(number) == int:
         return float(number), True
@@ -102,7 +104,7 @@ def fix_prediction(prediction, gold_answer, evaluator):
 def question_scorer(prediction, gold_answer):
     try:
         prediction = json.loads(prediction)
-    except:
+    except (json.JSONDecodeError, TypeError, ValueError):
         prediction = prediction
 
     answer_list = (

tests/assistantbench/test_evaluation.py

@@ -1,9 +1,12 @@
+"""Tests for AssistantBench evaluation utilities (issue #404: comma disambiguation)."""
+
 import json
 import pathlib
 
 import gymnasium as gym
 import pytest
 
+from browsergym.assistantbench.evaluation.evaluate_utils.utils import _fix_comma
 from browsergym.assistantbench.evaluation.evaluator import question_scorer
 from browsergym.experiments.benchmark.metadata.utils import (
     task_list_from_metadata,
@@ -29,7 +32,9 @@
         has_ans = data_point["has_ans"]
 
         data_points[original_id] = {
-            "task_id": task_list_from_metadata(metadata, {"original_id": original_id})[0],
+            "task_id": task_list_from_metadata(metadata, {"original_id": original_id})[
+                0
+            ],
             "answer": answer,
             "gold_answer": gold_answer,
             "score": score,
@@ -52,9 +57,108 @@ def test_evaluate(original_id: str):
     assert has_ans == expected_has_ans
 
 
+# ---------------------------------------------------------------------------
+# Comma-disambiguation tests — issue #404
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "number_str, expected",
+    [
+        # Multiple commas → US thousands grouping, all commas stripped
+        ("3,080,000", "3080000"),
+        ("1,010,000", "1010000"),
+        ("12,34,567", "1234567"),  # non-standard grouping still stripped
+        # Single comma + exactly 3 digits → thousands separator
+        ("1,000", "1000"),
+        ("1,234", "1234"),
+        ("-1,000", "-1000"),  # negative thousands
+        # Single comma + 3 digits + decimal part → thousands separator with decimals
+        ("1,000.50", "1000.50"),
+        ("1,234.56", "1234.56"),
+        # Single comma + 1–2 digits → European decimal comma
+        ("14,2", "14.2"),
+        ("3,14", "3.14"),
+        # Single comma + 4+ digits → left unchanged (ambiguous; float() will reject)
+        ("1,0000", "1,0000"),
+        # No comma → untouched
+        ("3080000", "3080000"),
+        ("14.2", "14.2"),
+        ("hello", "hello"),
+    ],
+)
+def test_fix_comma(number_str: str, expected: str):
+    """Unit-test _fix_comma directly to cover every branch."""
+    assert _fix_comma(number_str) == expected
+
+
+@pytest.mark.parametrize(
+    "prediction, gold_answer, expected_score",
+    [
+        # ── Previously broken: correct answers scored 0 ─────────────────────
+        # dev-split validation.26: house price with US thousands separator
+        ("3,080,000", "3080000", 1.0),
+        # dev-split validation.3: apartment price
+        ("1,010,000", "1010000", 1.0),
+        # Simple thousands
+        ("1,000", "1000", 1.0),
+        # Thousands separator with decimal part
+        ("1,000.50", "1000.50", 1.0),
+        ("1,234.56", "1234.56", 1.0),
+        # ── Previously broken: wrong answers scored 1.0 ──────────────────────
+        # "1,000" → 1.0 which matched gold 1 — 1000x error scored as perfect
+        ("1,000", "1", 0.0),
+        ("1,010", "1.01", 0.0),
+        # ── Must stay correct: European decimal comma ────────────────────────
+        # dev-split validation.2
+        ("14,2", "14.2", 1.0),
+        ("3,14", "3.14", 1.0),
+        # ── Control cases: no commas, must be unaffected ─────────────────────
+        ("1000", "1000", 1.0),
+        ("3080000", "3080000", 1.0),
+    ],
+)
+def test_evaluate_comma_numbers(
+    prediction: str, gold_answer: str, expected_score: float
+):
+    """End-to-end question_scorer test for comma-formatted numeric answers."""
+    score, _ = question_scorer(prediction, gold_answer)
+    assert score == expected_score, (
+        f"question_scorer({prediction!r}, {gold_answer!r}) "
+        f"returned {score}, expected {expected_score}"
+    )
+
+
+def test_evaluate_comma_numbers_in_dicts():
+    """
+    Validates the dict-evaluation path (evaluate_dicts.py) also applies the
+    comma fix.  A JSON value "1,000" against a gold numeric 1000 should score
+    1.0, not 0.5 (which the old code returned due to type mismatch after
+    misparse).
+    """
+    prediction = '[{"sender": "DHL", "price": "1,000"}]'
+    gold_answer = '{"sender": "DHL", "price": 1000}'
+
+    score, _ = question_scorer(prediction, gold_answer)
+    assert score == 1.0, f"Dict scorer returned {score}, expected 1.0"
+
+
+def test_evaluate_comma_numbers_negative():
+    """Negative numbers with thousands separators should be handled."""
+    score, _ = question_scorer("-1,000", "-1000")
+    assert score == 1.0
+
+
+# ---------------------------------------------------------------------------
+
+
 @pytest.mark.parametrize(
     "original_id",
-    [id for id in data_points.keys() if isinstance(data_points[id]["answer"], (str, float, int))],
+    [
+        id
+        for id in data_points.keys()
+        if isinstance(data_points[id]["answer"], (str, float, int))
+    ],
 )
 @pytest.mark.slow
 def test_evaluate_within_env(original_id: str):
@@ -69,7 +173,9 @@ def test_evaluate_within_env(original_id: str):
     obs, info = env.reset()
     assert not obs["last_action_error"]
 
-    obs, reward, terminated, truncated, info = env.step(f"send_msg_to_user({repr(str(answer))})")
+    obs, reward, terminated, truncated, info = env.step(
+        f"send_msg_to_user({repr(str(answer))})"
+    )
     assert not obs["last_action_error"]
     assert terminated
     assert reward == expected_score