fix(assistantbench): score empty dicts and mixed lists instead of crashing

browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py

@@ -31,6 +31,8 @@ def calc_recall(pred: Dict, gold: Dict, use_gold_for_eval: bool):
                 recall.append(0)
                 continue
             recall.append(evaluator(pred_value, gold_value))
+    if not recall:
+        return 0
     avg_recall = np.average(recall)
     return avg_recall
 
@@ -62,7 +64,7 @@ def evaluate_pair_of_dicts(pred: Dict, gold: Dict):
 
 
 def evaluate_dicts(pred: List[Dict], gold: List[Dict]):
-    if not (type(pred) == dict or len(pred) == 0 or (type(pred) == list and type(pred[0]) == dict)):
+    if not (type(pred) == dict or len(pred) == 0 or (type(pred) == list and all(type(p) == dict for p in pred))):
         return 0
     max_alignment_scores = _align_bags(pred, gold, evaluate_pair_of_dicts)
     return np.average(max_alignment_scores)

browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py

@@ -114,12 +114,12 @@ def question_scorer(prediction, gold_answer):
     prediction, run_eval = fix_prediction(prediction, gold_answer, evaluator)
 
     has_ans = 1.0
-    if (type(prediction) != float and len(prediction) == 0) or find_isnan(prediction):
+    if (hasattr(prediction, "__len__") and len(prediction) == 0) or find_isnan(prediction):
         has_ans = 0.0
 
     if type(prediction) == list:
         if all(
-            (type(pred) not in {float, int} and len(pred) == 0) or find_isnan(pred)
+            (hasattr(pred, "__len__") and len(pred) == 0) or find_isnan(pred)
             for pred in prediction
         ):
             has_ans = 0