diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py index 76136c06a..31ebcdb0a 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluate_utils/evaluate_dicts.py @@ -31,6 +31,8 @@ def calc_recall(pred: Dict, gold: Dict, use_gold_for_eval: bool): recall.append(0) continue recall.append(evaluator(pred_value, gold_value)) + if not recall: + return 0 avg_recall = np.average(recall) return avg_recall @@ -62,7 +64,7 @@ def evaluate_pair_of_dicts(pred: Dict, gold: Dict): def evaluate_dicts(pred: List[Dict], gold: List[Dict]): - if not (type(pred) == dict or len(pred) == 0 or (type(pred) == list and type(pred[0]) == dict)): + if not (type(pred) == dict or len(pred) == 0 or (type(pred) == list and all(type(p) == dict for p in pred))): return 0 max_alignment_scores = _align_bags(pred, gold, evaluate_pair_of_dicts) return np.average(max_alignment_scores) diff --git a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py index 7910eadf8..4aef444cf 100644 --- a/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py +++ b/browsergym/assistantbench/src/browsergym/assistantbench/evaluation/evaluator.py @@ -114,12 +114,12 @@ def question_scorer(prediction, gold_answer): prediction, run_eval = fix_prediction(prediction, gold_answer, evaluator) has_ans = 1.0 - if (type(prediction) != float and len(prediction) == 0) or find_isnan(prediction): + if (hasattr(prediction, "__len__") and len(prediction) == 0) or find_isnan(prediction): has_ans = 0.0 if type(prediction) == list: if all( - (type(pred) not in {float, int} and len(pred) == 0) or find_isnan(pred) + (hasattr(pred, "__len__") and len(pred) == 0) or find_isnan(pred) for pred in prediction ): has_ans = 0