From 4896e4269a8d3f72f5e32364fbcf9d2b6d60ccaf Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:31:57 +0800 Subject: [PATCH 1/6] [Eval] Include failed attempts in results table --- evals/README.md | 17 +++++++---- evals/table.py | 58 +++++++++++++++++++++++------------ tests/test_evals_table.py | 63 ++++++++++++++++++++++++++++++++++----- 3 files changed, 106 insertions(+), 32 deletions(-) diff --git a/evals/README.md b/evals/README.md index 2983f5a..e8230ca 100644 --- a/evals/README.md +++ b/evals/README.md @@ -44,10 +44,13 @@ state) and its rethink events in `agent/episode.json`; the count of chat-model c `result.json`. For `llm` the decisions are the chat calls that produced an `act`; a tick whose key was not offered is kept with `accepted: false`, counted as an `invalid_key`, and skipped by the replay. Its iteration cap is twice the game's budget, since the chat model spends turns on unknown keys. -A run that reaches the iteration cap has `result_type: error` in `extra`; its score still counts. An episode that -raised (a decisions failure, an env error) keeps `error` set: `summary.json` reports it under `errors`, computes -the score statistics over the `scored` episodes only, and `evals.table` skips trials whose `result.json` has -`exception_info`. +A run that reaches the iteration cap has `result_type: error` in `extra`; its recorded reward still counts as an +outcome because the episode did not raise. An episode that raised (a decisions failure, an env error) keeps +`exception_info`: it counts as an attempt and an error, but does not enter score statistics. The table reports +`attempts`, `errors`, and non-error `N` separately. Score uses only non-error trials with a recorded reward; an +actual reward of `0` is retained, while a missing reward is reported as score-unknown. Time and cost include failed attempts when +recorded and show their known-value denominator and unknown count; missing timestamps or cost are never treated as +zero. The legacy steps, decisions, and chat-call means remain scoped to non-error trials. ## Setup @@ -68,7 +71,11 @@ act against a bare loop (loop overhead); and, later, Jev's top probability again `summary.json` also holds decisions, chat calls, tokens (`chat_input_tokens`, `chat_output_tokens`, `chat_cache_tokens`) and `cost_usd` (Jev at $0.042 per M input tokens; the chat model at OpenRouter's catalogue price for `MODEL_NAME`, or custom rates from `CHAT_USD_PER_M_*`, see [docs/configuration.md](../docs/configuration.md)). `python -m evals.table evals/results` prints one row per eval and model over every -job folder. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across +job folder. Its `s / attempt` and `$ / attempt` columns are means across all attempts with known values, including +errors; `time (n/attempts)` and `cost (n/attempts)` show their denominators. The unknown columns count attempts +without those values. The score column remains the mean with a 95% bootstrap interval over the non-error scored +subset; `score (n)` states its count over all attempts, and `score unknown` counts non-error trials without a +reward. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across the six types. Every model plays the same tile draws because 2048 seeds the page's `Math.random`. ## Showcase runs and replays diff --git a/evals/table.py b/evals/table.py index 31d9926..51a4ff0 100644 --- a/evals/table.py +++ b/evals/table.py @@ -1,8 +1,9 @@ # coding: utf-8 -"""One table over every job folder: per eval and model, score, seconds, steps, decisions, chat calls and dollars. +"""One table over every job folder: scored outcomes and all-attempt operational totals. ``python -m evals.table evals/results`` reads the job folders that ``write_job`` produces, one ``result.json`` per -trial. Rows are keyed by the results root's child folder; a trial with ``exception_info`` is left out and counted in the errors column. +trial. Rows are keyed by the results root's child folder. Failed trials stay in all-attempt counts, timing, and cost +when those values were recorded; only explicitly present rewards from non-error trials enter score statistics. """ from __future__ import annotations @@ -18,7 +19,11 @@ from s1a.jobs import BOOTSTRAP_RESAMPLES, bootstrap_interval -COLUMNS = ("eval", "model", "N", "errors", "score", "s / episode", "steps", "decisions", "chat calls", "$ / episode") +COLUMNS = ( + "eval", "model", "attempts", "errors", "N", "score (n)", "score unknown", "score", + "time (n/attempts)", "s / attempt", "time unknown", "steps", "decisions", "chat calls", + "cost (n/attempts)", "$ / attempt", "cost unknown", +) @dataclass(frozen=True) @@ -26,20 +31,20 @@ class Trial: eval_name: str model: str errored: bool # result.json holds exception_info: no score, counted in the errors column only - score: float - elapsed_s: float + score: float | None + elapsed_s: float | None steps: int | None # None when the runner recorded no step count decisions: int chat_calls: int cost_usd: float | None -def window_s(result: dict[str, Any]) -> float: +def window_s(result: dict[str, Any]) -> float | None: """Seconds between a trial's recorded start and finish.""" window = result.get("agent_execution") or {} started, finished = window.get("started_at"), window.get("finished_at") if not started or not finished: - return 0.0 + return None return (datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds() @@ -58,12 +63,16 @@ def read_trial(trial_dir: Path, *, eval_name: str) -> Trial | None: agent_result = result.get("agent_result") or {} metadata = agent_result.get("metadata") or {} steps = metadata.get("steps") + reward = ((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") + elapsed = metadata.get("elapsed_s") + if elapsed is None: + elapsed = window_s(result) return Trial( eval_name=eval_name, model=model_label(result), errored=result.get("exception_info") is not None, - score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0), - elapsed_s=float(metadata.get("elapsed_s") or window_s(result)), + score=float(reward) if reward is not None else None, + elapsed_s=float(elapsed) if elapsed is not None else None, steps=int(steps) if steps is not None else None, decisions=int(metadata.get("decisions") or 0), chat_calls=int(metadata.get("chat_calls") or 0), @@ -82,16 +91,17 @@ def read_results(root: Path) -> list[Trial]: def rows(trials: list[Trial]) -> list[dict[str, Any]]: - """One row per (eval, model): N and errors, then the mean score with its 95 % bootstrap interval, medians and - means per played episode; a model whose every trial errored has no score.""" + """One row per (eval, model). Score uses non-error trials with a recorded reward; time and cost use every + attempt that recorded that value. Each count is exposed so missing values never silently become zero.""" groups: dict[tuple[str, str], list[Trial]] = {} for trial in trials: groups.setdefault((trial.eval_name, trial.model), []).append(trial) table = [] for (eval_name, model), all_members in sorted(groups.items()): members = [member for member in all_members if not member.errored] - scores = [member.score for member in members] - costs = [member.cost_usd for member in members] + scores = [member.score for member in members if member.score is not None] + elapsed = [member.elapsed_s for member in all_members if member.elapsed_s is not None] + costs = [member.cost_usd for member in all_members if member.cost_usd is not None] steps = [member.steps for member in members if member.steps is not None] table.append( { @@ -99,9 +109,14 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "model": model, "N": len(members), "errors": len(all_members) - len(members), + "attempts": len(all_members), + "score_n": len(scores), + "score_unknown": len(members) - len(scores), "mean_score": round(statistics.mean(scores), 3) if scores else None, "ci95": bootstrap_interval(scores, resamples=BOOTSTRAP_RESAMPLES, seed=0) if scores else None, - "median_s": round(statistics.median(member.elapsed_s for member in members), 1) if members else None, + "time_n": len(elapsed), + "mean_elapsed_all_s": round(statistics.mean(elapsed), 1) if elapsed else None, + "time_unknown": len(all_members) - len(elapsed), "mean_steps": round(statistics.mean(steps), 1) if steps else None, "mean_decisions": round(statistics.mean(member.decisions for member in members), 1) if members @@ -109,7 +124,9 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "mean_chat_calls": round(statistics.mean(member.chat_calls for member in members), 1) if members else None, - "mean_cost_usd": round(statistics.mean(costs), 4) if costs and None not in costs else None, + "cost_n": len(costs), + "mean_cost_all_usd": round(statistics.mean(costs), 4) if costs else None, + "cost_unknown": len(all_members) - len(costs), } ) return table @@ -118,15 +135,18 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: def markdown(table: list[dict[str, Any]]) -> str: lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)] for row in table: - cost = "n/a" if row["mean_cost_usd"] is None else f"{row['mean_cost_usd']:.4f}" + cost = "n/a" if row["mean_cost_all_usd"] is None else f"{row['mean_cost_all_usd']:.4f}" score = "n/a" if row["ci95"] is None else f"{row['mean_score']} [{row['ci95'][0]}, {row['ci95'][1]}]" cells = [ row[key] if row[key] is not None else "n/a" - for key in ("median_s", "mean_steps", "mean_decisions", "mean_chat_calls") + for key in ("mean_elapsed_all_s", "mean_steps", "mean_decisions", "mean_chat_calls") ] lines.append( - f"| {row['eval']} | {row['model']} | {row['N']} | {row['errors']} | {score} | " - f"{cells[0]} | {cells[1]} | {cells[2]} | {cells[3]} | {cost} |" + f"| {row['eval']} | {row['model']} | {row['attempts']} | {row['errors']} | {row['N']} | " + f"{row['score_n']}/{row['attempts']} | {row['score_unknown']} | {score} | " + f"{row['time_n']}/{row['attempts']} | " + f"{cells[0]} | {row['time_unknown']} | {cells[1]} | {cells[2]} | {cells[3]} | " + f"{row['cost_n']}/{row['attempts']} | {cost} | {row['cost_unknown']} |" ) return "\n".join(lines) diff --git a/tests/test_evals_table.py b/tests/test_evals_table.py index 870405d..ffee375 100644 --- a/tests/test_evals_table.py +++ b/tests/test_evals_table.py @@ -54,12 +54,14 @@ def test_rows_merge_job_folders_per_model(self) -> None: by_key = {(row["eval"], row["model"]): row for row in table} blackjack = by_key[("blackjack", "jev")] - self.assertEqual((blackjack["N"], blackjack["mean_score"], blackjack["median_s"]), (3, 0.333, 1.5)) self.assertEqual( - (blackjack["mean_steps"], blackjack["mean_decisions"], blackjack["mean_cost_usd"]), (2, 2, 0.0001) + (blackjack["N"], blackjack["attempts"], blackjack["score_n"], blackjack["mean_score"]), + (3, 3, 3, 0.333), ) - text = markdown(table) - self.assertIn("| blackjack | jev | 3 | 0 |", text) + self.assertEqual( + (blackjack["mean_steps"], blackjack["mean_decisions"], blackjack["mean_cost_all_usd"]), (2, 2, 0.0001) + ) + self.assertIn("| blackjack | jev | 3 | 0 | 3 | 3/3 | 0 |", markdown(table)) def test_errored_trials_are_counted_and_not_scored(self) -> None: failed = _episode(1, 0.0, 0.0) @@ -69,8 +71,12 @@ def test_errored_trials_are_counted_and_not_scored(self) -> None: write_job("blackjack", [_episode(0, 1.0, 0.0001), failed], results_dir=root) write_job("blackjack", [failed], results_dir=root) (row,) = rows(read_results(root)) - self.assertEqual((row["N"], row["errors"], row["mean_score"], row["mean_cost_usd"]), (1, 2, 1.0, 0.0001)) - self.assertIn("| blackjack | jev | 1 | 2 | 1.0 [1.0, 1.0] |", markdown([row])) + self.assertEqual( + (row["N"], row["attempts"], row["errors"], row["mean_score"], row["mean_cost_all_usd"]), + (1, 3, 2, 1.0, 0.0001), + ) + self.assertEqual((row["cost_n"], row["cost_unknown"]), (3, 0)) + self.assertIn("| blackjack | jev | 3 | 2 | 1 | 1/3 | 0 | 1.0 [1.0, 1.0] |", markdown([row])) def test_a_model_whose_every_trial_errored_has_no_score(self) -> None: failed = _episode(0, 0.0, None) @@ -79,7 +85,8 @@ def test_a_model_whose_every_trial_errored_has_no_score(self) -> None: write_job("blackjack", [failed], results_dir=Path(tmp)) (row,) = rows(read_results(Path(tmp))) self.assertEqual((row["N"], row["errors"], row["mean_score"]), (0, 1, None)) - self.assertIn("| blackjack | jev | 0 | 1 | n/a | n/a | n/a | n/a | n/a | n/a |", markdown([row])) + self.assertEqual((row["attempts"], row["errors"], row["score_n"], row["mean_score"]), (1, 1, 0, None)) + self.assertIn("| blackjack | jev | 1 | 1 | 0 | 0/1 | 0 | n/a |", markdown([row])) def test_rows_are_keyed_by_the_results_folder_not_the_recorded_source(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -97,5 +104,45 @@ def test_an_unknown_cost_makes_the_model_cost_unknown(self) -> None: root = Path(tmp) write_job("blackjack", [_episode(0, 1.0, 0.0001), _episode(1, 1.0, None)], results_dir=root) (row,) = rows(read_results(root)) - self.assertIsNone(row["mean_cost_usd"]) + self.assertIsNone(row["mean_cost_all_usd"]) + self.assertEqual((row["cost_n"], row["cost_unknown"]), (1, 1)) self.assertIn("n/a", markdown([row])) + + def test_missing_reward_and_timestamps_stay_unknown_but_zero_reward_and_cost_are_preserved(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + zero = _episode(0, 0.0, 0.0) + write_job("blackjack", [zero], results_dir=root) + result_file = next(root.glob("*/*/*/result.json")) + import json + + result = json.loads(result_file.read_text()) + result["agent_result"]["metadata"].pop("elapsed_s", None) + result["agent_execution"] = {} + (result["verifier_result"]["rewards"]).pop("reward", None) + result_file.write_text(json.dumps(result)) + (row,) = rows(read_results(root)) + self.assertEqual( + (row["attempts"], row["N"], row["score_n"], row["score_unknown"], row["mean_score"]), + (1, 1, 0, 1, None), + ) + self.assertEqual((row["time_n"], row["time_unknown"], row["mean_elapsed_all_s"]), (0, 1, None)) + self.assertEqual((row["cost_n"], row["cost_unknown"], row["mean_cost_all_usd"]), (1, 0, 0.0)) + + def test_error_cost_and_elapsed_are_included_when_recorded_and_unknowns_are_counted(self) -> None: + failed = _episode(1, 0.0, 0.0) + failed.error = "timeout" + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + write_job("blackjack", [failed], results_dir=root) + result_file = next(root.glob("*/*/*/result.json")) + import json + + result = json.loads(result_file.read_text()) + result["agent_result"].pop("cost_usd", None) + (result["agent_result"].get("metadata") or {}).pop("elapsed_s", None) + result["agent_execution"] = {} + result_file.write_text(json.dumps(result)) + (row,) = rows(read_results(root)) + self.assertEqual((row["attempts"], row["errors"], row["N"]), (1, 1, 0)) + self.assertEqual((row["cost_n"], row["cost_unknown"], row["time_n"], row["time_unknown"]), (0, 1, 0, 1)) From 33c247b64931352a24bee1159562af23cb19489e Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:33:16 +0800 Subject: [PATCH 2/6] [Eval] Preserve scored metrics with all-attempt totals --- evals/table.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/evals/table.py b/evals/table.py index 51a4ff0..f75604e 100644 --- a/evals/table.py +++ b/evals/table.py @@ -21,8 +21,8 @@ COLUMNS = ( "eval", "model", "attempts", "errors", "N", "score (n)", "score unknown", "score", - "time (n/attempts)", "s / attempt", "time unknown", "steps", "decisions", "chat calls", - "cost (n/attempts)", "$ / attempt", "cost unknown", + "scored median s", "steps", "decisions", "chat calls", "scored $ / episode", + "time (n/attempts)", "s / attempt", "time unknown", "cost (n/attempts)", "$ / attempt", "cost unknown", ) @@ -100,7 +100,9 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: for (eval_name, model), all_members in sorted(groups.items()): members = [member for member in all_members if not member.errored] scores = [member.score for member in members if member.score is not None] + scored_elapsed = [member.elapsed_s for member in members if member.elapsed_s is not None] elapsed = [member.elapsed_s for member in all_members if member.elapsed_s is not None] + scored_costs = [member.cost_usd for member in members] costs = [member.cost_usd for member in all_members if member.cost_usd is not None] steps = [member.steps for member in members if member.steps is not None] table.append( @@ -117,6 +119,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "time_n": len(elapsed), "mean_elapsed_all_s": round(statistics.mean(elapsed), 1) if elapsed else None, "time_unknown": len(all_members) - len(elapsed), + "median_s": round(statistics.median(scored_elapsed), 1) if scored_elapsed else None, "mean_steps": round(statistics.mean(steps), 1) if steps else None, "mean_decisions": round(statistics.mean(member.decisions for member in members), 1) if members @@ -124,6 +127,9 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "mean_chat_calls": round(statistics.mean(member.chat_calls for member in members), 1) if members else None, + "mean_cost_usd": round(statistics.mean(scored_costs), 4) + if scored_costs and None not in scored_costs + else None, "cost_n": len(costs), "mean_cost_all_usd": round(statistics.mean(costs), 4) if costs else None, "cost_unknown": len(all_members) - len(costs), @@ -136,17 +142,18 @@ def markdown(table: list[dict[str, Any]]) -> str: lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)] for row in table: cost = "n/a" if row["mean_cost_all_usd"] is None else f"{row['mean_cost_all_usd']:.4f}" + scored_cost = "n/a" if row["mean_cost_usd"] is None else f"{row['mean_cost_usd']:.4f}" score = "n/a" if row["ci95"] is None else f"{row['mean_score']} [{row['ci95'][0]}, {row['ci95'][1]}]" cells = [ row[key] if row[key] is not None else "n/a" - for key in ("mean_elapsed_all_s", "mean_steps", "mean_decisions", "mean_chat_calls") + for key in ("median_s", "mean_steps", "mean_decisions", "mean_chat_calls") ] lines.append( f"| {row['eval']} | {row['model']} | {row['attempts']} | {row['errors']} | {row['N']} | " f"{row['score_n']}/{row['attempts']} | {row['score_unknown']} | {score} | " - f"{row['time_n']}/{row['attempts']} | " - f"{cells[0]} | {row['time_unknown']} | {cells[1]} | {cells[2]} | {cells[3]} | " - f"{row['cost_n']}/{row['attempts']} | {cost} | {row['cost_unknown']} |" + f"{cells[0]} | {cells[1]} | {cells[2]} | {cells[3]} | {scored_cost} | " + f"{row['time_n']}/{row['attempts']} | {row['mean_elapsed_all_s'] if row['mean_elapsed_all_s'] is not None else 'n/a'} | " + f"{row['time_unknown']} | {row['cost_n']}/{row['attempts']} | {cost} | {row['cost_unknown']} |" ) return "\n".join(lines) From 9418a39c45b8e6d5350650cdb173c724764e16f2 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:34:34 +0800 Subject: [PATCH 3/6] [Eval] Format attempt reporting changes --- evals/table.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/evals/table.py b/evals/table.py index f75604e..ab2468a 100644 --- a/evals/table.py +++ b/evals/table.py @@ -20,9 +20,25 @@ from s1a.jobs import BOOTSTRAP_RESAMPLES, bootstrap_interval COLUMNS = ( - "eval", "model", "attempts", "errors", "N", "score (n)", "score unknown", "score", - "scored median s", "steps", "decisions", "chat calls", "scored $ / episode", - "time (n/attempts)", "s / attempt", "time unknown", "cost (n/attempts)", "$ / attempt", "cost unknown", + "eval", + "model", + "attempts", + "errors", + "N", + "score (n)", + "score unknown", + "score", + "scored median s", + "steps", + "decisions", + "chat calls", + "scored $ / episode", + "time (n/attempts)", + "s / attempt", + "time unknown", + "cost (n/attempts)", + "$ / attempt", + "cost unknown", ) From 8c8851f2cf59a34a143c57d8f780a6dfb184cb97 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:37:43 +0800 Subject: [PATCH 4/6] [Eval] Preserve precision for failure-inclusive cost --- evals/table.py | 4 ++-- tests/test_evals_table.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/table.py b/evals/table.py index ab2468a..30b9434 100644 --- a/evals/table.py +++ b/evals/table.py @@ -147,7 +147,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: if scored_costs and None not in scored_costs else None, "cost_n": len(costs), - "mean_cost_all_usd": round(statistics.mean(costs), 4) if costs else None, + "mean_cost_all_usd": round(statistics.mean(costs), 6) if costs else None, "cost_unknown": len(all_members) - len(costs), } ) @@ -157,7 +157,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: def markdown(table: list[dict[str, Any]]) -> str: lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)] for row in table: - cost = "n/a" if row["mean_cost_all_usd"] is None else f"{row['mean_cost_all_usd']:.4f}" + cost = "n/a" if row["mean_cost_all_usd"] is None else f"{row['mean_cost_all_usd']:.6f}" scored_cost = "n/a" if row["mean_cost_usd"] is None else f"{row['mean_cost_usd']:.4f}" score = "n/a" if row["ci95"] is None else f"{row['mean_score']} [{row['ci95'][0]}, {row['ci95'][1]}]" cells = [ diff --git a/tests/test_evals_table.py b/tests/test_evals_table.py index ffee375..ea25be4 100644 --- a/tests/test_evals_table.py +++ b/tests/test_evals_table.py @@ -73,7 +73,7 @@ def test_errored_trials_are_counted_and_not_scored(self) -> None: (row,) = rows(read_results(root)) self.assertEqual( (row["N"], row["attempts"], row["errors"], row["mean_score"], row["mean_cost_all_usd"]), - (1, 3, 2, 1.0, 0.0001), + (1, 3, 2, 1.0, 0.000033), ) self.assertEqual((row["cost_n"], row["cost_unknown"]), (3, 0)) self.assertIn("| blackjack | jev | 3 | 2 | 1 | 1/3 | 0 | 1.0 [1.0, 1.0] |", markdown([row])) @@ -104,7 +104,7 @@ def test_an_unknown_cost_makes_the_model_cost_unknown(self) -> None: root = Path(tmp) write_job("blackjack", [_episode(0, 1.0, 0.0001), _episode(1, 1.0, None)], results_dir=root) (row,) = rows(read_results(root)) - self.assertIsNone(row["mean_cost_all_usd"]) + self.assertEqual(row["mean_cost_all_usd"], 0.0001) self.assertEqual((row["cost_n"], row["cost_unknown"]), (1, 1)) self.assertIn("n/a", markdown([row])) From a4a24587856421eb0b2d138e1b952097635b4fb7 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:57:29 +0800 Subject: [PATCH 5/6] Fix replay handling of unknown elapsed time --- evals/replay/trial.py | 2 +- evals/table.py | 15 ++++++++++++--- tests/test_replay.py | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/evals/replay/trial.py b/evals/replay/trial.py index a7ed5b6..97868ba 100644 --- a/evals/replay/trial.py +++ b/evals/replay/trial.py @@ -53,7 +53,7 @@ def read_trial(path: Path) -> Trial: model=model_label(result), seed=int(seed_text) if seed_text.isdigit() else 0, score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0), - elapsed_s=float(metadata.get("elapsed_s") or window_s(result)), + elapsed_s=float(metadata.get("elapsed_s") if metadata.get("elapsed_s") is not None else window_s(result)), steps=len(decisions) if steps is None else int(steps), cost_usd=agent_result.get("cost_usd"), decisions=decisions, diff --git a/evals/table.py b/evals/table.py index 30b9434..cc6b1e3 100644 --- a/evals/table.py +++ b/evals/table.py @@ -55,8 +55,8 @@ class Trial: cost_usd: float | None -def window_s(result: dict[str, Any]) -> float | None: - """Seconds between a trial's recorded start and finish.""" +def recorded_window_s(result: dict[str, Any]) -> float | None: + """Seconds between a trial's recorded start and finish, or None if either is missing.""" window = result.get("agent_execution") or {} started, finished = window.get("started_at"), window.get("finished_at") if not started or not finished: @@ -64,6 +64,15 @@ def window_s(result: dict[str, Any]) -> float | None: return (datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds() +def window_s(result: dict[str, Any]) -> float: + """Seconds between a trial's recorded start and finish (zero if either timestamp is missing). + + Keep the historic float return contract for replay consumers; reporting code that needs to distinguish + missing timing data should use ``recorded_window_s``. + """ + return recorded_window_s(result) or 0.0 + + def model_label(result: dict[str, Any]) -> str: """The model a trial's ``result.json`` names: ``jev``, ``llm``, ``random`` or a rule name, after any ``/`` prefix.""" name = str((result.get("agent_info") or {}).get("name") or "") @@ -82,7 +91,7 @@ def read_trial(trial_dir: Path, *, eval_name: str) -> Trial | None: reward = ((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") elapsed = metadata.get("elapsed_s") if elapsed is None: - elapsed = window_s(result) + elapsed = recorded_window_s(result) return Trial( eval_name=eval_name, model=model_label(result), diff --git a/tests/test_replay.py b/tests/test_replay.py index f545dc2..c1a2d5d 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -238,6 +238,28 @@ def test_the_pair_page_draws_the_frame_at_the_clock(self) -> None: class TestTrial(TestCase): + def test_read_trial_preserves_recorded_zero_elapsed_without_timestamps(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + jev_dir, _ = _write_pair(Path(tmp)) + result_file = jev_dir / "result.json" + result = json.loads(result_file.read_text(encoding="utf-8")) + result["agent_result"]["metadata"]["elapsed_s"] = 0 + result["agent_execution"] = {} + result_file.write_text(json.dumps(result), encoding="utf-8") + trial = read_trial(jev_dir) + self.assertEqual(trial.elapsed_s, 0.0) + + def test_read_trial_uses_legacy_zero_when_elapsed_and_timestamps_are_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + jev_dir, _ = _write_pair(Path(tmp)) + result_file = jev_dir / "result.json" + result = json.loads(result_file.read_text(encoding="utf-8")) + result["agent_result"]["metadata"].pop("elapsed_s", None) + result["agent_execution"] = {} + result_file.write_text(json.dumps(result), encoding="utf-8") + trial = read_trial(jev_dir) + self.assertEqual(trial.elapsed_s, 0.0) + def test_read_trial_reads_the_harbor_folder(self) -> None: with tempfile.TemporaryDirectory() as tmp: jev_dir, _ = _write_pair(Path(tmp)) From 91e1d54027b0c1db2772accdf0a249b37777534e Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Fri, 9 Oct 2026 22:58:29 +0800 Subject: [PATCH 6/6] Preserve signed recorded durations --- evals/table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/table.py b/evals/table.py index cc6b1e3..cc3b208 100644 --- a/evals/table.py +++ b/evals/table.py @@ -70,7 +70,8 @@ def window_s(result: dict[str, Any]) -> float: Keep the historic float return contract for replay consumers; reporting code that needs to distinguish missing timing data should use ``recorded_window_s``. """ - return recorded_window_s(result) or 0.0 + elapsed = recorded_window_s(result) + return 0.0 if elapsed is None else elapsed def model_label(result: dict[str, Any]) -> str: