[Eval] Report failures and unknowns in results table

evals/README.md

@@ -75,10 +75,13 @@ state) and its rethink events in `agent/episode.json`; the count of chat-model c
 `result.json`. For `llm` the decisions are the chat calls that produced an `act`; a tick whose key was
 not offered is kept with `accepted: false`, counted as an `invalid_key`, and skipped by the replay. Its iteration
 cap is twice the game's budget, since the chat model spends turns on unknown keys.
-A run that reaches the iteration cap has `result_type: error` in `extra`; its score still counts. An episode that
-raised (a decisions failure, an env error) keeps `error` set: `summary.json` reports it under `errors`, computes
-the score statistics over the `scored` episodes only, and `evals.table` skips trials whose `result.json` has
-`exception_info`.
+A run that reaches the iteration cap has `result_type: error` in `extra`; its recorded reward still counts as an
+outcome because the episode did not raise. An episode that raised (a decisions failure, an env error) keeps
+`exception_info`: it counts as an attempt and an error, but does not enter score statistics. The table reports
+`attempts`, `errors`, and non-error `N` separately. Score uses only non-error trials with a recorded reward; an
+actual reward of `0` is retained, while a missing reward is reported as score-unknown. Time and cost include failed attempts when
+recorded and show their known-value denominator and unknown count; missing timestamps or cost are never treated as
+zero. The legacy steps, decisions, and chat-call means remain scoped to non-error trials.
 
 
 ### Live real-model recovery subset
@@ -186,7 +189,11 @@ act against a bare loop (loop overhead); and, later, Jev's top probability again
 `summary.json` also holds decisions, chat calls, tokens (`chat_input_tokens`, `chat_output_tokens`,
 `chat_cache_tokens`) and `cost_usd` (Jev at $0.042 per M input tokens; the chat model at OpenRouter's catalogue
 price for `MODEL_NAME`, or custom rates from `CHAT_USD_PER_M_*`, see [docs/configuration.md](../docs/configuration.md)). `python -m evals.table evals/results` prints one row per eval and model over every
-job folder. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across
+job folder. Its `s / attempt` and `$ / attempt` columns are means across all attempts with known values, including
+errors; `time (n/attempts)` and `cost (n/attempts)` show their denominators. The unknown columns count attempts
+without those values. The score column remains the mean with a 95% bootstrap interval over the non-error scored
+subset; `score (n)` states its count over all attempts, and `score unknown` counts non-error trials without a
+reward. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across
 the six types. Every model plays the same tile draws because 2048 seeds the page's `Math.random`.
 
 ## Showcase runs and replays

evals/replay/trial.py

@@ -53,7 +53,7 @@ def read_trial(path: Path) -> Trial:
         model=model_label(result),
         seed=int(seed_text) if seed_text.isdigit() else 0,
         score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0),
-        elapsed_s=float(metadata.get("elapsed_s") or window_s(result)),
+        elapsed_s=float(metadata.get("elapsed_s") if metadata.get("elapsed_s") is not None else window_s(result)),
         steps=len(decisions) if steps is None else int(steps),
         cost_usd=agent_result.get("cost_usd"),
         decisions=decisions,

evals/table.py

@@ -1,8 +1,9 @@
 # coding: utf-8
-"""One table over every job folder: per eval and model, score, seconds, steps, decisions, chat calls and dollars.
+"""One table over every job folder: scored outcomes and all-attempt operational totals.
 
 ``python -m evals.table evals/results`` reads the job folders that ``write_job`` produces, one ``result.json`` per
-trial. Rows are keyed by the results root's child folder; a trial with ``exception_info`` is left out and counted in the errors column.
+trial. Rows are keyed by the results root's child folder. Failed trials stay in all-attempt counts, timing, and cost
+when those values were recorded; only explicitly present rewards from non-error trials enter score statistics.
 """
 
 from __future__ import annotations
@@ -18,31 +19,61 @@
 
 from s1a.jobs import BOOTSTRAP_RESAMPLES, bootstrap_interval
 
-COLUMNS = ("eval", "model", "N", "errors", "score", "s / episode", "steps", "decisions", "chat calls", "$ / episode")
+COLUMNS = (
+    "eval",
+    "model",
+    "attempts",
+    "errors",
+    "N",
+    "score (n)",
+    "score unknown",
+    "score",
+    "scored median s",
+    "steps",
+    "decisions",
+    "chat calls",
+    "scored $ / episode",
+    "time (n/attempts)",
+    "s / attempt",
+    "time unknown",
+    "cost (n/attempts)",
+    "$ / attempt",
+    "cost unknown",
+)
 
 
 @dataclass(frozen=True)
 class Trial:
     eval_name: str
     model: str
     errored: bool  # result.json holds exception_info: no score, counted in the errors column only
-    score: float
-    elapsed_s: float
+    score: float | None
+    elapsed_s: float | None
     steps: int | None  # None when the runner recorded no step count
     decisions: int
     chat_calls: int
     cost_usd: float | None
 
 
-def window_s(result: dict[str, Any]) -> float:
-    """Seconds between a trial's recorded start and finish."""
+def recorded_window_s(result: dict[str, Any]) -> float | None:
+    """Seconds between a trial's recorded start and finish, or None if either is missing."""
     window = result.get("agent_execution") or {}
     started, finished = window.get("started_at"), window.get("finished_at")
     if not started or not finished:
-        return 0.0
+        return None
     return (datetime.fromisoformat(finished) - datetime.fromisoformat(started)).total_seconds()
 
 
+def window_s(result: dict[str, Any]) -> float:
+    """Seconds between a trial's recorded start and finish (zero if either timestamp is missing).
+
+    Keep the historic float return contract for replay consumers; reporting code that needs to distinguish
+    missing timing data should use ``recorded_window_s``.
+    """
+    elapsed = recorded_window_s(result)
+    return 0.0 if elapsed is None else elapsed
+
+
 def model_label(result: dict[str, Any]) -> str:
     """The model a trial's ``result.json`` names: ``jev``, ``llm``, ``random`` or a rule name, after any ``<suite>/`` prefix."""
     name = str((result.get("agent_info") or {}).get("name") or "")
@@ -58,12 +89,16 @@ def read_trial(trial_dir: Path, *, eval_name: str) -> Trial | None:
     agent_result = result.get("agent_result") or {}
     metadata = agent_result.get("metadata") or {}
     steps = metadata.get("steps")
+    reward = ((result.get("verifier_result") or {}).get("rewards") or {}).get("reward")
+    elapsed = metadata.get("elapsed_s")
+    if elapsed is None:
+        elapsed = recorded_window_s(result)
     return Trial(
         eval_name=eval_name,
         model=model_label(result),
         errored=result.get("exception_info") is not None,
-        score=float(((result.get("verifier_result") or {}).get("rewards") or {}).get("reward") or 0.0),
-        elapsed_s=float(metadata.get("elapsed_s") or window_s(result)),
+        score=float(reward) if reward is not None else None,
+        elapsed_s=float(elapsed) if elapsed is not None else None,
         steps=int(steps) if steps is not None else None,
         decisions=int(metadata.get("decisions") or 0),
         chat_calls=int(metadata.get("chat_calls") or 0),
@@ -82,34 +117,48 @@ def read_results(root: Path) -> list[Trial]:
 
 
 def rows(trials: list[Trial]) -> list[dict[str, Any]]:
-    """One row per (eval, model): N and errors, then the mean score with its 95 % bootstrap interval, medians and
-    means per played episode; a model whose every trial errored has no score."""
+    """One row per (eval, model). Score uses non-error trials with a recorded reward; time and cost use every
+    attempt that recorded that value. Each count is exposed so missing values never silently become zero."""
     groups: dict[tuple[str, str], list[Trial]] = {}
     for trial in trials:
         groups.setdefault((trial.eval_name, trial.model), []).append(trial)
     table = []
     for (eval_name, model), all_members in sorted(groups.items()):
         members = [member for member in all_members if not member.errored]
-        scores = [member.score for member in members]
-        costs = [member.cost_usd for member in members]
+        scores = [member.score for member in members if member.score is not None]
+        scored_elapsed = [member.elapsed_s for member in members if member.elapsed_s is not None]
+        elapsed = [member.elapsed_s for member in all_members if member.elapsed_s is not None]
+        scored_costs = [member.cost_usd for member in members]
+        costs = [member.cost_usd for member in all_members if member.cost_usd is not None]
         steps = [member.steps for member in members if member.steps is not None]
         table.append(
             {
                 "eval": eval_name,
                 "model": model,
                 "N": len(members),
                 "errors": len(all_members) - len(members),
+                "attempts": len(all_members),
+                "score_n": len(scores),
+                "score_unknown": len(members) - len(scores),
                 "mean_score": round(statistics.mean(scores), 3) if scores else None,
                 "ci95": bootstrap_interval(scores, resamples=BOOTSTRAP_RESAMPLES, seed=0) if scores else None,
-                "median_s": round(statistics.median(member.elapsed_s for member in members), 1) if members else None,
+                "time_n": len(elapsed),
+                "mean_elapsed_all_s": round(statistics.mean(elapsed), 1) if elapsed else None,
+                "time_unknown": len(all_members) - len(elapsed),
+                "median_s": round(statistics.median(scored_elapsed), 1) if scored_elapsed else None,
                 "mean_steps": round(statistics.mean(steps), 1) if steps else None,
                 "mean_decisions": round(statistics.mean(member.decisions for member in members), 1)
                 if members
                 else None,
                 "mean_chat_calls": round(statistics.mean(member.chat_calls for member in members), 1)
                 if members
                 else None,
-                "mean_cost_usd": round(statistics.mean(costs), 4) if costs and None not in costs else None,
+                "mean_cost_usd": round(statistics.mean(scored_costs), 4)
+                if scored_costs and None not in scored_costs
+                else None,
+                "cost_n": len(costs),
+                "mean_cost_all_usd": round(statistics.mean(costs), 6) if costs else None,
+                "cost_unknown": len(all_members) - len(costs),
             }
         )
     return table
@@ -118,15 +167,19 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]:
 def markdown(table: list[dict[str, Any]]) -> str:
     lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)]
     for row in table:
-        cost = "n/a" if row["mean_cost_usd"] is None else f"{row['mean_cost_usd']:.4f}"
+        cost = "n/a" if row["mean_cost_all_usd"] is None else f"{row['mean_cost_all_usd']:.6f}"
+        scored_cost = "n/a" if row["mean_cost_usd"] is None else f"{row['mean_cost_usd']:.4f}"
         score = "n/a" if row["ci95"] is None else f"{row['mean_score']} [{row['ci95'][0]}, {row['ci95'][1]}]"
         cells = [
             row[key] if row[key] is not None else "n/a"
             for key in ("median_s", "mean_steps", "mean_decisions", "mean_chat_calls")
         ]
         lines.append(
-            f"| {row['eval']} | {row['model']} | {row['N']} | {row['errors']} | {score} | "
-            f"{cells[0]} | {cells[1]} | {cells[2]} | {cells[3]} | {cost} |"
+            f"| {row['eval']} | {row['model']} | {row['attempts']} | {row['errors']} | {row['N']} | "
+            f"{row['score_n']}/{row['attempts']} | {row['score_unknown']} | {score} | "
+            f"{cells[0]} | {cells[1]} | {cells[2]} | {cells[3]} | {scored_cost} | "
+            f"{row['time_n']}/{row['attempts']} | {row['mean_elapsed_all_s'] if row['mean_elapsed_all_s'] is not None else 'n/a'} | "
+            f"{row['time_unknown']} | {row['cost_n']}/{row['attempts']} | {cost} | {row['cost_unknown']} |"
         )
     return "\n".join(lines)
 

tests/test_evals_table.py

@@ -54,12 +54,14 @@ def test_rows_merge_job_folders_per_model(self) -> None:
 
         by_key = {(row["eval"], row["model"]): row for row in table}
         blackjack = by_key[("blackjack", "jev")]
-        self.assertEqual((blackjack["N"], blackjack["mean_score"], blackjack["median_s"]), (3, 0.333, 1.5))
         self.assertEqual(
-            (blackjack["mean_steps"], blackjack["mean_decisions"], blackjack["mean_cost_usd"]), (2, 2, 0.0001)
+            (blackjack["N"], blackjack["attempts"], blackjack["score_n"], blackjack["mean_score"]),
+            (3, 3, 3, 0.333),
         )
-        text = markdown(table)
-        self.assertIn("| blackjack | jev | 3 | 0 |", text)
+        self.assertEqual(
+            (blackjack["mean_steps"], blackjack["mean_decisions"], blackjack["mean_cost_all_usd"]), (2, 2, 0.0001)
+        )
+        self.assertIn("| blackjack | jev | 3 | 0 | 3 | 3/3 | 0 |", markdown(table))
 
     def test_errored_trials_are_counted_and_not_scored(self) -> None:
         failed = _episode(1, 0.0, 0.0)
@@ -69,8 +71,12 @@ def test_errored_trials_are_counted_and_not_scored(self) -> None:
             write_job("blackjack", [_episode(0, 1.0, 0.0001), failed], results_dir=root)
             write_job("blackjack", [failed], results_dir=root)
             (row,) = rows(read_results(root))
-        self.assertEqual((row["N"], row["errors"], row["mean_score"], row["mean_cost_usd"]), (1, 2, 1.0, 0.0001))
-        self.assertIn("| blackjack | jev | 1 | 2 | 1.0 [1.0, 1.0] |", markdown([row]))
+        self.assertEqual(
+            (row["N"], row["attempts"], row["errors"], row["mean_score"], row["mean_cost_all_usd"]),
+            (1, 3, 2, 1.0, 0.000033),
+        )
+        self.assertEqual((row["cost_n"], row["cost_unknown"]), (3, 0))
+        self.assertIn("| blackjack | jev | 3 | 2 | 1 | 1/3 | 0 | 1.0 [1.0, 1.0] |", markdown([row]))
 
     def test_a_model_whose_every_trial_errored_has_no_score(self) -> None:
         failed = _episode(0, 0.0, None)
@@ -79,7 +85,8 @@ def test_a_model_whose_every_trial_errored_has_no_score(self) -> None:
             write_job("blackjack", [failed], results_dir=Path(tmp))
             (row,) = rows(read_results(Path(tmp)))
         self.assertEqual((row["N"], row["errors"], row["mean_score"]), (0, 1, None))
-        self.assertIn("| blackjack | jev | 0 | 1 | n/a | n/a | n/a | n/a | n/a | n/a |", markdown([row]))
+        self.assertEqual((row["attempts"], row["errors"], row["score_n"], row["mean_score"]), (1, 1, 0, None))
+        self.assertIn("| blackjack | jev | 1 | 1 | 0 | 0/1 | 0 | n/a |", markdown([row]))
 
     def test_rows_are_keyed_by_the_results_folder_not_the_recorded_source(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
@@ -97,5 +104,45 @@ def test_an_unknown_cost_makes_the_model_cost_unknown(self) -> None:
             root = Path(tmp)
             write_job("blackjack", [_episode(0, 1.0, 0.0001), _episode(1, 1.0, None)], results_dir=root)
             (row,) = rows(read_results(root))
-        self.assertIsNone(row["mean_cost_usd"])
+        self.assertEqual(row["mean_cost_all_usd"], 0.0001)
+        self.assertEqual((row["cost_n"], row["cost_unknown"]), (1, 1))
         self.assertIn("n/a", markdown([row]))
+
+    def test_missing_reward_and_timestamps_stay_unknown_but_zero_reward_and_cost_are_preserved(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            zero = _episode(0, 0.0, 0.0)
+            write_job("blackjack", [zero], results_dir=root)
+            result_file = next(root.glob("*/*/*/result.json"))
+            import json
+
+            result = json.loads(result_file.read_text())
+            result["agent_result"]["metadata"].pop("elapsed_s", None)
+            result["agent_execution"] = {}
+            (result["verifier_result"]["rewards"]).pop("reward", None)
+            result_file.write_text(json.dumps(result))
+            (row,) = rows(read_results(root))
+        self.assertEqual(
+            (row["attempts"], row["N"], row["score_n"], row["score_unknown"], row["mean_score"]),
+            (1, 1, 0, 1, None),
+        )
+        self.assertEqual((row["time_n"], row["time_unknown"], row["mean_elapsed_all_s"]), (0, 1, None))
+        self.assertEqual((row["cost_n"], row["cost_unknown"], row["mean_cost_all_usd"]), (1, 0, 0.0))
+
+    def test_error_cost_and_elapsed_are_included_when_recorded_and_unknowns_are_counted(self) -> None:
+        failed = _episode(1, 0.0, 0.0)
+        failed.error = "timeout"
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            write_job("blackjack", [failed], results_dir=root)
+            result_file = next(root.glob("*/*/*/result.json"))
+            import json
+
+            result = json.loads(result_file.read_text())
+            result["agent_result"].pop("cost_usd", None)
+            (result["agent_result"].get("metadata") or {}).pop("elapsed_s", None)
+            result["agent_execution"] = {}
+            result_file.write_text(json.dumps(result))
+            (row,) = rows(read_results(root))
+        self.assertEqual((row["attempts"], row["errors"], row["N"]), (1, 1, 0))
+        self.assertEqual((row["cost_n"], row["cost_unknown"], row["time_n"], row["time_unknown"]), (0, 1, 0, 1))

tests/test_replay.py

@@ -238,6 +238,28 @@ def test_the_pair_page_draws_the_frame_at_the_clock(self) -> None:
 
 
 class TestTrial(TestCase):
+    def test_read_trial_preserves_recorded_zero_elapsed_without_timestamps(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            jev_dir, _ = _write_pair(Path(tmp))
+            result_file = jev_dir / "result.json"
+            result = json.loads(result_file.read_text(encoding="utf-8"))
+            result["agent_result"]["metadata"]["elapsed_s"] = 0
+            result["agent_execution"] = {}
+            result_file.write_text(json.dumps(result), encoding="utf-8")
+            trial = read_trial(jev_dir)
+        self.assertEqual(trial.elapsed_s, 0.0)
+
+    def test_read_trial_uses_legacy_zero_when_elapsed_and_timestamps_are_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            jev_dir, _ = _write_pair(Path(tmp))
+            result_file = jev_dir / "result.json"
+            result = json.loads(result_file.read_text(encoding="utf-8"))
+            result["agent_result"]["metadata"].pop("elapsed_s", None)
+            result["agent_execution"] = {}
+            result_file.write_text(json.dumps(result), encoding="utf-8")
+            trial = read_trial(jev_dir)
+        self.assertEqual(trial.elapsed_s, 0.0)
+
     def test_read_trial_reads_the_harbor_folder(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             jev_dir, _ = _write_pair(Path(tmp))