From 284e3d727562423cba44be1076afd97c786ef7d8 Mon Sep 17 00:00:00 2001 From: chendanyang Date: Sat, 10 Oct 2026 07:36:41 +0800 Subject: [PATCH 1/2] Add planned attempt reconciliation --- evals/README.md | 27 ++++ evals/reconcile.py | 239 ++++++++++++++++++++++++++++++++++ evals/table.py | 12 +- tests/test_evals_reconcile.py | 160 +++++++++++++++++++++++ 4 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 evals/reconcile.py create mode 100644 tests/test_evals_reconcile.py diff --git a/evals/README.md b/evals/README.md index e8230ca..18fce1c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -78,6 +78,33 @@ subset; `score (n)` states its count over all attempts, and `score unknown` coun reward. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across the six types. Every model plays the same tile draws because 2048 seeds the page's `Math.random`. +### Planned-attempt coverage + +The table above can count only trials with a saved `result.json`. To include attempts that have not produced a +result, create a version-1 JSON manifest and pass it with `--attempt-plan`: + +```json +{ + "schema_version": 1, + "attempts": [ + {"id": "2048-jev-seed-0", "eval": "game2048", "model": "jev", "task_name": "game2048/0", "status": "planned"}, + {"id": "2048-jev-seed-1", "eval": "game2048", "model": "jev", "task_name": "game2048/1", "status": "interrupted", "reason": "worker exited before result.json was written"}, + {"id": "desktop-cua-1", "eval": "desktop", "model": "cua", "task_name": "desktop/1", "status": "unsupported", "reason": "display backend unavailable"} + ] +} +``` + +`planned` with no artifact is reported as `missing`; explicitly declared `interrupted` and `unsupported` attempts +remain separate. A saved exception result is an `error` and remains a recorded attempt. The report also flags +results absent from the manifest and results that contradict an `unsupported` declaration. It matches using the +existing result fields `(eval folder, model label, task_name)`, so that tuple must be unique in a plan; use distinct +task names (normally including the seed) for repeated tasks. Missing artifacts alone do not prove that a run was +interrupted. This additive report does not modify the score, error, timing, cost, or attempt semantics of the table. + +```sh +uv run python -m evals.table evals/results --attempt-plan evals/attempt-plan.json +``` + ## Showcase runs and replays The matrix runs headless and records nothing visual. Every episode still writes ``views`` into ``agent/episode.json``: diff --git a/evals/reconcile.py b/evals/reconcile.py new file mode 100644 index 0000000..8cff4e2 --- /dev/null +++ b/evals/reconcile.py @@ -0,0 +1,239 @@ +# coding: utf-8 +"""Reconcile planned evaluation attempts with saved trial result artifacts. + +Results are matched by the existing ``(eval folder, model label, task_name)`` record fields. The +reconciler reports denominator coverage and artifact mismatches; it does not compute or change scores. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +PlanStatus = Literal["planned", "interrupted", "unsupported"] +AttemptKey = tuple[str, str, str] + + +@dataclass(frozen=True) +class PlannedAttempt: + attempt_id: str + eval_name: str + model: str + task_name: str + status: PlanStatus + reason: str | None = None + + @property + def key(self) -> AttemptKey: + return self.eval_name, self.model, self.task_name + + +@dataclass(frozen=True) +class ResultArtifact: + eval_name: str + model: str + task_name: str + errored: bool + path: str + + @property + def key(self) -> AttemptKey: + return self.eval_name, self.model, self.task_name + + +@dataclass(frozen=True) +class ReconciledAttempt: + attempt_id: str + eval_name: str + model: str + task_name: str + status: str + reason: str | None = None + artifact: str | None = None + + +def load_plan(path: Path) -> list[PlannedAttempt]: + """Load and validate a version-1 attempt plan JSON file.""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read attempt plan {path}: {exc}") from exc + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ValueError("attempt plan must be an object with schema_version: 1") + raw_attempts = payload.get("attempts") + if not isinstance(raw_attempts, list) or not raw_attempts: + raise ValueError("attempt plan must contain a non-empty attempts list") + + attempts: list[PlannedAttempt] = [] + ids: set[str] = set() + keys: set[AttemptKey] = set() + for index, item in enumerate(raw_attempts): + if not isinstance(item, dict): + raise ValueError(f"attempts[{index}] must be an object") + text_fields = {name: item.get(name) for name in ("id", "eval", "model", "task_name", "status")} + if any(not isinstance(value, str) or not value.strip() for value in text_fields.values()): + raise ValueError(f"attempts[{index}] requires non-empty string id, eval, model, task_name and status") + attempt_id = str(item["id"]).strip() + eval_name = str(item["eval"]).strip() + model = str(item["model"]).strip() + task_name = str(item["task_name"]).strip() + status = str(item["status"]).strip() + if status not in {"planned", "interrupted", "unsupported"}: + raise ValueError(f"attempts[{index}] has unsupported status {status!r}") + reason = item.get("reason") + if status in {"interrupted", "unsupported"} and (not isinstance(reason, str) or not reason.strip()): + raise ValueError(f"attempts[{index}] with status {status!r} requires a non-empty reason") + if attempt_id in ids: + raise ValueError(f"duplicate attempt id {attempt_id!r}") + key = eval_name, model, task_name + if key in keys: + raise ValueError(f"duplicate result match key {key!r}; task_name must be unique per eval and model") + ids.add(attempt_id) + keys.add(key) + attempts.append( + PlannedAttempt( + attempt_id=attempt_id, + eval_name=eval_name, + model=model, + task_name=task_name, + status=status, # type: ignore[arg-type] + reason=reason.strip() if isinstance(reason, str) and reason.strip() else None, + ) + ) + return attempts + + +def _model_label(result: dict[str, Any]) -> str: + """Use the same label as evals.table, after any ``/`` prefix.""" + name = str((result.get("agent_info") or {}).get("name") or "") + return name.partition("/")[2] or name + + +def read_artifacts(results_root: Path) -> list[ResultArtifact]: + """Read every saved trial result using the same folder shape as evals.table.""" + if not results_root.is_dir(): + raise ValueError(f"results root is not a directory: {results_root}") + artifacts = [] + for result_file in sorted(results_root.glob("*/*/*/result.json")): + try: + result = json.loads(result_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read result artifact {result_file}: {exc}") from exc + if not isinstance(result, dict): + raise ValueError(f"result artifact must contain an object: {result_file}") + task_name = result.get("task_name") + artifacts.append( + ResultArtifact( + eval_name=result_file.relative_to(results_root).parts[0], + model=_model_label(result), + task_name=str(task_name) if isinstance(task_name, str) else "", + errored=result.get("exception_info") is not None, + path=result_file.relative_to(results_root).as_posix(), + ) + ) + return artifacts + + +def reconcile(attempts: list[PlannedAttempt], artifacts: list[ResultArtifact]) -> list[ReconciledAttempt]: + """Match planned attempts to their exact saved task results; retain every extra artifact as unplanned.""" + by_key: dict[AttemptKey, list[ResultArtifact]] = {} + for artifact in artifacts: + by_key.setdefault(artifact.key, []).append(artifact) + rows: list[ReconciledAttempt] = [] + for attempt in attempts: + matches = by_key.get(attempt.key, []) + artifact = matches.pop(0) if matches else None + if artifact is None: + status = "missing" if attempt.status == "planned" else attempt.status + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status=status, + reason=attempt.reason, + ) + ) + continue + if attempt.status == "unsupported": + status, reason = "unexpected-result", attempt.reason + else: + status = "error" if artifact.errored else "recorded" + reason = f"Manifest says interrupted: {attempt.reason}" if attempt.status == "interrupted" else None + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status=status, + reason=reason, + artifact=artifact.path, + ) + ) + for remaining in by_key.values(): + for artifact in remaining: + rows.append( + ReconciledAttempt( + attempt_id="", + eval_name=artifact.eval_name, + model=artifact.model, + task_name=artifact.task_name, + status="unplanned-result", + reason="Result has no matching entry in the attempt plan.", + artifact=artifact.path, + ) + ) + return rows + + +def markdown(rows: list[ReconciledAttempt]) -> str: + """Render the attempt denominator and artifact status without changing score statistics.""" + counts: dict[str, int] = {} + for row in rows: + counts[row.status] = counts.get(row.status, 0) + 1 + planned_total = sum(bool(row.attempt_id) for row in rows) + lines = [ + f"Planned attempts: {planned_total}; recorded outcomes: " + f"{counts.get('recorded', 0) + counts.get('error', 0) + counts.get('unexpected-result', 0)}; " + f"missing: {counts.get('missing', 0)}; interrupted: {counts.get('interrupted', 0)}; " + f"unsupported: {counts.get('unsupported', 0)}; errors: {counts.get('error', 0)}; " + f"unexpected results: {counts.get('unexpected-result', 0) + counts.get('unplanned-result', 0)}.", + "", + "| Attempt | Eval | Model | Task | Status | Artifact | Reason |", + "|---|---|---|---|---|---|---|", + ] + for row in rows: + cells = ( + row.attempt_id, + row.eval_name, + row.model, + row.task_name, + row.status, + row.artifact or "", + row.reason or "", + ) + lines.append("| " + " | ".join(_cell(value) for value in cells) + " |") + return "\n".join(lines) + + +def _cell(value: str) -> str: + return value.replace("|", "\\|").replace("\r", " ").replace("\n", " ") + + +def main() -> None: + """Print a manifest-to-artifact coverage report.""" + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results_root", type=Path, help="results root, e.g. evals/results") + parser.add_argument("attempt_plan", type=Path, help="version-1 JSON file declaring planned attempts") + args = parser.parse_args() + print(markdown(reconcile(load_plan(args.attempt_plan), read_artifacts(args.results_root)))) + + +if __name__ == "__main__": + main() diff --git a/evals/table.py b/evals/table.py index cc3b208..45f9372 100644 --- a/evals/table.py +++ b/evals/table.py @@ -153,7 +153,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "mean_chat_calls": round(statistics.mean(member.chat_calls for member in members), 1) if members else None, - "mean_cost_usd": round(statistics.mean(scored_costs), 4) + "mean_cost_usd": round(statistics.mean([cost for cost in scored_costs if cost is not None]), 4) if scored_costs and None not in scored_costs else None, "cost_n": len(costs), @@ -187,8 +187,18 @@ def markdown(table: list[dict[str, Any]]) -> str: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("root", type=Path, help="the results root, e.g. evals/results") + parser.add_argument( + "--attempt-plan", + type=Path, + help="optional version-1 JSON attempt manifest; append a coverage report without changing score rows", + ) args = parser.parse_args() print(markdown(rows(read_results(args.root)))) + if args.attempt_plan is not None: + from evals.reconcile import load_plan, markdown as reconciliation_markdown, read_artifacts, reconcile + + print("\n## Attempt coverage\n") + print(reconciliation_markdown(reconcile(load_plan(args.attempt_plan), read_artifacts(args.root)))) if __name__ == "__main__": diff --git a/tests/test_evals_reconcile.py b/tests/test_evals_reconcile.py new file mode 100644 index 0000000..778affb --- /dev/null +++ b/tests/test_evals_reconcile.py @@ -0,0 +1,160 @@ +# coding: utf-8 +"""Attempt-manifest coverage is additive to the existing score table.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from unittest import TestCase + +from evals.reconcile import load_plan, markdown, read_artifacts, reconcile +from evals.table import read_results, rows as score_rows + + +def _result(root: Path, eval_name: str, model: str, task_name: str, *, errored: bool = False) -> Path: + folder = root / eval_name / "job-1" / task_name.rsplit("/", 1)[-1] + folder.mkdir(parents=True) + result = { + "task_name": task_name, + "agent_info": {"name": f"s1a-evals/{model}"}, + "exception_info": {"error": "fixture error"} if errored else None, + "agent_result": {"metadata": {"steps": 2}, "cost_usd": 0.0}, + "verifier_result": {"rewards": {"reward": 0}}, + } + path = folder / "result.json" + path.write_text(json.dumps(result), encoding="utf-8") + return path + + +class TestReconcile(TestCase): + def _plan(self, path: Path) -> list: + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + { + "id": "done", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/0", + "status": "planned", + }, + { + "id": "failed", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/1", + "status": "planned", + }, + { + "id": "missing", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/2", + "status": "planned", + }, + { + "id": "stopped", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/3", + "status": "interrupted", + "reason": "worker stopped", + }, + { + "id": "unsupported", + "eval": "desktop", + "model": "cua", + "task_name": "desktop/0", + "status": "unsupported", + "reason": "no display", + }, + { + "id": "contradiction", + "eval": "desktop", + "model": "cua", + "task_name": "desktop/1", + "status": "unsupported", + "reason": "no display", + }, + ], + } + ), + encoding="utf-8", + ) + return load_plan(path) + + def test_reports_every_attempt_and_unplanned_or_contradictory_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0") + _result(root, "game2048", "jev", "game2048/1", errored=True) + _result(root, "desktop", "cua", "desktop/1") + _result(root, "blackjack", "jev", "blackjack/99") + attempts = self._plan(Path(tmp) / "plan.json") + result = reconcile(attempts, read_artifacts(root)) + + self.assertEqual( + [(row.attempt_id, row.status) for row in result], + [ + ("done", "recorded"), + ("failed", "error"), + ("missing", "missing"), + ("stopped", "interrupted"), + ("unsupported", "unsupported"), + ("contradiction", "unexpected-result"), + ("", "unplanned-result"), + ], + ) + report = markdown(result) + self.assertIn( + "Planned attempts: 6; recorded outcomes: 3; missing: 1; interrupted: 1; unsupported: 1; errors: 1; unexpected results: 2.", + report, + ) + self.assertIn("game2048/job-1/1/result.json", report) + + def test_reconciliation_does_not_mutate_existing_score_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0") + before = score_rows(read_results(root)) + attempts = self._plan(Path(tmp) / "plan.json") + reconcile(attempts, read_artifacts(root)) + after = score_rows(read_results(root)) + self.assertEqual(before, after) + self.assertEqual((before[0]["attempts"], before[0]["score_n"], before[0]["mean_score"]), (1, 1, 0.0)) + + def test_manifest_rejects_duplicate_match_keys_and_terminal_state_without_reason(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "plan.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + {"id": "a", "eval": "e", "model": "m", "task_name": "t", "status": "planned"}, + {"id": "b", "eval": "e", "model": "m", "task_name": "t", "status": "planned"}, + ], + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "duplicate result match key"): + load_plan(path) + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + {"id": "a", "eval": "e", "model": "m", "task_name": "t", "status": "interrupted"}, + ], + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "requires a non-empty reason"): + load_plan(path) From cafcb947dc6e22df2b70e284b08ea2a7ac93c616 Mon Sep 17 00:00:00 2001 From: chendanyang Date: Sat, 10 Oct 2026 07:46:20 +0800 Subject: [PATCH 2/2] Mark duplicate attempt artifacts ambiguous --- evals/README.md | 9 ++++++--- evals/reconcile.py | 26 ++++++++++++++++++++++---- tests/test_evals_reconcile.py | 32 ++++++++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/evals/README.md b/evals/README.md index 18fce1c..0f09a70 100644 --- a/evals/README.md +++ b/evals/README.md @@ -97,9 +97,12 @@ result, create a version-1 JSON manifest and pass it with `--attempt-plan`: `planned` with no artifact is reported as `missing`; explicitly declared `interrupted` and `unsupported` attempts remain separate. A saved exception result is an `error` and remains a recorded attempt. The report also flags results absent from the manifest and results that contradict an `unsupported` declaration. It matches using the -existing result fields `(eval folder, model label, task_name)`, so that tuple must be unique in a plan; use distinct -task names (normally including the seed) for repeated tasks. Missing artifacts alone do not prove that a run was -interrupted. This additive report does not modify the score, error, timing, cost, or attempt semantics of the table. +existing result fields `(eval folder, model label, task_name)`, so that tuple must be unique in a plan. If multiple +artifacts have the same tuple, the planned attempt is `ambiguous`; all candidate paths are shown and none is assigned +as its outcome. The current writer repeats `task_name` when the same seed is run again in another job, so use distinct +task names for repeats when those runs must be associated with separate planned entries. Missing artifacts alone do +not prove that a run was interrupted. This additive report does not modify the score, error, timing, cost, or attempt +semantics of the table. ```sh uv run python -m evals.table evals/results --attempt-plan evals/attempt-plan.json diff --git a/evals/reconcile.py b/evals/reconcile.py index 8cff4e2..11180ce 100644 --- a/evals/reconcile.py +++ b/evals/reconcile.py @@ -137,14 +137,31 @@ def read_artifacts(results_root: Path) -> list[ResultArtifact]: def reconcile(attempts: list[PlannedAttempt], artifacts: list[ResultArtifact]) -> list[ReconciledAttempt]: - """Match planned attempts to their exact saved task results; retain every extra artifact as unplanned.""" + """Match planned attempts only when their result key identifies exactly one artifact. + + Duplicate artifacts with the same key are ambiguous: retain all candidate paths on the planned row rather than + assigning one by path or input order. Unmatched artifacts remain visible as unplanned results. + """ by_key: dict[AttemptKey, list[ResultArtifact]] = {} for artifact in artifacts: by_key.setdefault(artifact.key, []).append(artifact) rows: list[ReconciledAttempt] = [] for attempt in attempts: - matches = by_key.get(attempt.key, []) - artifact = matches.pop(0) if matches else None + matches = sorted(by_key.pop(attempt.key, []), key=lambda item: item.path) + if len(matches) > 1: + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status="ambiguous", + reason=f"{len(matches)} result artifacts match this key; no result was assigned.", + artifact="; ".join(item.path for item in matches), + ) + ) + continue + artifact = matches[0] if matches else None if artifact is None: status = "missing" if attempt.status == "planned" else attempt.status rows.append( @@ -200,7 +217,8 @@ def markdown(rows: list[ReconciledAttempt]) -> str: f"Planned attempts: {planned_total}; recorded outcomes: " f"{counts.get('recorded', 0) + counts.get('error', 0) + counts.get('unexpected-result', 0)}; " f"missing: {counts.get('missing', 0)}; interrupted: {counts.get('interrupted', 0)}; " - f"unsupported: {counts.get('unsupported', 0)}; errors: {counts.get('error', 0)}; " + f"unsupported: {counts.get('unsupported', 0)}; ambiguous: {counts.get('ambiguous', 0)}; " + f"errors: {counts.get('error', 0)}; " f"unexpected results: {counts.get('unexpected-result', 0) + counts.get('unplanned-result', 0)}.", "", "| Attempt | Eval | Model | Task | Status | Artifact | Reason |", diff --git a/tests/test_evals_reconcile.py b/tests/test_evals_reconcile.py index 778affb..166e2f7 100644 --- a/tests/test_evals_reconcile.py +++ b/tests/test_evals_reconcile.py @@ -8,12 +8,14 @@ from pathlib import Path from unittest import TestCase -from evals.reconcile import load_plan, markdown, read_artifacts, reconcile +from evals.reconcile import PlannedAttempt, load_plan, markdown, read_artifacts, reconcile from evals.table import read_results, rows as score_rows -def _result(root: Path, eval_name: str, model: str, task_name: str, *, errored: bool = False) -> Path: - folder = root / eval_name / "job-1" / task_name.rsplit("/", 1)[-1] +def _result( + root: Path, eval_name: str, model: str, task_name: str, *, errored: bool = False, job_name: str = "job-1" +) -> Path: + folder = root / eval_name / job_name / task_name.rsplit("/", 1)[-1] folder.mkdir(parents=True) result = { "task_name": task_name, @@ -111,7 +113,7 @@ def test_reports_every_attempt_and_unplanned_or_contradictory_artifacts(self) -> ) report = markdown(result) self.assertIn( - "Planned attempts: 6; recorded outcomes: 3; missing: 1; interrupted: 1; unsupported: 1; errors: 1; unexpected results: 2.", + "Planned attempts: 6; recorded outcomes: 3; missing: 1; interrupted: 1; unsupported: 1; ambiguous: 0; errors: 1; unexpected results: 2.", report, ) self.assertIn("game2048/job-1/1/result.json", report) @@ -128,6 +130,28 @@ def test_reconciliation_does_not_mutate_existing_score_table(self) -> None: self.assertEqual(before, after) self.assertEqual((before[0]["attempts"], before[0]["score_n"], before[0]["mean_score"]), (1, 1, 0.0)) + def test_multiple_artifacts_for_one_key_are_order_invariant_and_ambiguous(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0", job_name="job-a") + _result(root, "game2048", "jev", "game2048/0", errored=True, job_name="job-b") + artifacts = read_artifacts(root) + attempt = PlannedAttempt("repeat", "game2048", "jev", "game2048/0", "planned") + forward = reconcile([attempt], artifacts) + reverse = reconcile([attempt], list(reversed(artifacts))) + + self.assertEqual(forward, reverse) + self.assertEqual(len(forward), 1) + self.assertEqual(forward[0].status, "ambiguous") + self.assertEqual(forward[0].reason, "2 result artifacts match this key; no result was assigned.") + self.assertEqual( + forward[0].artifact, + "game2048/job-a/0/result.json; game2048/job-b/0/result.json", + ) + self.assertIn("ambiguous: 1", markdown(forward)) + self.assertIn("recorded outcomes: 0", markdown(forward)) + def test_manifest_rejects_duplicate_match_keys_and_terminal_state_without_reason(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "plan.json"