diff --git a/evals/README.md b/evals/README.md index 69adce7..e283018 100644 --- a/evals/README.md +++ b/evals/README.md @@ -196,6 +196,36 @@ subset; `score (n)` states its count over all attempts, and `score unknown` coun reward. ALFWorld's game files sort by task type; `--stride 11` from offset 0 takes twelve games across the six types. Every model plays the same tile draws because 2048 seeds the page's `Math.random`. +### Planned-attempt coverage + +The table above can count only trials with a saved `result.json`. To include attempts that have not produced a +result, create a version-1 JSON manifest and pass it with `--attempt-plan`: + +```json +{ + "schema_version": 1, + "attempts": [ + {"id": "2048-jev-seed-0", "eval": "game2048", "model": "jev", "task_name": "game2048/0", "status": "planned"}, + {"id": "2048-jev-seed-1", "eval": "game2048", "model": "jev", "task_name": "game2048/1", "status": "interrupted", "reason": "worker exited before result.json was written"}, + {"id": "desktop-cua-1", "eval": "desktop", "model": "cua", "task_name": "desktop/1", "status": "unsupported", "reason": "display backend unavailable"} + ] +} +``` + +`planned` with no artifact is reported as `missing`; explicitly declared `interrupted` and `unsupported` attempts +remain separate. A saved exception result is an `error` and remains a recorded attempt. The report also flags +results absent from the manifest and results that contradict an `unsupported` declaration. It matches using the +existing result fields `(eval folder, model label, task_name)`, so that tuple must be unique in a plan. If multiple +artifacts have the same tuple, the planned attempt is `ambiguous`; all candidate paths are shown and none is assigned +as its outcome. The current writer repeats `task_name` when the same seed is run again in another job, so use distinct +task names for repeats when those runs must be associated with separate planned entries. Missing artifacts alone do +not prove that a run was interrupted. This additive report does not modify the score, error, timing, cost, or attempt +semantics of the table. + +```sh +uv run python -m evals.table evals/results --attempt-plan evals/attempt-plan.json +``` + ## Showcase runs and replays The matrix runs headless and records nothing visual. Every episode still writes ``views`` into ``agent/episode.json``: diff --git a/evals/reconcile.py b/evals/reconcile.py new file mode 100644 index 0000000..11180ce --- /dev/null +++ b/evals/reconcile.py @@ -0,0 +1,257 @@ +# coding: utf-8 +"""Reconcile planned evaluation attempts with saved trial result artifacts. + +Results are matched by the existing ``(eval folder, model label, task_name)`` record fields. The +reconciler reports denominator coverage and artifact mismatches; it does not compute or change scores. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +PlanStatus = Literal["planned", "interrupted", "unsupported"] +AttemptKey = tuple[str, str, str] + + +@dataclass(frozen=True) +class PlannedAttempt: + attempt_id: str + eval_name: str + model: str + task_name: str + status: PlanStatus + reason: str | None = None + + @property + def key(self) -> AttemptKey: + return self.eval_name, self.model, self.task_name + + +@dataclass(frozen=True) +class ResultArtifact: + eval_name: str + model: str + task_name: str + errored: bool + path: str + + @property + def key(self) -> AttemptKey: + return self.eval_name, self.model, self.task_name + + +@dataclass(frozen=True) +class ReconciledAttempt: + attempt_id: str + eval_name: str + model: str + task_name: str + status: str + reason: str | None = None + artifact: str | None = None + + +def load_plan(path: Path) -> list[PlannedAttempt]: + """Load and validate a version-1 attempt plan JSON file.""" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read attempt plan {path}: {exc}") from exc + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + raise ValueError("attempt plan must be an object with schema_version: 1") + raw_attempts = payload.get("attempts") + if not isinstance(raw_attempts, list) or not raw_attempts: + raise ValueError("attempt plan must contain a non-empty attempts list") + + attempts: list[PlannedAttempt] = [] + ids: set[str] = set() + keys: set[AttemptKey] = set() + for index, item in enumerate(raw_attempts): + if not isinstance(item, dict): + raise ValueError(f"attempts[{index}] must be an object") + text_fields = {name: item.get(name) for name in ("id", "eval", "model", "task_name", "status")} + if any(not isinstance(value, str) or not value.strip() for value in text_fields.values()): + raise ValueError(f"attempts[{index}] requires non-empty string id, eval, model, task_name and status") + attempt_id = str(item["id"]).strip() + eval_name = str(item["eval"]).strip() + model = str(item["model"]).strip() + task_name = str(item["task_name"]).strip() + status = str(item["status"]).strip() + if status not in {"planned", "interrupted", "unsupported"}: + raise ValueError(f"attempts[{index}] has unsupported status {status!r}") + reason = item.get("reason") + if status in {"interrupted", "unsupported"} and (not isinstance(reason, str) or not reason.strip()): + raise ValueError(f"attempts[{index}] with status {status!r} requires a non-empty reason") + if attempt_id in ids: + raise ValueError(f"duplicate attempt id {attempt_id!r}") + key = eval_name, model, task_name + if key in keys: + raise ValueError(f"duplicate result match key {key!r}; task_name must be unique per eval and model") + ids.add(attempt_id) + keys.add(key) + attempts.append( + PlannedAttempt( + attempt_id=attempt_id, + eval_name=eval_name, + model=model, + task_name=task_name, + status=status, # type: ignore[arg-type] + reason=reason.strip() if isinstance(reason, str) and reason.strip() else None, + ) + ) + return attempts + + +def _model_label(result: dict[str, Any]) -> str: + """Use the same label as evals.table, after any ``/`` prefix.""" + name = str((result.get("agent_info") or {}).get("name") or "") + return name.partition("/")[2] or name + + +def read_artifacts(results_root: Path) -> list[ResultArtifact]: + """Read every saved trial result using the same folder shape as evals.table.""" + if not results_root.is_dir(): + raise ValueError(f"results root is not a directory: {results_root}") + artifacts = [] + for result_file in sorted(results_root.glob("*/*/*/result.json")): + try: + result = json.loads(result_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"cannot read result artifact {result_file}: {exc}") from exc + if not isinstance(result, dict): + raise ValueError(f"result artifact must contain an object: {result_file}") + task_name = result.get("task_name") + artifacts.append( + ResultArtifact( + eval_name=result_file.relative_to(results_root).parts[0], + model=_model_label(result), + task_name=str(task_name) if isinstance(task_name, str) else "", + errored=result.get("exception_info") is not None, + path=result_file.relative_to(results_root).as_posix(), + ) + ) + return artifacts + + +def reconcile(attempts: list[PlannedAttempt], artifacts: list[ResultArtifact]) -> list[ReconciledAttempt]: + """Match planned attempts only when their result key identifies exactly one artifact. + + Duplicate artifacts with the same key are ambiguous: retain all candidate paths on the planned row rather than + assigning one by path or input order. Unmatched artifacts remain visible as unplanned results. + """ + by_key: dict[AttemptKey, list[ResultArtifact]] = {} + for artifact in artifacts: + by_key.setdefault(artifact.key, []).append(artifact) + rows: list[ReconciledAttempt] = [] + for attempt in attempts: + matches = sorted(by_key.pop(attempt.key, []), key=lambda item: item.path) + if len(matches) > 1: + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status="ambiguous", + reason=f"{len(matches)} result artifacts match this key; no result was assigned.", + artifact="; ".join(item.path for item in matches), + ) + ) + continue + artifact = matches[0] if matches else None + if artifact is None: + status = "missing" if attempt.status == "planned" else attempt.status + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status=status, + reason=attempt.reason, + ) + ) + continue + if attempt.status == "unsupported": + status, reason = "unexpected-result", attempt.reason + else: + status = "error" if artifact.errored else "recorded" + reason = f"Manifest says interrupted: {attempt.reason}" if attempt.status == "interrupted" else None + rows.append( + ReconciledAttempt( + attempt_id=attempt.attempt_id, + eval_name=attempt.eval_name, + model=attempt.model, + task_name=attempt.task_name, + status=status, + reason=reason, + artifact=artifact.path, + ) + ) + for remaining in by_key.values(): + for artifact in remaining: + rows.append( + ReconciledAttempt( + attempt_id="", + eval_name=artifact.eval_name, + model=artifact.model, + task_name=artifact.task_name, + status="unplanned-result", + reason="Result has no matching entry in the attempt plan.", + artifact=artifact.path, + ) + ) + return rows + + +def markdown(rows: list[ReconciledAttempt]) -> str: + """Render the attempt denominator and artifact status without changing score statistics.""" + counts: dict[str, int] = {} + for row in rows: + counts[row.status] = counts.get(row.status, 0) + 1 + planned_total = sum(bool(row.attempt_id) for row in rows) + lines = [ + f"Planned attempts: {planned_total}; recorded outcomes: " + f"{counts.get('recorded', 0) + counts.get('error', 0) + counts.get('unexpected-result', 0)}; " + f"missing: {counts.get('missing', 0)}; interrupted: {counts.get('interrupted', 0)}; " + f"unsupported: {counts.get('unsupported', 0)}; ambiguous: {counts.get('ambiguous', 0)}; " + f"errors: {counts.get('error', 0)}; " + f"unexpected results: {counts.get('unexpected-result', 0) + counts.get('unplanned-result', 0)}.", + "", + "| Attempt | Eval | Model | Task | Status | Artifact | Reason |", + "|---|---|---|---|---|---|---|", + ] + for row in rows: + cells = ( + row.attempt_id, + row.eval_name, + row.model, + row.task_name, + row.status, + row.artifact or "", + row.reason or "", + ) + lines.append("| " + " | ".join(_cell(value) for value in cells) + " |") + return "\n".join(lines) + + +def _cell(value: str) -> str: + return value.replace("|", "\\|").replace("\r", " ").replace("\n", " ") + + +def main() -> None: + """Print a manifest-to-artifact coverage report.""" + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results_root", type=Path, help="results root, e.g. evals/results") + parser.add_argument("attempt_plan", type=Path, help="version-1 JSON file declaring planned attempts") + args = parser.parse_args() + print(markdown(reconcile(load_plan(args.attempt_plan), read_artifacts(args.results_root)))) + + +if __name__ == "__main__": + main() diff --git a/evals/table.py b/evals/table.py index cc3b208..45f9372 100644 --- a/evals/table.py +++ b/evals/table.py @@ -153,7 +153,7 @@ def rows(trials: list[Trial]) -> list[dict[str, Any]]: "mean_chat_calls": round(statistics.mean(member.chat_calls for member in members), 1) if members else None, - "mean_cost_usd": round(statistics.mean(scored_costs), 4) + "mean_cost_usd": round(statistics.mean([cost for cost in scored_costs if cost is not None]), 4) if scored_costs and None not in scored_costs else None, "cost_n": len(costs), @@ -187,8 +187,18 @@ def markdown(table: list[dict[str, Any]]) -> str: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("root", type=Path, help="the results root, e.g. evals/results") + parser.add_argument( + "--attempt-plan", + type=Path, + help="optional version-1 JSON attempt manifest; append a coverage report without changing score rows", + ) args = parser.parse_args() print(markdown(rows(read_results(args.root)))) + if args.attempt_plan is not None: + from evals.reconcile import load_plan, markdown as reconciliation_markdown, read_artifacts, reconcile + + print("\n## Attempt coverage\n") + print(reconciliation_markdown(reconcile(load_plan(args.attempt_plan), read_artifacts(args.root)))) if __name__ == "__main__": diff --git a/tests/test_evals_reconcile.py b/tests/test_evals_reconcile.py new file mode 100644 index 0000000..166e2f7 --- /dev/null +++ b/tests/test_evals_reconcile.py @@ -0,0 +1,184 @@ +# coding: utf-8 +"""Attempt-manifest coverage is additive to the existing score table.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from unittest import TestCase + +from evals.reconcile import PlannedAttempt, load_plan, markdown, read_artifacts, reconcile +from evals.table import read_results, rows as score_rows + + +def _result( + root: Path, eval_name: str, model: str, task_name: str, *, errored: bool = False, job_name: str = "job-1" +) -> Path: + folder = root / eval_name / job_name / task_name.rsplit("/", 1)[-1] + folder.mkdir(parents=True) + result = { + "task_name": task_name, + "agent_info": {"name": f"s1a-evals/{model}"}, + "exception_info": {"error": "fixture error"} if errored else None, + "agent_result": {"metadata": {"steps": 2}, "cost_usd": 0.0}, + "verifier_result": {"rewards": {"reward": 0}}, + } + path = folder / "result.json" + path.write_text(json.dumps(result), encoding="utf-8") + return path + + +class TestReconcile(TestCase): + def _plan(self, path: Path) -> list: + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + { + "id": "done", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/0", + "status": "planned", + }, + { + "id": "failed", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/1", + "status": "planned", + }, + { + "id": "missing", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/2", + "status": "planned", + }, + { + "id": "stopped", + "eval": "game2048", + "model": "jev", + "task_name": "game2048/3", + "status": "interrupted", + "reason": "worker stopped", + }, + { + "id": "unsupported", + "eval": "desktop", + "model": "cua", + "task_name": "desktop/0", + "status": "unsupported", + "reason": "no display", + }, + { + "id": "contradiction", + "eval": "desktop", + "model": "cua", + "task_name": "desktop/1", + "status": "unsupported", + "reason": "no display", + }, + ], + } + ), + encoding="utf-8", + ) + return load_plan(path) + + def test_reports_every_attempt_and_unplanned_or_contradictory_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0") + _result(root, "game2048", "jev", "game2048/1", errored=True) + _result(root, "desktop", "cua", "desktop/1") + _result(root, "blackjack", "jev", "blackjack/99") + attempts = self._plan(Path(tmp) / "plan.json") + result = reconcile(attempts, read_artifacts(root)) + + self.assertEqual( + [(row.attempt_id, row.status) for row in result], + [ + ("done", "recorded"), + ("failed", "error"), + ("missing", "missing"), + ("stopped", "interrupted"), + ("unsupported", "unsupported"), + ("contradiction", "unexpected-result"), + ("", "unplanned-result"), + ], + ) + report = markdown(result) + self.assertIn( + "Planned attempts: 6; recorded outcomes: 3; missing: 1; interrupted: 1; unsupported: 1; ambiguous: 0; errors: 1; unexpected results: 2.", + report, + ) + self.assertIn("game2048/job-1/1/result.json", report) + + def test_reconciliation_does_not_mutate_existing_score_table(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0") + before = score_rows(read_results(root)) + attempts = self._plan(Path(tmp) / "plan.json") + reconcile(attempts, read_artifacts(root)) + after = score_rows(read_results(root)) + self.assertEqual(before, after) + self.assertEqual((before[0]["attempts"], before[0]["score_n"], before[0]["mean_score"]), (1, 1, 0.0)) + + def test_multiple_artifacts_for_one_key_are_order_invariant_and_ambiguous(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "results" + root.mkdir() + _result(root, "game2048", "jev", "game2048/0", job_name="job-a") + _result(root, "game2048", "jev", "game2048/0", errored=True, job_name="job-b") + artifacts = read_artifacts(root) + attempt = PlannedAttempt("repeat", "game2048", "jev", "game2048/0", "planned") + forward = reconcile([attempt], artifacts) + reverse = reconcile([attempt], list(reversed(artifacts))) + + self.assertEqual(forward, reverse) + self.assertEqual(len(forward), 1) + self.assertEqual(forward[0].status, "ambiguous") + self.assertEqual(forward[0].reason, "2 result artifacts match this key; no result was assigned.") + self.assertEqual( + forward[0].artifact, + "game2048/job-a/0/result.json; game2048/job-b/0/result.json", + ) + self.assertIn("ambiguous: 1", markdown(forward)) + self.assertIn("recorded outcomes: 0", markdown(forward)) + + def test_manifest_rejects_duplicate_match_keys_and_terminal_state_without_reason(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "plan.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + {"id": "a", "eval": "e", "model": "m", "task_name": "t", "status": "planned"}, + {"id": "b", "eval": "e", "model": "m", "task_name": "t", "status": "planned"}, + ], + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "duplicate result match key"): + load_plan(path) + path.write_text( + json.dumps( + { + "schema_version": 1, + "attempts": [ + {"id": "a", "eval": "e", "model": "m", "task_name": "t", "status": "interrupted"}, + ], + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "requires a non-empty reason"): + load_plan(path)