diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e7ed24..a9963dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ defaults: jobs: lint: - name: actionlint, shellcheck and the report's syntax + name: actionlint, shellcheck and the report's tests runs-on: ubuntu-24.04 timeout-minutes: 10 env: @@ -41,8 +41,8 @@ jobs: run: '"$RUNNER_TEMP/actionlint" -color' - name: Lint the scripts run: shellcheck scripts/*.sh - - name: Compile the report - run: python3 -m py_compile scripts/report.py + - name: Test the report + run: python3 -m unittest discover --start-directory tests --pattern 'test_*.py' --verbose install: name: Install ${{ matrix.version }} on ${{ matrix.os }} @@ -99,10 +99,13 @@ jobs: MEAN_REWARD: ${{ steps.hbench.outputs.mean-reward }} N_TRIALS: ${{ steps.hbench.outputs.n-trials }} N_ERRORS: ${{ steps.hbench.outputs.n-errors }} + N_SOLVED: ${{ steps.hbench.outputs.n-solved }} + PASS_RATE: ${{ steps.hbench.outputs.pass-rate }} RESULT_PATH: ${{ steps.hbench.outputs.result-path }} ARTIFACT_ID: ${{ steps.hbench.outputs.artifact-id }} run: | [[ "$MEAN_REWARD" == 1.0 && "$N_TRIALS" == 1 && "$N_ERRORS" == 0 ]] + [[ "$N_SOLVED" == 1 && "$PASS_RATE" == 1.0 ]] [[ -f "$RESULT_PATH" && -n "$ARTIFACT_ID" ]] run-below-threshold: @@ -128,8 +131,9 @@ jobs: OUTCOME: ${{ steps.hbench.outcome }} MEAN_REWARD: ${{ steps.hbench.outputs.mean-reward }} N_ERRORS: ${{ steps.hbench.outputs.n-errors }} + N_SOLVED: ${{ steps.hbench.outputs.n-solved }} run: | - [[ "$OUTCOME" == failure && "$MEAN_REWARD" == 0.0 && "$N_ERRORS" == 0 ]] + [[ "$OUTCOME" == failure && "$MEAN_REWARD" == 0.0 && "$N_ERRORS" == 0 && "$N_SOLVED" == 0 ]] run-oci-dataset: name: Run the oracle on an OCI dataset, locked by digest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/README.md b/README.md index 6830a52..5752f46 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,13 @@ For hbench itself, the docs are at [nodeq.tech/docs](https://nodeq.tech/docs): ``` With none of `path`, `dataset` or `config`, the action only installs hbench, and later steps can call -`hbench` themselves. Useful outputs: `mean-reward`, `n-trials`, `n-errors` and `job-dir`. +`hbench` themselves. Useful outputs: `pass-rate`, `n-solved`, `mean-reward`, `cost-usd` and `job-dir`. + +The job summary shows, for each agent and model, how many tasks it solved, its mean reward and pass@k, its +errors, the time it spent, its tokens and its cost per solved task; the trials by outcome and each eval's +mean reward, each with a bar; the command that opens the job in +hbench's interactive dashboard on your machine; what ran (the hbench release, the agent, the model, the tasks +and their digest); the errors by type; and a row per task. A `registry.json` lists datasets and where each of their tasks is: a git repository at a commit, or a folder in your repository, read from `working-directory`: diff --git a/action.yml b/action.yml index 01b0a3b..34ea22f 100644 --- a/action.yml +++ b/action.yml @@ -96,6 +96,15 @@ outputs: n-errors: description: Trials that ended in an error. value: ${{ steps.run.outputs.n-errors }} + n-solved: + description: Trials solved, with a reward of 1. + value: ${{ steps.run.outputs.n-solved }} + pass-rate: + description: Solved trials over all trials, from 0 to 1. + value: ${{ steps.run.outputs.pass-rate }} + cost-usd: + description: The agents' cost in US dollars, where hbench knows it. Empty for agents that report none, such as the oracle. + value: ${{ steps.run.outputs.cost-usd }} artifact-id: description: ID of the uploaded artifact. value: ${{ steps.upload.outputs.artifact-id }} @@ -134,6 +143,7 @@ runs: INPUT_JOBS_DIR: ${{ inputs.jobs-dir }} INPUT_MIN_MEAN_REWARD: ${{ inputs.min-mean-reward }} INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }} + INPUT_UPLOAD_ARTIFACT: ${{ inputs.upload-artifact }} run: '"$GITHUB_ACTION_PATH/scripts/run.sh"' - name: Upload the job directory diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index dfbb7da..3ef72aa 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -47,10 +47,30 @@ The [README](../README.md) has the short version. | `mean-reward` | Mean reward over every trial of every eval. | | `n-trials` | Trials run. | | `n-errors` | Trials that ended in an error. | +| `n-solved` | Trials solved, with a reward of 1. | +| `pass-rate` | Solved trials over all trials, from 0 to 1. | +| `cost-usd` | The agents' cost in US dollars, where hbench knows it. Empty for agents that report none, such as the oracle. | | `artifact-id` | ID of the uploaded artifact. | -`mean-reward` weighs each eval's mean reward by its trials. The job summary has a row per eval and lists the -trials below a reward of 1. +A trial is solved when its reward is 1, so `pass-rate` is the share of trials that got full marks. +`mean-reward` weighs each eval's mean reward by its trials; it differs from `pass-rate` when tasks give +partial rewards. + +The job summary has: + +- a row per eval (agent, model and tasks): solved trials and the pass rate, mean reward, pass@k, errors, + the agent's time in total and per trial (median), tokens in and out, and the cost in total and per + solved trial; +- whether the mean reward reached `min-mean-reward`; +- two tables with bars of up to 20 blocks: the trials solved, failed and ended in an error, with each + share, and each eval's mean reward; +- when the job is uploaded, the command that downloads it and opens it in hbench's interactive dashboard, + `gh run download -R -n -D hbench-jobs/ && hbench view hbench-jobs`; +- what ran: the hbench release and results epoch, the agent and model, the tasks with their registry and + manifest digest, attempts and concurrency, and when the job ran; +- the tasks' own metrics, the other keys of their `reward.json`, as means; +- the errors by type, with the trials they hit; +- a row per trial, errors and failures first: result, reward, agent time, tokens and cost. ## More examples @@ -162,15 +182,16 @@ Pass any other `hbench run` flag through `args`, one argument per line: action.yml the inputs, outputs and three steps: install, run, upload scripts/install.sh download, checksum, provenance, tool cache scripts/run.sh build the hbench run command line, run it, report -scripts/report.py result.json to outputs, job summary and the threshold +scripts/report.py the job's results to outputs, job summary and the threshold +tests/test_report.py the report, on the two job directories in tests/report/ tests/tasks/ a one-task folder the tests run ``` -`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint and shellcheck; installing -`latest` and a pinned release on Linux x86_64 and arm64 and macOS arm64 and x86_64, with provenance checked; -the oracle on the local task with `min-mean-reward: '1'`; the `nop` agent, which must fail the step on its -score; two tasks of an OCI dataset with `--lock-write`; and a dataset listed in a `registry.json`, by path and -by URL. +`.github/workflows/test.yml` runs on GitHub-hosted runners only: actionlint, shellcheck and the report's unit +tests; installing `latest` and a pinned release on Linux x86_64 and arm64 and macOS arm64 and x86_64, with +provenance checked; the oracle on the local task with `min-mean-reward: '1'`; the `nop` agent, which must fail +the step on its score; two tasks of an OCI dataset with `--lock-write`; and a dataset listed in a +`registry.json`, by path and by URL. To try the scripts outside GitHub, set the variables the runner sets (`RUNNER_TEMP`, `GITHUB_ACTION_PATH`, `GITHUB_OUTPUT`, `GITHUB_PATH`, `GITHUB_STEP_SUMMARY`) and the `INPUT_*` ones `action.yml` passes, then run diff --git a/scripts/report.py b/scripts/report.py index 844b932..17a18e6 100644 --- a/scripts/report.py +++ b/scripts/report.py @@ -1,43 +1,140 @@ -"""Write a job's scores to the step's outputs and summary, and check them against min-mean-reward. +"""Write a job's results to the step's outputs and summary, and check them against min-mean-reward. Usage: report.py RESULT_JSON -Reads hbench's result.json. Each eval (agent, model and dataset) gets a row with its trials, errors, mean -reward and pass@k; the trials that did not reach a reward of 1 are listed below. An eval's mean reward is -its first metric's `reward` (a task's reward.json) or `mean` (a task's reward.txt). mean-reward is the mean -over every trial of every eval. Exits 1 when INPUT_MIN_MEAN_REWARD is set and mean-reward is below it. +Reads hbench's result.json, and beside it the job's config.json and lock.json and each trial's result.json. +A trial is solved when its reward is 1. Each eval (agent, model and tasks) gets a row: solved trials and the +pass rate, mean reward, pass@k, errors, agent time, tokens and cost. Two tables follow with bars of up to +20 blocks: the solved, failed and errored trials with each share, and each eval's mean reward. Below come what ran (hbench, agent, +model, tasks and digest), the command that opens the job in hbench's dashboard when the job is uploaded +(HBENCH_ARTIFACT_NAME), the tasks' own metrics, the errors by type, and a row per trial, errors and +failures first. mean-reward is the trial-weighted mean of the evals' mean rewards, as hbench computes them +(a task's reward.json `reward`, or its reward.txt). Exits 1 when INPUT_MIN_MEAN_REWARD is set and +mean-reward is below it. """ +import datetime import json import os +import pathlib +import re +import statistics import sys -LISTED_TRIALS = 50 +LISTED_TRIALS = 500 +PRIMARY_METRICS = ("reward", "mean") +SHARE_BAR_WIDTH = 20 def numeric(value): + if isinstance(value, bool): + return None try: return float(value) except (TypeError, ValueError): return None -def eval_rows(stats): - rows = [] - for name, entry in sorted((stats.get("evals") or {}).items()): - metrics = entry.get("metrics") or [{}] - rewards = (entry.get("reward_stats") or {}).get("reward") or {} - below_one = sorted( - trial for value, trials in rewards.items() if (numeric(value) or 0.0) < 1.0 for trial in trials +def timestamp(value): + if not isinstance(value, str): + return None + trimmed = re.sub(r"(\.\d{6})\d+", r"\1", value).replace("Z", "+00:00") + try: + return datetime.datetime.fromisoformat(trimmed) + except ValueError: + return None + + +def seconds_between(span): + if not isinstance(span, dict): + return None + started, finished = timestamp(span.get("started_at")), timestamp(span.get("finished_at")) + return (finished - started).total_seconds() if started and finished else None + + +def read_json(path): + try: + with open(path, encoding="utf-8") as stream: + return json.load(stream) + except (OSError, ValueError): + return {} + + +def eval_key(trial): + agent = (trial.get("agent_info") or {}).get("name") or "agent" + model = ((trial.get("agent_info") or {}).get("model_info") or {}).get("name") + source = trial.get("source") or "tasks" + return "__".join(part for part in (agent, model, source) if part) + + +def load_trials(job_directory): + trials = [] + for result_path in sorted(pathlib.Path(job_directory).glob("*/result.json")): + result = read_json(result_path) + if "trial_name" not in result: + continue + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + usage = result.get("agent_result") or {} + exception = result.get("exception_info") or {} + trials.append( + { + "eval": eval_key(result), + "task": result.get("task_name") or result["trial_name"], + "trial": result["trial_name"], + "reward": numeric(rewards.get("reward")), + "error": exception.get("exception_type"), + "agent_seconds": seconds_between(result.get("agent_execution")), + "input_tokens": usage.get("n_input_tokens"), + "cache_tokens": usage.get("n_cache_tokens"), + "output_tokens": usage.get("n_output_tokens"), + "cost": numeric(usage.get("cost_usd")), + } ) + return trials + + +def solved(trial): + return trial["error"] is None and trial["reward"] is not None and trial["reward"] >= 1.0 + + +def total(values): + present = [value for value in values if isinstance(value, (int, float)) and not isinstance(value, bool)] + return sum(present) if present else None + + +def eval_rows(stats, trials): + evals = stats.get("evals") or {} + rows = [] + for name, entry in sorted(evals.items()): + metrics = (entry.get("metrics") or [{}])[0] + mine = [trial for trial in trials if trial["eval"] == name] + if not mine and len(evals) == 1: + mine = trials + agent_times = [trial["agent_seconds"] for trial in mine if trial["agent_seconds"] is not None] + n_solved = sum(1 for trial in mine if solved(trial)) + cost = total(trial["cost"] for trial in mine) rows.append( { "name": name, - "trials": entry.get("n_trials") or 0, + "trials": entry.get("n_trials") or len(mine), "errors": entry.get("n_errors") or 0, - "mean": numeric(metrics[0].get("reward", metrics[0].get("mean"))), - "pass_at_k": entry.get("pass_at_k") or {}, - "below_one": below_one, + "mean": numeric(metrics.get("reward", metrics.get("mean"))), + "other_metrics": { + key: numeric(value) + for key, value in sorted(metrics.items()) + if key not in PRIMARY_METRICS and numeric(value) is not None + }, + "pass_at_k": {k: numeric(v) for k, v in sorted((entry.get("pass_at_k") or {}).items())}, + "exceptions": {kind: sorted(names) for kind, names in sorted((entry.get("exception_stats") or {}).items())}, + "solved": n_solved, + "graded": len(mine), + "agent_total": sum(agent_times) if agent_times else None, + "agent_median": statistics.median(agent_times) if agent_times else None, + "input_tokens": total(trial["input_tokens"] for trial in mine), + "cache_tokens": total(trial["cache_tokens"] for trial in mine), + "output_tokens": total(trial["output_tokens"] for trial in mine), + "cost": cost, + "cost_per_solved": cost / n_solved if cost is not None and n_solved else None, } ) return rows @@ -49,48 +146,204 @@ def overall_mean(rows): return sum(mean * count for mean, count in weighted) / trials if trials else None -def text(value, digits=3): +def score(value, digits=3): return "–" if value is None else f"{value:.{digits}f}" -def summary(result, rows, mean, threshold): - stats = result.get("stats") or {} +def duration(seconds): + if seconds is None: + return "–" + seconds = round(seconds) + hours, rest = divmod(seconds, 3600) + minutes, seconds = divmod(rest, 60) + if hours: + return f"{hours}h {minutes:02d}m" + if minutes: + return f"{minutes}m {seconds:02d}s" + return f"{seconds}s" + + +def tokens(count): + if count is None: + return "–" + for limit, suffix in ((1_000_000_000, "B"), (1_000_000, "M"), (1_000, "K")): + if count >= limit: + return f"{count / limit:.1f}{suffix}" + return str(count) + + +def dollars(amount): + if amount is None: + return "–" + return f"${amount:.2f}" if amount >= 1 else f"${amount:.4f}" + + +def percent(part, whole): + return f"{100 * part / whole:.1f}%" if whole else "–" + + +def scorecard(rows): lines = [ - "## hbench", - "", - "| Eval | Trials | Errors | Mean reward | pass@k |", - "|---|---:|---:|---:|---|", + "| Eval | Solved | Mean reward | pass@k | Errors | Agent time | Tokens in / out | Cost |", + "|---|---:|---:|---|---:|---|---|---|", ] for row in rows: - pass_at_k = ", ".join(f"pass@{k} {text(numeric(v))}" for k, v in sorted(row["pass_at_k"].items())) or "–" - lines.append(f"| `{row['name']}` | {row['trials']} | {row['errors']} | {text(row['mean'])} | {pass_at_k} |") - lines.append("") - verdict = f"Mean reward **{text(mean)}** over {sum(row['trials'] for row in rows)} trials" - if threshold is not None: - verdict += f", {'at or above' if mean is not None and mean >= threshold else 'below'} min-mean-reward {threshold:g}" - lines.append(verdict + ".") - tokens = [ - f"{label} {stats[key]:,}" - for key, label in (("n_input_tokens", "input"), ("n_cache_tokens", "cached"), ("n_output_tokens", "output")) - if isinstance(stats.get(key), int) + pass_at_k = ", ".join(f"@{k} {score(v)}" for k, v in row["pass_at_k"].items()) or "–" + agent_time = "–" + if row["agent_total"] is not None: + agent_time = f"{duration(row['agent_total'])} total, {duration(row['agent_median'])} median" + token_text = "–" + if row["input_tokens"] is not None or row["output_tokens"] is not None: + token_text = f"{tokens(row['input_tokens'])} / {tokens(row['output_tokens'])}" + if row["cache_tokens"]: + token_text += f" ({tokens(row['cache_tokens'])} cached)" + cost_text = dollars(row["cost"]) + if row["cost_per_solved"] is not None: + cost_text += f" ({dollars(row['cost_per_solved'])} per solved)" + lines.append( + f"| `{row['name']}` | **{row['solved']} / {row['trials']}** ({percent(row['solved'], row['trials'])}) " + f"| {score(row['mean'])} | {pass_at_k} | {row['errors']} | {agent_time} | {token_text} | {cost_text} |" + ) + return lines + + +def shown_path(path): + absolute, here = os.path.abspath(path), os.getcwd() + return os.path.relpath(absolute, here) if os.path.commonpath([absolute, here]) == here else path + + +def run_details(result, config, lock): + hbench = lock.get("hbench") or {} + lines = [] + if hbench.get("version"): + epoch = f", results epoch {hbench['results_epoch']}" if hbench.get("results_epoch") else "" + lines.append(f"- **hbench** {hbench['version']}{epoch}") + for agent in config.get("agents") or []: + model = f", model `{agent['model_name']}`" if agent.get("model_name") else "" + lines.append(f"- **Agent** `{agent.get('name') or 'oracle'}`{model}") + for dataset in config.get("datasets") or []: + if dataset.get("name"): + where = dataset.get("registry") or dataset.get("registry_url") or dataset.get("registry_path") + label = f"`{dataset['name']}@{dataset['version']}`" if dataset.get("version") else f"`{dataset['name']}`" + lines.append( + f"- **Tasks** {label}" + + (f" from `{where}`" if where else "") + + (f", manifest `{dataset['digest']}`" if dataset.get("digest") else "") + ) + elif dataset.get("path"): + lines.append(f"- **Tasks** `{shown_path(dataset['path'])}`") + attempts = config.get("n_attempts") or 1 + concurrent = lock.get("n_concurrent_trials") or config.get("n_concurrent_trials") + lines.append(f"- **Attempts** {attempts} per task" + (f", {concurrent} at a time" if concurrent else "")) + started, finished = timestamp(result.get("started_at")), timestamp(result.get("finished_at")) + if started and finished: + lines.append( + f"- **Ran** {started:%Y-%m-%d %H:%M} to {finished:%H:%M} UTC, " + f"{duration((finished - started).total_seconds())}" + ) + return lines + + +def trial_table(trials): + def order(trial): + return (0 if trial["error"] else 1 if not solved(trial) else 2, trial["task"], trial["trial"]) + + ordered = sorted(trials, key=order) + shown = ordered[:LISTED_TRIALS] + lines = [ + "| Task | Result | Reward | Agent time | Tokens in / out | Cost |", + "|---|---|---:|---|---|---|", + ] + for trial in shown: + result = f"⚠️ {trial['error']}" if trial["error"] else "✅ solved" if solved(trial) else "❌ failed" + token_text = "–" + if trial["input_tokens"] is not None or trial["output_tokens"] is not None: + token_text = f"{tokens(trial['input_tokens'])} / {tokens(trial['output_tokens'])}" + lines.append( + f"| `{trial['task']}` | {result} | {score(trial['reward'])} | {duration(trial['agent_seconds'])} " + f"| {token_text} | {dollars(trial['cost'])} |" + ) + if len(ordered) > len(shown): + lines.append(f"| and {len(ordered) - len(shown)} more | | | | | |") + return lines + + +def chart_label(name): + return re.sub(r'["\[\]]', "", name.replace("__", " / ")) + + +def charts(rows, trials): + n_errors = sum(1 for trial in trials if trial["error"]) if trials else sum(row["errors"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + n_failed = (len(trials) if trials else sum(row["trials"] for row in rows)) - n_solved - n_errors + n_all = n_solved + n_failed + n_errors + lines = [] + if n_all: + lines += ["| Outcome | Trials | Share |", "|---|---:|---|"] + for label, count in (("Solved", n_solved), ("Failed", n_failed), ("Error", n_errors)): + cells = max(1, round(SHARE_BAR_WIDTH * count / n_all)) if count else 0 + lines.append(f"| {label} | {count} | {'█' * cells} {percent(count, n_all)} |") + lines.append("") + scored = [row for row in rows if row["mean"] is not None] + if scored: + lines += ["| Eval | Mean reward |", "|---|---|"] + for row in scored: + share = min(max(row["mean"], 0.0), 1.0) + cells = max(1, round(SHARE_BAR_WIDTH * share)) if share > 0 else 0 + lines.append(f"| `{chart_label(row['name'])}` | {'█' * cells} {score(row['mean'])} |") + lines.append("") + return lines + + +def dashboard_command(job_name): + artifact = os.environ.get("HBENCH_ARTIFACT_NAME") + repository, run_id = os.environ.get("GITHUB_REPOSITORY"), os.environ.get("GITHUB_RUN_ID") + if not (artifact and repository and run_id): + return [] + return [ + "View interactive dashboard by hbench:", + "", + "```bash", + f"gh run download {run_id} -R {repository} -n {artifact} -D hbench-jobs/{job_name} && hbench view hbench-jobs", + "```", ] - if tokens: - lines.append(f"Tokens: {', '.join(tokens)}.") - if isinstance(stats.get("cost_usd"), (int, float)): - lines.append(f"Cost: ${stats['cost_usd']:.4f}.") + + +def summary(result, config, lock, rows, trials, mean, threshold, job_name): + n_trials = sum(row["trials"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + lines = ["## Harness Benchmark", "", *scorecard(rows), ""] + verdict = ( + f"**{n_solved} of {n_trials}** trials solved ({percent(n_solved, n_trials)}), mean reward **{score(mean)}**." + ) + if threshold is not None: + passed = mean is not None and mean >= threshold + verdict += f" {'✅ At or above' if passed else '❌ Below'} min-mean-reward {threshold:g}." + lines += [verdict, "", *charts(rows, trials)] + command = dashboard_command(job_name) + if command: + lines += [*command, ""] + lines += ["### What ran", "", *run_details(result, config, lock)] for row in rows: - if row["below_one"]: - shown = row["below_one"][:LISTED_TRIALS] - more = len(row["below_one"]) - len(shown) - lines += [ - "", - f"
{len(row['below_one'])} trials of {row['name']} below a reward of 1", - "", - *[f"- `{trial}`" for trial in shown], - *([f"- and {more} more"] if more > 0 else []), - "", - "
", - ] + if row["other_metrics"]: + metrics = ", ".join(f"`{key}` {score(value)}" for key, value in row["other_metrics"].items()) + lines += ["", f"The tasks' own metrics for `{row['name']}`, as means over its trials: {metrics}."] + errors = [(row["name"], kind, names) for row in rows for kind, names in row["exceptions"].items()] + if errors: + lines += ["", "### Errors", ""] + for name, kind, names in errors: + shown = ", ".join(f"`{trial}`" for trial in names[:10]) + more = f" and {len(names) - 10} more" if len(names) > 10 else "" + lines.append(f"- **{kind}** in `{name}`: {len(names)} ({shown}{more})") + if trials: + lines += [ + "", + f"
{len(trials)} trials, errors and failures first", + "", + *trial_table(trials), + "", + "
", + ] return "\n".join(lines) + "\n" @@ -105,23 +358,32 @@ def main(arguments): if len(arguments) != 1: print(__doc__, file=sys.stderr) return 2 - with open(arguments[0], encoding="utf-8") as stream: - result = json.load(stream) - rows = eval_rows(result.get("stats") or {}) + result_path = pathlib.Path(arguments[0]) + result = read_json(result_path) + job_directory = result_path.parent + config, lock = read_json(job_directory / "config.json"), read_json(job_directory / "lock.json") + trials = load_trials(job_directory) + rows = eval_rows(result.get("stats") or {}, trials) mean = overall_mean(rows) threshold = numeric(os.environ.get("INPUT_MIN_MEAN_REWARD") or None) + n_trials = sum(row["trials"] for row in rows) + n_solved = sum(row["solved"] for row in rows) + cost = total(row["cost"] for row in rows) append( "GITHUB_OUTPUT", - f"result-path={arguments[0]}\n" + f"result-path={result_path}\n" f"mean-reward={'' if mean is None else round(mean, 6)}\n" - f"n-trials={sum(row['trials'] for row in rows)}\n" - f"n-errors={sum(row['errors'] for row in rows)}\n", + f"n-trials={n_trials}\n" + f"n-errors={sum(row['errors'] for row in rows)}\n" + f"n-solved={n_solved}\n" + f"pass-rate={round(n_solved / n_trials, 6) if n_trials else ''}\n" + f"cost-usd={'' if cost is None else round(cost, 6)}\n", ) - report = summary(result, rows, mean, threshold) + report = summary(result, config, lock, rows, trials, mean, threshold, job_directory.name) append("GITHUB_STEP_SUMMARY", report) print(report) if threshold is not None and (mean is None or mean < threshold): - print(f"::error title=hbench::Mean reward {text(mean)} is below min-mean-reward {threshold:g}") + print(f"::error title=hbench::Mean reward {score(mean)} is below min-mean-reward {threshold:g}") return 1 return 0 diff --git a/scripts/run.sh b/scripts/run.sh index b04fdff..ecc57cc 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -65,7 +65,8 @@ arguments+=(--jobs-dir "$jobs_directory" --job-name "$job_name") add_lines "" "$INPUT_ARGS" echo "job-dir=$job_directory" >> "$GITHUB_OUTPUT" -echo "artifact-name=${INPUT_ARTIFACT_NAME:-hbench-$job_name}" >> "$GITHUB_OUTPUT" +artifact_name="${INPUT_ARTIFACT_NAME:-hbench-$job_name}" +echo "artifact-name=$artifact_name" >> "$GITHUB_OUTPUT" printf 'hbench' printf ' %q' "${arguments[@]}" @@ -78,6 +79,9 @@ if [[ ! -f "$job_directory/result.json" ]]; then fi report_status=0 +if [[ "${INPUT_UPLOAD_ARTIFACT:-true}" == true ]]; then + export HBENCH_ARTIFACT_NAME="$artifact_name" +fi python3 "$GITHUB_ACTION_PATH/scripts/report.py" "$job_directory/result.json" || report_status=$? if [[ "$status" -ne 0 ]]; then fail "hbench run exited $status" diff --git a/tests/report/agent-job/config.json b/tests/report/agent-job/config.json new file mode 100644 index 0000000..6c6acf1 --- /dev/null +++ b/tests/report/agent-job/config.json @@ -0,0 +1,17 @@ +{ + "job_name": "agent-job", + "n_attempts": null, + "agents": [ + { + "name": "claude-code", + "model_name": "anthropic/claude-opus-5-5" + } + ], + "datasets": [ + { + "name": "example", + "version": "1.0", + "registry_path": "registry.json" + } + ] +} diff --git a/tests/report/agent-job/count-words__b2/result.json b/tests/report/agent-job/count-words__b2/result.json new file mode 100644 index 0000000..319dedb --- /dev/null +++ b/tests/report/agent-job/count-words__b2/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "count-words__b2", + "task_name": "example/count-words", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 80000, + "n_cache_tokens": 0, + "n_output_tokens": 2000, + "cost_usd": 0.0434 + }, + "verifier_result": { + "rewards": { + "reward": 0.0, + "field_score": 0.5 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:22:45.123456789Z" + } +} diff --git a/tests/report/agent-job/fix-median__a1/result.json b/tests/report/agent-job/fix-median__a1/result.json new file mode 100644 index 0000000..5f2a856 --- /dev/null +++ b/tests/report/agent-job/fix-median__a1/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "fix-median__a1", + "task_name": "example/fix-median", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 120000, + "n_cache_tokens": 100000, + "n_output_tokens": 3500, + "cost_usd": 0.06 + }, + "verifier_result": { + "rewards": { + "reward": 1.0, + "field_score": 1.0 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:23:30.123456789Z" + } +} diff --git a/tests/report/agent-job/hello-world__c3/result.json b/tests/report/agent-job/hello-world__c3/result.json new file mode 100644 index 0000000..cacc71c --- /dev/null +++ b/tests/report/agent-job/hello-world__c3/result.json @@ -0,0 +1,30 @@ +{ + "trial_name": "hello-world__c3", + "task_name": "example/hello-world", + "source": "example", + "agent_info": { + "name": "claude-code", + "version": "2.1.0", + "model_info": { + "name": "claude-opus-5-5", + "provider": "anthropic" + } + }, + "agent_result": { + "n_input_tokens": 200000, + "n_cache_tokens": 150000, + "n_output_tokens": 9000, + "cost_usd": 0.02 + }, + "verifier_result": null, + "exception_info": { + "exception_type": "AgentTimeoutError", + "exception_message": "agent timed out", + "exception_traceback": "", + "occurred_at": "2026-10-22T05:32:00Z" + }, + "agent_execution": { + "started_at": "2026-10-22T05:22:00.123456789Z", + "finished_at": "2026-10-22T05:32:00.123456789Z" + } +} diff --git a/tests/report/agent-job/lock.json b/tests/report/agent-job/lock.json new file mode 100644 index 0000000..3bb1bd6 --- /dev/null +++ b/tests/report/agent-job/lock.json @@ -0,0 +1,8 @@ +{ + "schema_version": 3, + "hbench": { + "version": "0.3.0", + "results_epoch": "0.2" + }, + "n_concurrent_trials": 3 +} diff --git a/tests/report/agent-job/result.json b/tests/report/agent-job/result.json new file mode 100644 index 0000000..f20f0c9 --- /dev/null +++ b/tests/report/agent-job/result.json @@ -0,0 +1,38 @@ +{ + "started_at": "2026-10-22T05:21:31.249358767Z", + "finished_at": "2026-10-22T05:33:01.100000000Z", + "n_total_trials": 3, + "stats": { + "n_completed_trials": 3, + "n_errored_trials": 1, + "cost_usd": 0.1234, + "evals": { + "claude-code__claude-opus-5-5__example": { + "n_trials": 3, + "n_errors": 1, + "metrics": [ + { + "reward": 0.5, + "field_score": 0.75 + } + ], + "pass_at_k": {}, + "reward_stats": { + "reward": { + "1.0": [ + "fix-median__a1" + ], + "0.0": [ + "count-words__b2" + ] + } + }, + "exception_stats": { + "AgentTimeoutError": [ + "hello-world__c3" + ] + } + } + } + } +} diff --git a/tests/report/oracle-job/config.json b/tests/report/oracle-job/config.json new file mode 100644 index 0000000..9028060 --- /dev/null +++ b/tests/report/oracle-job/config.json @@ -0,0 +1,13 @@ +{ + "agents": [ + { + "name": "oracle", + "model_name": null + } + ], + "datasets": [ + { + "path": "tasks" + } + ] +} diff --git a/tests/report/oracle-job/hello-world__d4/result.json b/tests/report/oracle-job/hello-world__d4/result.json new file mode 100644 index 0000000..8501b7e --- /dev/null +++ b/tests/report/oracle-job/hello-world__d4/result.json @@ -0,0 +1,26 @@ +{ + "trial_name": "hello-world__d4", + "task_name": "hbench/hello-world", + "source": "tasks", + "agent_info": { + "name": "oracle", + "version": null, + "model_info": null + }, + "agent_result": { + "n_input_tokens": null, + "n_cache_tokens": null, + "n_output_tokens": null, + "cost_usd": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "agent_execution": { + "started_at": "2026-10-22T06:00:02Z", + "finished_at": "2026-10-22T06:00:03Z" + } +} diff --git a/tests/report/oracle-job/lock.json b/tests/report/oracle-job/lock.json new file mode 100644 index 0000000..4b37c71 --- /dev/null +++ b/tests/report/oracle-job/lock.json @@ -0,0 +1,6 @@ +{ + "hbench": { + "version": "0.3.0", + "results_epoch": "0.2" + } +} diff --git a/tests/report/oracle-job/result.json b/tests/report/oracle-job/result.json new file mode 100644 index 0000000..38cbc7f --- /dev/null +++ b/tests/report/oracle-job/result.json @@ -0,0 +1,27 @@ +{ + "started_at": "2026-10-22T06:00:00Z", + "finished_at": "2026-10-22T06:00:12Z", + "n_total_trials": 1, + "stats": { + "evals": { + "oracle__tasks": { + "n_trials": 1, + "n_errors": 0, + "metrics": [ + { + "mean": 1.0 + } + ], + "pass_at_k": {}, + "reward_stats": { + "reward": { + "1.0": [ + "hello-world__d4" + ] + } + }, + "exception_stats": {} + } + } + } +} diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..349bff0 --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,115 @@ +import contextlib +import io +import os +import pathlib +import sys +import tempfile +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent / "scripts")) + +import report # noqa: E402 + +AGENT_JOB = HERE / "report" / "agent-job" / "result.json" +ORACLE_JOB = HERE / "report" / "oracle-job" / "result.json" + + +def run_report(result_path, min_mean_reward="", artifact=""): + with tempfile.TemporaryDirectory() as directory: + output, summary = pathlib.Path(directory, "output"), pathlib.Path(directory, "summary") + environment = { + "GITHUB_OUTPUT": str(output), + "GITHUB_STEP_SUMMARY": str(summary), + "INPUT_MIN_MEAN_REWARD": min_mean_reward, + "HBENCH_ARTIFACT_NAME": artifact, + "GITHUB_REPOSITORY": "you/tasks", + "GITHUB_RUN_ID": "123456", + } + with mock.patch.dict(os.environ, environment), contextlib.redirect_stdout(io.StringIO()): + status = report.main([str(result_path)]) + outputs = dict(line.split("=", 1) for line in output.read_text().splitlines()) + return status, outputs, summary.read_text() + + +class AgentJobTest(unittest.TestCase): + def setUp(self): + self.status, self.outputs, self.summary = run_report(AGENT_JOB) + + def test_outputs_count_solved_trials_and_cost(self): + self.assertEqual(self.status, 0) + self.assertEqual(self.outputs["n-trials"], "3") + self.assertEqual(self.outputs["n-solved"], "1") + self.assertEqual(self.outputs["pass-rate"], "0.333333") + self.assertEqual(self.outputs["n-errors"], "1") + self.assertEqual(self.outputs["mean-reward"], "0.5") + self.assertEqual(self.outputs["cost-usd"], "0.1234") + + def test_scorecard_has_pass_rate_time_tokens_and_cost_per_solved_task(self): + self.assertIn("**1 / 3** (33.3%)", self.summary) + self.assertIn("12m 15s total, 1m 30s median", self.summary) + self.assertIn("400.0K / 14.5K (250.0K cached)", self.summary) + self.assertIn("$0.1234 ($0.1234 per solved)", self.summary) + + def test_what_ran_names_hbench_agent_model_and_tasks(self): + self.assertIn("- **hbench** 0.3.0, results epoch 0.2", self.summary) + self.assertIn("- **Agent** `claude-code`, model `anthropic/claude-opus-5-5`", self.summary) + self.assertIn("- **Tasks** `example@1.0` from `registry.json`", self.summary) + self.assertIn("- **Attempts** 1 per task, 3 at a time", self.summary) + self.assertIn("- **Ran** 2026-10-22 05:21 to 05:33 UTC, 11m 30s", self.summary) + + def test_tasks_metrics_and_errors_are_listed(self): + self.assertIn("`field_score` 0.750", self.summary) + self.assertIn("- **AgentTimeoutError** in `claude-code__claude-opus-5-5__example`: 1 (`hello-world__c3`)", self.summary) + + def test_trials_list_errors_then_failures_then_solved(self): + error = self.summary.index("| `example/hello-world` | ⚠️ AgentTimeoutError |") + failure = self.summary.index("| `example/count-words` | ❌ failed | 0.000 |") + solved = self.summary.index("| `example/fix-median` | ✅ solved | 1.000 |") + self.assertLess(error, failure) + self.assertLess(failure, solved) + + def test_charts_show_the_trials_by_outcome_and_the_mean_reward_by_eval(self): + self.assertIn("| Solved | 1 | ███████ 33.3% |", self.summary) + self.assertIn("| Failed | 1 | ███████ 33.3% |", self.summary) + self.assertIn("| Error | 1 | ███████ 33.3% |", self.summary) + self.assertIn("| `claude-code / claude-opus-5-5 / example` | ██████████ 0.500 |", self.summary) + self.assertNotIn("```mermaid", self.summary) + + def test_an_uploaded_job_gets_the_command_that_opens_it_in_the_dashboard(self): + summary = run_report(AGENT_JOB, artifact="hbench-bench")[2] + self.assertIn("View interactive dashboard by hbench:", summary) + self.assertIn( + "gh run download 123456 -R you/tasks -n hbench-bench -D hbench-jobs/agent-job && hbench view hbench-jobs", + summary, + ) + + def test_a_job_that_is_not_uploaded_gets_no_dashboard_command(self): + self.assertNotIn("View interactive dashboard", self.summary) + + def test_mean_reward_below_the_threshold_fails(self): + self.assertEqual(run_report(AGENT_JOB, "0.6")[0], 1) + self.assertEqual(run_report(AGENT_JOB, "0.5")[0], 0) + + +class OracleJobTest(unittest.TestCase): + def test_reward_txt_mean_and_no_usage(self): + status, outputs, summary = run_report(ORACLE_JOB, "1") + self.assertEqual(status, 0) + self.assertEqual(outputs["mean-reward"], "1.0") + self.assertEqual(outputs["pass-rate"], "1.0") + self.assertEqual(outputs["cost-usd"], "") + self.assertIn("**1 / 1** (100.0%)", summary) + self.assertIn("- **Agent** `oracle`", summary) + self.assertIn("- **Tasks** `tasks`", summary) + self.assertIn("✅ At or above min-mean-reward 1.", summary) + self.assertNotIn("### Errors", summary) + self.assertIn("| Solved | 1 | ████████████████████ 100.0% |", summary) + self.assertIn("| Failed | 0 | 0.0% |", summary) + self.assertIn("| Error | 0 | 0.0% |", summary) + self.assertIn("| `oracle / tasks` | ████████████████████ 1.000 |", summary) + + +if __name__ == "__main__": + unittest.main()