From 76ea033919c07536867879539354dbd4c8be24b4 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Thu, 8 Oct 2026 10:55:52 +0800 Subject: [PATCH 1/6] [Feat] Add reproducible public231 decision benchmarks --- CHANGELOG.md | 3 + benchmarks/README.md | 83 +++++ benchmarks/__init__.py | 1 + benchmarks/client.py | 68 ++++ benchmarks/evidence.py | 161 +++++++++ benchmarks/legs/__init__.py | 1 + benchmarks/legs/leg1_public231.py | 248 +++++++++++++ benchmarks/results/public231-h800.md | 92 +++++ tests/benchmarks/test_public231.py | 520 +++++++++++++++++++++++++++ 9 files changed, 1177 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/client.py create mode 100644 benchmarks/evidence.py create mode 100644 benchmarks/legs/__init__.py create mode 100644 benchmarks/legs/leg1_public231.py create mode 100644 benchmarks/results/public231-h800.md create mode 100644 tests/benchmarks/test_public231.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4297334..ef3966b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ### Added +- Public231 decision-model benchmark with raw HTTP evidence, pinned upstream scoring, + complete-denominator quality metrics and per-round service latency. + - Snake recipe and game client: `evals/snake` vendors the laya-mlx snake CLI (Apache-2.0) with single-game paced recording and a 16-game multigrid mode against any `/v1/systemone` backend; `recipes/snake` documents setup, diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..19f462e --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,83 @@ +# Decision model benchmarks + +Compare an existing decision service on JevBench's 231 public questions: easy48, +original72 and hard111. The collector saves every request and response; a separate +command re-scores them with the pinned upstream scorer. + +[H800 results](results/public231-h800.md) compare five open checkpoints. This is the +public231 set, not the article's 248-question or 23-model leaderboard. + +## Run + +Use the repository's development environment and a running service. Clone +[JevBench](https://github.com/fstandhartinger/jevbench) at +`7ce310c7262ed49cc85853339a8a42459298e3f3`; the tool verifies the content hashes of +all three datasets and every scorer module it imports. No dataset is vendored. + +```bash +export JEVBENCH_ROOT=/absolute/path/to/jevbench +python -m benchmarks.legs.leg1_public231 predict \ + --endpoint http://127.0.0.1:8000 --path /v1/systemone \ + --model open-jev-9b --thinking off --warmup 5 --rounds 3 \ + --run-meta /absolute/path/to/environment.json \ + --out /absolute/path/to/results/open-jev-9b +python -m benchmarks.legs.leg1_public231 aggregate \ + --run /absolute/path/to/results/open-jev-9b \ + --out /absolute/path/to/results/open-jev-9b +``` + +Omni native Open-Jev uses `/v1/systemone`; Intern's official service uses +`/v1/decisions`. `--model` is optional. The collector preserves state, questions and +option order, sends `thinking.enabled=false`, and never sends targets or provenance. +It makes one request at a time, without retries. It does not start services or choose GPUs. + +`environment.json` describes the **actual service**, without credentials: + +```json +{ + "checkpoint": {"repo": "owner/model", "revision": "full-checkpoint-commit"}, + "backend": {"name": "serving-backend", "version": "full-source-commit"}, + "sampling": {"temperature": 1.0}, + "hardware": {"gpu": "GPU model"} +} +``` + +Also record dtype, tokenizer/base/adapter/head revisions, loaded libraries, context +limits, cache settings and job identity. The file is copied byte-for-byte and hashed; +all original fields must match the run metadata. Omit `stop_reason` and conflicting +collector fields. Temperature is recorded, not configured by the client. Authenticate +outside this tool if needed; it does not accept or save API keys. + +## Reading the results + +- Each round's accuracy is **correct / 231**. Failures and unattempted questions stay + in the denominator. Three repeated rounds provide timing samples, not 693 unique questions. +- Upstream schema validity, Brier and ECE are reported separately. Missing probabilities + are not replaced with one-hot predictions. Native `noul` scalars become `{no: 1-p, yes: p}`. +- Per-round and pooled p50/p95 cover HTTP POST through full response receipt. All attempts, + successes and failures have separate counts; empty latency sets are null. Warmups are excluded. +- `sufficient=true` requires five warmups and three complete 231-question rounds. + `--limit`, fewer rounds or missing attempts produce partial observations. +- HTTP 401/403/429, interruption or three consecutive infrastructure failures stop collection. + HTTP 422 counts as a failed question and does not trigger that stop counter. + +Evidence goes in a fresh directory **outside the repository**. Each round contains +`results.jsonl` and `raw/.json`; warmups are separate. Raw records retain +the request, response text, decoded response, HTTP status, error, timestamp and latency. +`run.meta.json` and `completion.json` record the plan and any stop reason. + +Aggregation checks source hashes, the environment, input order, raw hashes and record +consistency before re-scoring. Orphan or missing raw files are errors. It never overwrites +an existing summary; use a new output directory for another recomputation. These checks +detect inconsistent evidence, not coordinated rewriting of an unsigned archive. + +## Tests + +```bash +JEVBENCH_ROOT=/absolute/path/to/jevbench python -m pytest tests/benchmarks -q --confcutdir=tests/benchmarks +ruff check benchmarks tests/benchmarks +ruff format --check benchmarks tests/benchmarks +``` + +Reference-dependent tests skip if `JEVBENCH_ROOT` is absent. Mock and loopback tests +check collection and scoring behavior; model quality and latency come from the linked real runs. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..391ce6c --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Offline decision benchmark tools, separate from agent runtime.""" diff --git a/benchmarks/client.py b/benchmarks/client.py new file mode 100644 index 0000000..38d6f00 --- /dev/null +++ b/benchmarks/client.py @@ -0,0 +1,68 @@ +"""One synchronous POST per decision request; no retries or model inference.""" + +import json +import math +import time +from urllib.parse import urlsplit + +import httpx + + +def decode(text): + def invalid_constant(value): + raise ValueError(f"Non-JSON numeric constant: {value}") + + def finite_float(value): + result = float(value) + if not math.isfinite(result): + raise ValueError(f"Non-finite JSON number: {value}") + return result + + return json.loads(text, parse_constant=invalid_constant, parse_float=finite_float) + + +class DecisionClient: + def __init__(self, endpoint, *, path, model=None, timeout_s=120, transport=None): + if not math.isfinite(timeout_s) or timeout_s <= 0: + raise ValueError("timeout_s must be finite and positive") + url = urlsplit(endpoint) + if ( + url.scheme not in ("http", "https") + or not url.netloc + or url.username + or url.password + or url.query + or url.fragment + ): + raise ValueError("Use an HTTP endpoint without credentials or query parameters") + if not path.startswith("/") or "?" in path or "#" in path: + raise ValueError("path must be an absolute API path without query parameters") + self.url, self.model = endpoint.rstrip("/") + path, model + self.http = httpx.Client(timeout=timeout_s, transport=transport, follow_redirects=False, trust_env=False) + + def request(self, body): + request = dict(body) + if self.model is not None: + request.setdefault("model", self.model) + request = decode(json.dumps(request, ensure_ascii=False, allow_nan=False)) + raw = dict(request=request, response_body=None, response=None, http_status=None, client_error=None) + raw["ts"] = time.time() + started = time.perf_counter() + try: + response = self.http.post(self.url, json=request) + raw.update(http_status=response.status_code, response_body=response.text) + except httpx.HTTPError as exc: + raw["client_error"] = f"{type(exc).__name__}: {exc}" + except KeyboardInterrupt: + raw["client_error"] = "KeyboardInterrupt" + finally: + raw["latency_s"] = time.perf_counter() - started + if raw["response_body"] is not None: + try: + raw["response"] = decode(raw["response_body"]) + except ValueError: + pass # The original body remains available for offline diagnosis. + return raw + + def close(self): + self.http.close() diff --git a/benchmarks/evidence.py b/benchmarks/evidence.py new file mode 100644 index 0000000..3940380 --- /dev/null +++ b/benchmarks/evidence.py @@ -0,0 +1,161 @@ +"""Pinned upstream scoring and reproducible public231 evidence.""" + +import hashlib +import importlib +import json +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +from benchmarks.client import decode + +PIN = "7ce310c7262ed49cc85853339a8a42459298e3f3" +HASHES = { + "jevbench/__init__.py": "64c28dc1534502c429a3b67eee79ab7909fab995b7d116c9b089f0f3bb8652f7", + "jevbench/tasks.py": "b9c2f8ba9a7301519656d79d47c31b8ab0542a5545498ce3b339d5e93b976812", + "jevbench/scoring.py": "6aa17d212ea20a3eeb6876149367842e8462c11dcb54d73df062fb406368331e", + "jevbench/summarize.py": "aa5f24da407d004cd80bad1cd957b6987aba152591084c096d1de51dd9775cbe", + "jevbench/metrics.py": "4038c8423ad1cf28601afb6babe777b71f08e379b9ca125b09b52c0ac7d60515", + "datasets/public/easy.jsonl": "231df3c2c8e88a1a8c137ebe85de96ba70fabd330849098ac7b3c52c70b7172b", + "datasets/public/original.jsonl": "5c2414edb3006b8bfcb70fda433f0f9ca015759433849f8d3104328a1f7c4180", + "datasets/public/hard.jsonl": "89e9e6becb33ed88c1de7d42dcc87531b2fb64cfaef4e1986faf7c37b3f80ebb", +} + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def dump(value): + return json.dumps(value, ensure_ascii=False, allow_nan=False) + + +def write_json(path, value): + with Path(path).open("x", encoding="utf-8") as stream: + stream.write(dump(value) + "\n") + + +def outside_repo(path): + path = Path(path).resolve() + if path.is_relative_to(Path(__file__).resolve().parents[1]): + raise ValueError("Evidence must be outside the source repository") + return path + + +def load_reference(): + root = Path(os.environ["JEVBENCH_ROOT"]).resolve() + for name, expected in HASHES.items(): + if digest((root / name).read_bytes()) != expected: + raise ValueError(f"Pinned reference hash mismatch: {name}") + for name, module in tuple(sys.modules.items()): + if name == "jevbench" or name.startswith("jevbench."): + if not Path(module.__file__).resolve().is_relative_to(root): + raise ValueError("Another JevBench root is already imported") + sys.path.insert(0, str(root)) + upstream = SimpleNamespace( + **{name: importlib.import_module(f"jevbench.{name}") for name in ("tasks", "scoring", "summarize", "metrics")} + ) + items = [] + for tier in ("easy", "original", "hard"): + for line, text in enumerate((root / f"datasets/public/{tier}.jsonl").read_text().split("\n")[:-1], 1): + task = upstream.tasks.Task.from_dict(decode(text)) + identity = dict( + suite=f"jevbench-{tier}", + task_id=task.id, + source_file=f"{tier}.jsonl", + source_line=line, + family=task.family, + tier=tier, + split=task.split, + ) + items.append((identity, task)) + dataset = dict( + jevbench_pin=PIN, + root=str(root), + hashes=HASHES, + dataset_hash=upstream.tasks.dataset_hash([task for _, task in items]), + ) + return upstream, items, dataset + + +def body_for(task, model): + body = {"state": task.state, "questions": {"decision": task.question}, "thinking": {"enabled": False}} + if model is not None: + body["model"] = model + return body + + +def score_record(upstream, identity, task, raw, model, round_number, kind, raw_hash): + status, error, probs, source = "ok", None, None, "unavailable" + response = raw["response"] + code = raw["http_status"] + if raw["client_error"] is not None: + status, error = "client_error", raw["client_error"] + elif code is None or not 200 <= code < 300: + status, error = ("refused" if code == 422 else "http_error"), raw["response_body"] + else: + answers = response.get("answers") if isinstance(response, dict) else None + answer = answers.get("decision") if isinstance(answers, dict) else None + if not isinstance(answer, dict) or answer.get("type") != task.question["type"]: + status, error = "parse_error", "Missing or mismatched typed decision answer" + else: + probs = answer.get("probabilities") + source = "response_probabilities" if probs is not None else "unavailable" + if probs is None and task.question["type"] == "noul": + value = answer.get("noul") + if isinstance(value, (float, int)) and not isinstance(value, bool): + probs, source = {"no": 1 - value, "yes": value}, "native_noul_scalar" + try: + scored = upstream.scoring.score_task(probs, task) + except OverflowError: + scored = upstream.scoring.score_task(None, task) + scored["error"] = "Probability exceeds the scorer's numeric range" + return dict( + identity, + round=round_number, + kind=kind, + ts=raw["ts"], + attempted=True, + ok=status == "ok", + status=status, + status_code=code, + error=error or scored.get("error"), + latency_s=raw["latency_s"], + latency_basis="client_single_request_wall_seconds", + **{key: scored.get(key) for key in ("valid", "strict_valid", "renormalized", "correct", "predicted", "probs")}, + expected=task.expected, + candidate_count=len(task.labels), + probs_as_returned=probs, + probs_source=source, + score_source="jevbench_7ce310c7", + model_configured=model, + model=response.get("model") if isinstance(response, dict) else None, + usage=response.get("usage") if isinstance(response, dict) else None, + raw_sha256=raw_hash, + ) + + +def summarize(upstream, tasks, records): + # Upstream accuracy is attempted-only; keep it explicitly diagnostic. + projections = [dict(row, model=str(row["model"]) if row["model"] is not None else "") for row in records] + original = upstream.summarize.summarize(tasks, projections) + correct = sum(bool(row["correct"]) for row in records) + latency = { + name: upstream.metrics.latency_summary([row["latency_s"] for row in rows]) + for name, rows in ( + ("all_attempts", records), + ("success", [r for r in records if r["ok"]]), + ("failure", [r for r in records if not r["ok"]]), + ) + } + return dict( + planned=len(tasks), + attempted=len(records), + valid=sum(r["valid"] for r in records), + correct=correct, + accuracy=correct / len(tasks), + coverage=len(records) / len(tasks), + latency=latency, + upstream_attempted_metrics=original, + ) diff --git a/benchmarks/legs/__init__.py b/benchmarks/legs/__init__.py new file mode 100644 index 0000000..56d7447 --- /dev/null +++ b/benchmarks/legs/__init__.py @@ -0,0 +1 @@ +"""Benchmark collection and aggregation command lines.""" diff --git a/benchmarks/legs/leg1_public231.py b/benchmarks/legs/leg1_public231.py new file mode 100644 index 0000000..7630fc1 --- /dev/null +++ b/benchmarks/legs/leg1_public231.py @@ -0,0 +1,248 @@ +"""Collect serial public231 rounds or validate and aggregate saved evidence.""" + +import argparse +import copy +from contextlib import closing +import math +import os +import sys +import time +from pathlib import Path + +from benchmarks.client import DecisionClient, decode +from benchmarks.evidence import ( + body_for, + digest, + dump, + load_reference, + outside_repo, + score_record, + summarize, + write_json, +) + + +def validate_environment(meta): + for group, fields in (("checkpoint", ("repo", "revision")), ("backend", ("name", "version"))): + if any(not meta.get(group, {}).get(field) for field in fields): + raise ValueError(f"run-meta requires actual {group}: {fields}") + value = meta.get("sampling", {}).get("temperature") + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + raise ValueError("run-meta requires actual positive sampling.temperature") + if not meta.get("hardware"): + raise ValueError("run-meta requires actual hardware") + + +def predict(args): + upstream, items, dataset = load_reference() + environment = args.run_meta.read_bytes() + meta = decode(environment) + validate_environment(meta) + if "stop_reason" in meta: + raise ValueError("stop_reason is collector-owned state; omit it from the supplied environment") + if args.rounds < 1 or not 0 <= args.warmup <= 231 or not 1 <= args.limit <= 231: + raise ValueError("Invalid rounds, warmup or limit") + output = outside_repo(args.out) + plan = dict( + warmup=args.warmup, + rounds=args.rounds, + tasks=231, + limit=args.limit, + order="dataset sequential", + identities=[identity for identity, _ in items], + ) + collector = dict( + run_id=output.name, + argv=sys.argv, + endpoint=args.endpoint, + path=args.path, + model_configured=args.model, + response_parse="typed", + thinking=False, + dataset=dataset, + plan=plan, + env_file=str(args.run_meta.resolve()), + env_sha256=digest(environment), + stop_reason=None, + ts=time.time(), + ) + if any(key in meta and meta[key] != value for key, value in collector.items()): + raise ValueError("Supplied environment conflicts with collector metadata") + meta.update(collector) + with closing(DecisionClient(args.endpoint, path=args.path, model=args.model, timeout_s=args.timeout)) as client: + output.mkdir(parents=True, exist_ok=False) + with (output / "environment.json").open("xb") as stream: + stream.write(environment) + write_json(output / "run.meta.json", meta) + stop, errors = None, 0 + try: + for number in range(args.rounds + 1): + kind = "warmup" if number == 0 else "measured" + directory = output / ("warmup" if number == 0 else f"round-{number}") + (directory / "raw").mkdir(parents=True) + selected = items[: args.warmup if number == 0 else args.limit] + write_json( + directory / "predict.meta.json", dict(round=number, planned=args.warmup if number == 0 else 231) + ) + with (directory / ("records.jsonl" if number == 0 else "results.jsonl")).open("x") as stream: + for identity, task in selected: + raw = client.request(body_for(task, args.model)) + raw_path = directory / "raw" / (digest(task.id.encode()) + ".json") + write_json(raw_path, raw) + record = score_record( + upstream, identity, task, raw, args.model, number, kind, digest(raw_path.read_bytes()) + ) + stream.write(dump(record) + "\n") + stream.flush() + os.fsync(stream.fileno()) + if raw["client_error"] == "KeyboardInterrupt": + stop = "interrupted" + break + errors = errors + 1 if not record["ok"] and record["status_code"] != 422 else 0 + if record["status_code"] in (401, 403, 429) or errors >= 3: + stop = "access/rate limit or three consecutive infrastructure errors" + break + if stop: + break + except KeyboardInterrupt: + stop = "interrupted" + except Exception as exc: + stop = f"{type(exc).__name__}: {exc}" + raise + finally: + write_json(output / "completion.json", dict(stop_reason=stop, ts=time.time())) + meta["stop_reason"] = stop + write_json(output / "run.meta.final.json", meta) + (output / "run.meta.final.json").replace(output / "run.meta.json") + return output + + +def read_round(directory, expected, upstream, model, number): + path = directory / ("records.jsonl" if number == 0 else "results.jsonl") + records = [] + if path.exists(): + with path.open(encoding="utf-8") as stream: + records = [decode(line) for line in stream] + if len(records) > len(expected): + raise ValueError("Too many or duplicate records") + expected_raw = {digest(task.id.encode()) + ".json" for _, task in expected[: len(records)]} + actual_raw = {path.name for path in (directory / "raw").glob("*")} + if expected_raw != actual_raw: + raise ValueError( + f"Raw/record mismatch in {directory}: missing={sorted(expected_raw - actual_raw)}, " + f"orphan={sorted(actual_raw - expected_raw)}" + ) + for row, (identity, task) in zip(records, expected): + raw_path = directory / "raw" / (digest(task.id.encode()) + ".json") + data = raw_path.read_bytes() + raw = decode(data) + if dump(raw["request"]) != dump(body_for(task, model)): + raise ValueError("Raw request differs from frozen task") + parsed = None + if raw["response_body"] is not None: + try: + parsed = decode(raw["response_body"]) + except ValueError: + pass + if raw["response"] != parsed: + raise ValueError("Decoded response differs from original body") + if ( + not isinstance(raw["latency_s"], (int, float)) + or not math.isfinite(raw["latency_s"]) + or raw["latency_s"] < 0 + ): + raise ValueError("Invalid measured latency") + recomputed = score_record( + upstream, identity, task, raw, model, number, "warmup" if number == 0 else "measured", digest(data) + ) + if row != recomputed: + raise ValueError("Record differs from raw evidence or planned identity/order") + return records + + +def aggregate(args): + upstream, items, dataset = load_reference() + run = outside_repo(args.run) + meta = decode((run / "run.meta.json").read_text()) + validate_environment(meta) + environment = (run / "environment.json").read_bytes() + if digest(environment) != meta["env_sha256"]: + raise ValueError("Environment source hash mismatch") + env = decode(environment) + if any(key not in meta or meta[key] != value for key, value in env.items()): + raise ValueError("Run metadata differs from supplied environment") + plan = meta["plan"] + if ( + any(meta["dataset"][key] != dataset[key] for key in ("jevbench_pin", "hashes", "dataset_hash")) + or plan["identities"] != [identity for identity, _ in items] + or plan["tasks"] != 231 + ): + raise ValueError("Frozen plan/reference mismatch") + if plan["rounds"] < 1 or not 0 <= plan["warmup"] <= 231 or not 1 <= plan["limit"] <= 231: + raise ValueError("Invalid frozen run plan") + expected_dirs = {f"round-{i}" for i in range(1, plan["rounds"] + 1)} + if {p.name for p in run.glob("round-*")} - expected_dirs: + raise ValueError("Unknown round directory") + warmup = read_round(run / "warmup", items[: plan["warmup"]], upstream, meta["model_configured"], 0) + rounds, pooled_tasks, pooled_records = [], [], [] + for number in range(1, plan["rounds"] + 1): + records = read_round( + run / f"round-{number}", items[: plan["limit"]], upstream, meta["model_configured"], number + ) + tasks = [task for _, task in items] + rounds.append(dict(round=number, **summarize(upstream, tasks, records))) + # Give repeated tasks distinct IDs only for upstream duplicate checking. + for task in tasks: + clone = copy.copy(task) + clone.id = f"{number}:{task.id}" + clone.group = f"{number}:{task.group}" if task.group else None + pooled_tasks.append(clone) + pooled_records.extend(dict(row, task_id=f"{number}:{row['task_id']}") for row in records) + sufficient = ( + plan["rounds"] == 3 + and plan["warmup"] == 5 + and plan["limit"] == 231 + and len(warmup) == 5 + and all(r["attempted"] == 231 for r in rounds) + ) + result = dict( + sufficient=sufficient, + status="sufficient" if sufficient else "insufficient", + warmup_attempted=len(warmup), + rounds=rounds, + pooled=summarize(upstream, pooled_tasks, pooled_records), + distinct_tasks=231, + cost_usd=None, + cost_basis="unmetered is not free", + latency_success_basis="HTTP success with a typed decision structure; schema validity is separate", + ) + output = outside_repo(args.out) + output.mkdir(parents=True, exist_ok=True) + write_json(output / "summary.json", result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + collector = commands.add_parser("predict") + collector.add_argument("--endpoint", required=True) + collector.add_argument("--path", required=True, choices=("/v1/systemone", "/v1/decisions")) + collector.add_argument("--model") + collector.add_argument("--thinking", choices=("off",), default="off") + collector.add_argument("--warmup", type=int, default=5) + collector.add_argument("--rounds", type=int, default=3) + collector.add_argument("--limit", type=int, default=231) + collector.add_argument("--timeout", type=float, default=120) + collector.add_argument("--run-meta", type=Path, required=True) + collector.add_argument("--out", type=Path, required=True) + merger = commands.add_parser("aggregate") + merger.add_argument("--run", type=Path, required=True) + merger.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + result = predict(args) if args.command == "predict" else aggregate(args) + print(result if isinstance(result, Path) else dump(result)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/results/public231-h800.md b/benchmarks/results/public231-h800.md new file mode 100644 index 0000000..c08a993 --- /dev/null +++ b/benchmarks/results/public231-h800.md @@ -0,0 +1,92 @@ +# Public231 on one H800 + +Five open checkpoints, one NVIDIA H800, one HTTP request at a time. Each service +received five warmups followed by the same 231 questions in the same order for three +rounds. Latency covers the complete HTTP request and response. No profiler was active. + +Open-Jev used System1-Omni's native Rust/CUDA service; Intern-Decision used its +upstream HF service. These results compare deployed configurations. Their ratios +include different architectures, templates and execution paths, not just model size +or kernel speed. + +## Results + +| Model | Service | Correct / 231 | Accuracy | Validity | Brier | ECE | p50 / p95 ms | +|---|---|---:|---:|---:|---:|---:|---:| +| Intern-Decision-0.8B | HF / T=1 | 164/231 | 70.9957% | 100% | 0.369906 | 0.111077 | 80.493 / 175.237 | +| Intern-Decision-2B | HF / T=1 | 181/231 | 78.3550% | 100% | 0.306049 | 0.115420 | 79.054 / 175.276 | +| Intern-Decision-4B | HF / T=1 | 200/231 | 86.5801% | 100% | 0.189601 | 0.067214 | 104.552 / 235.291 | +| Open-Jev-9B | Omni native | 179/231 | 77.4892% | 100% | 0.320752 | 0.086786 | 42.688 / 327.057 | +| Open-Jev-27B-v1.1 | Omni native | 198/231 | 85.7143% | 100% | 0.240493 | 0.130333 | 68.877 / 1068.817 | + +All five configurations completed 693/693 measured requests, with no failed or invalid +responses. Predictions and probabilities matched across the three rounds. Accuracy +therefore has 231 unique questions; the repeated rounds are timing repetitions. +Brier/ECE use all 693 valid distributions. Failure latency has n=0 and null quantiles. + +Open-Jev 9B had the lowest median latency. Intern 4B answered 200/231 correctly versus +Open-Jev 27B's 198/231, and had a lower p95; Open-Jev 27B had a lower median. This small +set does not establish a general quality ranking or a uniform 2–3× speed advantage. + +## Per-round latency + +| Model | Round 1 p50 / p95 ms | Round 2 p50 / p95 ms | Round 3 p50 / p95 ms | +|---|---:|---:|---:| +| Intern-Decision-0.8B | 84.534 / 180.725 | 73.301 / 173.430 | 73.485 / 172.817 | +| Intern-Decision-2B | 81.454 / 180.338 | 74.251 / 173.638 | 74.154 / 173.411 | +| Intern-Decision-4B | 107.711 / 240.246 | 98.628 / 232.829 | 98.065 / 232.709 | +| Open-Jev-9B | 43.166 / 317.867 | 42.624 / 317.644 | 42.525 / 317.579 | +| Open-Jev-27B-v1.1 | 68.925 / 1030.605 | 68.892 / 1034.555 | 68.821 / 1033.953 | + +Pooled quantiles above are computed from all raw timings, not from these three quantiles. + +## Configuration + +- BF16; client concurrency 1; no retries; thinking disabled. +- Intern: official source `3572c8a68b5df5dafe02d0e093989ba8ec0183bc`, + PyTorch 2.9.1+cu128, torchvision 0.24.1+cu128, Transformers 5.14.1, SDPA, + temperature 1, no calibration artifact, no KV cache. Optional FLA/causal-conv1d + fast paths were unavailable, so the service used the upstream Torch fallback. +- Open-Jev: Omni `4a79980d8a75fd063cb3f8247e06215e288b18ac`, CUDA 13, + merged LoRA and decision head, Graph and prefix cache disabled. 9B evaluates + candidates sequentially; 27B packs candidates. A single service request is not + necessarily one candidate sequence. +- Context limits: Intern 8192 tokens for the complete chat; Open-Jev 16384 per + candidate. Native usage counts tokens summed over candidates, so the two services' + token counts are not directly comparable. + +| Checkpoint | Revision | Temperature | Spawn-to-ready s | +|---|---|---:|---:| +| [Intern 0.8B](https://huggingface.co/internlm/Intern-Decision-0.8B) | `85a0cc5a99d67ea8d56dfe98115689212867171d` | 1 | 16.084 | +| [Intern 2B](https://huggingface.co/internlm/Intern-Decision-2B) | `8797836c65fc91a2435b1fb6850b5f0aabd75cc3` | 1 | 14.045 | +| [Intern 4B](https://huggingface.co/internlm/Intern-Decision-4B) | `0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd` | 1 | 16.048 | +| [Open-Jev 9B](https://huggingface.co/ZefanCai/Open-Jev-9B) | `47e966881e489511c0c7f5633a9e1960a676a551` | 1.8969118766 | 8.015 | +| [Open-Jev 27B v1.1](https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1) | `28cf73067d5b337860bbef3c85b8b82ba8730956` | 2.5343690298 | 20.029 | + +Cold startup is excluded from request latency and includes native internal warmup. +Open 9B's base is `c202236235762e1c871ad0ccb60c8ee5ba337b9a`; Open 27B's base is +`1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0`. + +## Inputs and verification + +[JevBench](https://github.com/fstandhartinger/jevbench/tree/7ce310c7262ed49cc85853339a8a42459298e3f3) +provides easy48, original72 and hard111 and the scorer. Source and data hashes are +pinned in `benchmarks/evidence.py`. The article's complete 248-question set was not +available; this is not a reproduction of its 23-model leaderboard. + +The run used Agents base `9790b52ef4d78a2215e8dd5a526d3a8daea9875a`, dated +2026-10-07 UTC, Slurm job 415076. All 3490 raw requests, including warmups, were +checked for source identity, order, response hash and absence of target leakage. +A non-author independently re-scored the saved responses with the pinned scorer +and recomputed latency from raw samples. The final collector's offline aggregation +matched counts exactly and floating-point metrics within 1e-12. + +Collection preceded fixes to metadata validation, URL validation before file writes, +and JSONL physical-line reading. These fixes preserve the HTTP and scoring paths. +Verification replays the saved responses; it is not a second model run. + +Actual loaded-library maps and the weight hashes were recorded. The native binary +SHA-256 was `a0694633308ea0ad0ca2a302bfb16cbb73fd3367cd36cd142596761a71d23afc`; +CUDA library SHA-256 was `a0db4905df4fe2fb5d006acaaa09b5b51dbd3933cc8e9b20ec8114f23def8b20`. +Use the [collection and replay commands](../README.md#run) with the fixed inputs and +record the actual serving environment. Large raw traces and weights stay outside Git. diff --git a/tests/benchmarks/test_public231.py b/tests/benchmarks/test_public231.py new file mode 100644 index 0000000..0f5720e --- /dev/null +++ b/tests/benchmarks/test_public231.py @@ -0,0 +1,520 @@ +"""CPU protocol fixtures only; no model service or network connection.""" + +import copy +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest + +from benchmarks.client import DecisionClient +from benchmarks.evidence import body_for, digest, dump, load_reference, score_record, summarize +from benchmarks.legs import leg1_public231 as cli + + +@pytest.fixture +def reference(): + if not os.environ.get("JEVBENCH_ROOT"): + pytest.skip("Set JEVBENCH_ROOT to the pinned external reference") + return load_reference() + + +def test_multifield_single_post_preserves_input(): + body = { + "state": [{"context": "structured"}], + "questions": { + "second": {"type": "choice", "criteria": {"z": "last", "a": "first"}}, + "first": {"type": "noul", "instructions": "accept?"}, + }, + "thinking": {"enabled": False}, + } + before, calls = copy.deepcopy(body), [] + + def handler(request): + calls.append(json.loads(request.content)) + assert str(request.url) == "http://fixture.invalid/v1/decisions" + return httpx.Response(200, text=' {"answers": {"second": {}, "first": {}}, "model":"fixture"}\n') + + client = DecisionClient("http://fixture.invalid", path="/v1/decisions", transport=httpx.MockTransport(handler)) + raw = client.request(body) + client.close() + assert body == before and calls == [body] + assert list(calls[0]["questions"]) == ["second", "first"] + assert list(calls[0]["questions"]["second"]["criteria"]) == ["z", "a"] + assert raw["response_body"].startswith(" ") and raw["response_body"].endswith("\n") + assert set(raw["response"]["answers"]) == {"second", "first"} + assert raw["http_status"] == 200 and raw["latency_s"] > 0 + + +@pytest.mark.parametrize("status,text", [(422, "too long"), (500, "failure"), (200, "broken json")]) +def test_raw_bad_responses_survive(status, text): + client = DecisionClient( + "http://fixture.invalid", + path="/v1/systemone", + transport=httpx.MockTransport(lambda _: httpx.Response(status, text=text)), + ) + raw = client.request({"questions": {}}) + client.close() + assert (raw["http_status"], raw["response_body"], raw["response"]) == (status, text, None) + assert raw["latency_s"] > 0 + + +@pytest.mark.parametrize("error", [httpx.ReadTimeout("fixture timeout"), KeyboardInterrupt()]) +def test_network_error_has_real_elapsed(error): + def handler(_): + raise error + + client = DecisionClient("http://fixture.invalid", path="/v1/systemone", transport=httpx.MockTransport(handler)) + raw = client.request({"state": "fixture"}) + client.close() + assert raw["http_status"] is None and raw["response_body"] is None + assert raw["client_error"] and raw["latency_s"] > 0 + + +@pytest.mark.parametrize("url", ["http://user:secret@fixture.invalid", "http://fixture.invalid?key=x"]) +def test_credentials_cannot_enter_endpoint_evidence(url): + with pytest.raises(ValueError): + DecisionClient(url, path="/v1/systemone") + + +def raw_answer(answer, *, status=200, seconds=1): + response = {"answers": {"decision": answer}, "model": "fixture", "usage": {"decision_count": 1}} + return dict( + request={}, + response=response, + response_body=dump(response), + http_status=status, + client_error=None, + latency_s=seconds, + ts=123.0, + ) + + +@pytest.mark.parametrize( + "probs,valid,renormalized", + [ + ({"no": 0.4, "yes": 0.595}, True, True), + ({"no": 0.4, "yes": 0.5}, False, False), + ({"no": 0.4}, False, False), + ({"no": 0.4, "yes": 0.6, "extra": 0}, False, False), + ({"no": -0.1, "yes": 1.1}, False, False), + ], +) +def test_upstream_alone_decides_probability_band(reference, probs, valid, renormalized): + upstream, items, _ = reference + identity, task = next(item for item in items if item[1].question["type"] == "noul") + row = score_record( + upstream, + identity, + task, + raw_answer({"type": "noul", "probabilities": probs}), + "configured", + 1, + "measured", + "fixture-hash", + ) + assert row["ok"] and row["valid"] is valid and row["renormalized"] is renormalized + assert row["probs_as_returned"] == probs + if renormalized: + assert row["strict_valid"] is False and sum(row["probs"].values()) == pytest.approx(1) + if not valid: + assert row["correct"] is False and row["probs"] is None + + +def test_native_scalar_and_missing_probabilities(reference): + upstream, items, _ = reference + for kind in ("noul", "choice", "score"): + identity, task = next(item for item in items if item[1].question["type"] == kind) + answer = {"type": kind, "noul": 0.6} if kind == "noul" else {"type": kind, kind: task.expected} + row = score_record(upstream, identity, task, raw_answer(answer), None, 1, "measured", "h") + assert row["valid"] is (kind == "noul") + assert row["model_configured"] is None and row["model"] == "fixture" + if kind == "noul": + assert row["probs"] == {"no": 0.4, "yes": 0.6} + else: + assert row["probs"] is None and row["probs_as_returned"] is None + + +@pytest.mark.parametrize("answers", [None, [], "bad", {"decision": None}, {"decision": {"type": "wrong"}}]) +def test_malformed_typed_structure_is_recorded(reference, answers): + upstream, items, _ = reference + identity, task = items[0] + raw = raw_answer({}) + raw["response"]["answers"] = answers + row = score_record(upstream, identity, task, raw, None, 1, "measured", "h") + assert row["status"] == "parse_error" and not row["ok"] and not row["valid"] + + +def test_nonfinite_response_is_not_json(): + client = DecisionClient( + "http://fixture.invalid", + path="/v1/systemone", + transport=httpx.MockTransport( + lambda _: httpx.Response( + 200, text='{"answers":{"decision":{"type":"noul","probabilities":{"no":NaN,"yes":1}}}}' + ) + ), + ) + raw = client.request({}) + client.close() + assert raw["response"] is None and "NaN" in raw["response_body"] + + +@pytest.fixture +def collect(reference, monkeypatch, tmp_path): + upstream, items, _ = reference + env = tmp_path / "actual-environment.json" + env.write_text( + dump( + dict( + checkpoint={"repo": "CPU-fixture", "revision": "fixture-only"}, + backend={"name": "httpx.MockTransport", "version": httpx.__version__}, + sampling={"temperature": 1.0}, + hardware="CPU protocol fixture, no inference", + dtype="fixture-original", + tokenizer={"revision": "fixture-tokenizer"}, + base={"revision": "fixture-base"}, + adapter={"revision": "fixture-adapter"}, + head={"revision": "fixture-head"}, + context_limit=16384, + cache={"enabled": False}, + job_id="CPU-fixture", + ) + ) + ) + lookup = {dump(body_for(task, "configured")): task for _, task in items} + + def run(*, warmup=5, rounds=3, limit=231, statuses=None, response_text=None): + calls = [] + + def handler(request): + body = json.loads(request.content) + assert "targets" not in body and "provenance" not in body and "expected" not in body + assert body["thinking"] == {"enabled": False} + task = lookup[dump(body)] + calls.append(body) + status = statuses[min(len(calls) - 1, len(statuses) - 1)] if statuses else 200 + if response_text is not None: + return httpx.Response(status, text=response_text) + answer = { + "type": task.question["type"], + "probabilities": {label: float(label == str(task.expected)) for label in task.labels}, + } + return httpx.Response(status, json={"answers": {"decision": answer}, "model": "fixture-response"}) + + class FixtureClient(DecisionClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, transport=httpx.MockTransport(handler)) + + def request(self, body): + raw = super().request(body) + raw["latency_s"] = 999 if len(calls) <= warmup else (1 if raw["http_status"] == 200 else 120) + return raw + + monkeypatch.setattr(cli, "DecisionClient", FixtureClient) + output = tmp_path / f"run-{len(list(tmp_path.glob('run-*')))}" + args = SimpleNamespace( + endpoint="http://fixture.invalid", + path="/v1/decisions", + model="configured", + timeout=10, + run_meta=env, + out=output, + warmup=warmup, + rounds=rounds, + limit=limit, + ) + cli.predict(args) + return output, calls, args + + return run + + +def aggregate(run, out=None): + return cli.aggregate(SimpleNamespace(run=run, out=out or run)) + + +@pytest.mark.parametrize( + "endpoint", ["http://user:DUMMY_SECRET@fixture.invalid", "http://fixture.invalid?key=DUMMY_SECRET"] +) +def test_predict_cli_rejects_credentials_before_output(collect, monkeypatch, endpoint): + run, calls, args = collect(warmup=0, rounds=1, limit=1) + args.out = run.parent / "rejected-endpoint" + monkeypatch.setattr( + "sys.argv", + [ + "public231", + "predict", + "--endpoint", + endpoint, + "--path", + args.path, + "--run-meta", + str(args.run_meta), + "--out", + str(args.out), + ], + ) + with pytest.raises(ValueError, match="without credentials"): + cli.main() + assert not args.out.exists() and len(calls) == 1 + assert all("DUMMY_SECRET" not in path.read_text() for path in run.parent.rglob("*") if path.is_file()) + + +@pytest.mark.parametrize("failure", ["mkdir", "environment.json", "run.meta.json"]) +def test_predict_closes_client_when_output_setup_fails(collect, monkeypatch, failure): + run, calls, args = collect(warmup=0, rounds=1, limit=1) + args.out = run.parent / "failed-output" + created, original_client, original_open = [], cli.DecisionClient, Path.open + + def client(*args, **kwargs): + result = original_client(*args, **kwargs) + created.append(result) + return result + + def open_file(path, *options, **kwargs): + if path == args.out / failure: + raise OSError("fixture output failure") + return original_open(path, *options, **kwargs) + + if failure == "mkdir": + args.out.mkdir() + monkeypatch.setattr(cli, "DecisionClient", client) + monkeypatch.setattr(Path, "open", open_file) + with pytest.raises(OSError): + cli.predict(args) + assert len(created) == 1 and created[0].http.is_closed + assert len(calls) == 1 + + +@pytest.mark.parametrize("separator", ["\u2028", "\u0085", "\u2029"]) +def test_collect_and_aggregate_preserve_unicode_in_error_body(collect, separator): + run, calls, _ = collect( + warmup=0, rounds=1, limit=1, statuses=[422], response_text=dump({"detail": f"bad{separator}input"}) + ) + with (run / "round-1/results.jsonl").open() as stream: + records = [json.loads(line) for line in stream] + assert len(calls) == len(records) == 1 and records[0]["status"] == "refused" + assert separator in records[0]["error"] + result = aggregate(run) + assert result["pooled"]["planned"] == 231 and result["pooled"]["attempted"] == 1 + assert result["pooled"]["correct"] == result["pooled"]["accuracy"] == 0 + assert not result["sufficient"] + + +@pytest.mark.parametrize("line", ["\n", "{broken}\n"]) +def test_aggregate_rejects_blank_or_malformed_physical_line(collect, line): + run, _, _ = collect(warmup=0, rounds=1, limit=1) + (run / "round-1/results.jsonl").write_text(line) + with pytest.raises(ValueError): + aggregate(run) + + +def test_complete_5_plus_3_rounds_and_no_warmup_in_latency(collect): + run, calls, _ = collect() + result = aggregate(run) + assert len(calls) == 698 and result["sufficient"] + assert result["pooled"]["planned"] == result["pooled"]["correct"] == 693 + assert result["pooled"]["latency"]["all_attempts"] == {"n": 693, "p50_s": 1, "p95_s": 1} + assert result["pooled"]["latency"]["failure"] == {"n": 0, "p50_s": None, "p95_s": None} + assert all(round_["attempted"] == 231 for round_ in result["rounds"]) + assert len(list((run / "warmup/raw").glob("*.json"))) == 5 + second = aggregate(run, run / "offline-recompute") + assert result == second + with pytest.raises(FileExistsError): + aggregate(run) + + +@pytest.mark.parametrize("options", [{"limit": 10}, {"warmup": 0}, {"rounds": 1}]) +def test_debug_run_is_insufficient_with_fixed_denominator(collect, options): + run, _, _ = collect(**options) + result = aggregate(run) + assert not result["sufficient"] + if "limit" in options: + assert result["pooled"]["accuracy"] == 30 / 693 + assert result["rounds"][0]["accuracy"] == 10 / 231 + + +@pytest.mark.parametrize( + "statuses,attempted", [([401], 1), ([403], 1), ([429], 1), ([500], 3), ([500, 500, 422, 500, 500, 500], 6)] +) +def test_stop_policy_leaves_unattempted_in_plan(collect, statuses, attempted): + run, calls, _ = collect(warmup=0, statuses=statuses) + result = aggregate(run) + assert len(calls) == attempted + assert result["pooled"]["accuracy"] == 0 and result["pooled"]["planned"] == 693 + assert result["pooled"]["attempted"] == attempted and not result["sufficient"] + assert json.loads((run / "run.meta.json").read_text())["stop_reason"] + + +def test_failures_and_partial_denominator_with_two_latencies(collect): + run, _, _ = collect(warmup=0, limit=2, rounds=1, statuses=[200, 500]) + result = aggregate(run)["rounds"][0] + assert result["accuracy"] == 1 / 231 + assert result["latency"]["success"] == {"n": 1, "p50_s": 1, "p95_s": 1} + assert result["latency"]["failure"] == {"n": 1, "p50_s": 120, "p95_s": 120} + assert result["latency"]["all_attempts"]["p50_s"] == 60.5 + assert result["latency"]["all_attempts"]["p95_s"] == pytest.approx(114.05) + + +@pytest.mark.parametrize("change", ["duplicate", "unknown", "score", "raw", "environment", "order"]) +def test_offline_tampering_rejected(collect, change): + run, _, _ = collect(warmup=0, rounds=1, limit=2) + path = run / "round-1/results.jsonl" + rows = [json.loads(line) for line in path.read_text().splitlines()] + if change == "duplicate": + rows[1] = rows[0] + elif change == "unknown": + rows[0]["task_id"] = "unknown" + elif change == "score": + rows[0]["correct"] = False + elif change == "order": + rows.reverse() + elif change == "raw": + raw = run / "round-1/raw" / (digest(rows[0]["task_id"].encode()) + ".json") + raw.write_text(raw.read_text() + " ") + else: + path_env = run / "environment.json" + path_env.write_text(path_env.read_text() + " ") + path.write_text("".join(dump(row) + "\n" for row in rows)) + with pytest.raises(ValueError): + aggregate(run) + + +def test_output_and_environment_boundaries(collect, tmp_path): + run, _, args = collect(warmup=0, limit=1, rounds=1) + with pytest.raises(FileExistsError): + cli.predict(args) + args.out = Path(__file__).resolve().parents[2] / "forbidden-evidence" + with pytest.raises(ValueError, match="outside"): + cli.predict(args) + env = json.loads(args.run_meta.read_text()) + env["sampling"]["temperature"] = 0 + args.run_meta.write_text(dump(env)) + args.out = tmp_path / "invalid-temperature" + with pytest.raises(ValueError, match="temperature"): + cli.predict(args) + assert not args.out.exists() + + +def test_reference_hash_tampering_rejected(reference, monkeypatch, tmp_path): + _, _, dataset = reference + from benchmarks.evidence import HASHES + + root = tmp_path / "reference" + for name in HASHES: + destination = root / name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes((Path(dataset["root"]) / name).read_bytes()) + (root / "jevbench/scoring.py").write_text("raise RuntimeError('must not execute')\n") + monkeypatch.setenv("JEVBENCH_ROOT", str(root)) + with pytest.raises(ValueError, match="hash mismatch"): + load_reference() + + +def test_empty_summary_has_null_latency(reference): + upstream, items, _ = reference + result = summarize(upstream, [task for _, task in items], []) + assert result["accuracy"] == 0 and result["coverage"] == 0 + assert all(value == {"n": 0, "p50_s": None, "p95_s": None} for value in result["latency"].values()) + + +@pytest.mark.parametrize("literal", ["1e309", "NaN", "Infinity"]) +def test_nonfinite_wire_numbers_are_retained_as_parse_failures(literal): + response = '{"answers":{"decision":{"type":"noul","noul":' + literal + "}}}" + client = DecisionClient( + "http://fixture.invalid", + path="/v1/systemone", + transport=httpx.MockTransport(lambda _: httpx.Response(200, text=response)), + ) + raw = client.request({}) + client.close() + assert raw["response"] is None and raw["response_body"] == response + + +def test_scorer_numeric_overflow_counts_wrong(reference): + upstream, items, _ = reference + identity, task = next(item for item in items if item[1].question["type"] == "noul") + probs = {"no": 10**400, "yes": 0} + row = score_record( + upstream, identity, task, raw_answer({"type": "noul", "probabilities": probs}), "configured", 1, "measured", "h" + ) + assert row["ok"] and not row["valid"] and not row["correct"] + assert row["probs_as_returned"] == probs + + +@pytest.mark.parametrize("directory,filename", [("warmup", "records.jsonl"), ("round-1", "results.jsonl")]) +@pytest.mark.parametrize("change", ["missing_row", "missing_records_file", "missing_raw", "unknown_raw"]) +def test_raw_and_records_must_correspond(collect, directory, filename, change): + run, _, _ = collect(warmup=2, rounds=1, limit=2) + directory = run / directory + path = directory / filename + rows = path.read_text().splitlines() + row = json.loads(rows[-1]) + raw = directory / "raw" / (digest(row["task_id"].encode()) + ".json") + changed = raw + if change == "missing_row": + path.write_text(rows[0] + "\n") + elif change == "missing_records_file": + path.unlink() + elif change == "missing_raw": + raw.unlink() + else: + changed = directory / "raw/unknown.json" + changed.write_bytes(raw.read_bytes()) + with pytest.raises(ValueError, match="Raw/record mismatch") as error: + aggregate(run) + assert changed.name in str(error.value) + assert not (run / "summary.json").exists() + + +@pytest.mark.parametrize("field", ["dtype", "tokenizer", "base", "adapter", "head", "context_limit", "cache", "job_id"]) +@pytest.mark.parametrize("change", ["different", "missing"]) +def test_all_environment_fields_must_match(collect, field, change): + run, _, _ = collect(warmup=0, rounds=1, limit=1) + path = run / "run.meta.json" + meta = json.loads(path.read_text()) + if change == "different": + meta[field] = "fixture-contradiction" + else: + del meta[field] + path.write_text(dump(meta)) + with pytest.raises(ValueError, match="supplied environment"): + aggregate(run) + assert not (run / "summary.json").exists() + + +def test_conflicting_environment_controls_rejected_before_collection(collect): + run, calls, args = collect(warmup=0, rounds=1, limit=1) + env = json.loads(args.run_meta.read_text()) + env["endpoint"] = "http://conflicting.invalid" + args.run_meta.write_text(dump(env)) + args.out = run.parent / "conflicting-run" + with pytest.raises(ValueError, match="collector"): + cli.predict(args) + assert len(calls) == 1 and not args.out.exists() + + +def test_matching_environment_controls_are_compatible(collect): + run, _, args = collect(warmup=0, rounds=1, limit=1) + env = json.loads(args.run_meta.read_text()) + env.update(endpoint=args.endpoint, path=args.path, thinking=False) + args.run_meta.write_text(dump(env)) + args.out = run.parent / "matching-run" + cli.predict(args) + assert aggregate(args.out)["pooled"]["attempted"] == 1 + + +@pytest.mark.parametrize("stop_reason", [None, "stale stop reason"]) +def test_environment_stop_reason_rejected_before_request(collect, stop_reason): + run, calls, args = collect(warmup=0, rounds=1, limit=1, statuses=[401]) + assert aggregate(run)["pooled"]["attempted"] == 1 + env = json.loads(args.run_meta.read_text()) + env["stop_reason"] = stop_reason + args.run_meta.write_text(dump(env)) + args.out = run.parent / "invalid-stop-reason" + with pytest.raises(ValueError, match="stop_reason"): + cli.predict(args) + assert len(calls) == 1 and not args.out.exists() From 40bf742e2b84ad6157bf2990dd5e4bab73571702 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Thu, 8 Oct 2026 11:25:49 +0800 Subject: [PATCH 2/6] Fix benchmark evidence encoding across locales --- benchmarks/evidence.py | 4 ++- benchmarks/legs/leg1_public231.py | 6 ++-- tests/benchmarks/test_public231.py | 49 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/benchmarks/evidence.py b/benchmarks/evidence.py index 3940380..dff9de5 100644 --- a/benchmarks/evidence.py +++ b/benchmarks/evidence.py @@ -58,7 +58,9 @@ def load_reference(): ) items = [] for tier in ("easy", "original", "hard"): - for line, text in enumerate((root / f"datasets/public/{tier}.jsonl").read_text().split("\n")[:-1], 1): + for line, text in enumerate( + (root / f"datasets/public/{tier}.jsonl").read_text(encoding="utf-8").split("\n")[:-1], 1 + ): task = upstream.tasks.Task.from_dict(decode(text)) identity = dict( suite=f"jevbench-{tier}", diff --git a/benchmarks/legs/leg1_public231.py b/benchmarks/legs/leg1_public231.py index 7630fc1..97e88cc 100644 --- a/benchmarks/legs/leg1_public231.py +++ b/benchmarks/legs/leg1_public231.py @@ -84,7 +84,9 @@ def predict(args): write_json( directory / "predict.meta.json", dict(round=number, planned=args.warmup if number == 0 else 231) ) - with (directory / ("records.jsonl" if number == 0 else "results.jsonl")).open("x") as stream: + with (directory / ("records.jsonl" if number == 0 else "results.jsonl")).open( + "x", encoding="utf-8" + ) as stream: for identity, task in selected: raw = client.request(body_for(task, args.model)) raw_path = directory / "raw" / (digest(task.id.encode()) + ".json") @@ -163,7 +165,7 @@ def read_round(directory, expected, upstream, model, number): def aggregate(args): upstream, items, dataset = load_reference() run = outside_repo(args.run) - meta = decode((run / "run.meta.json").read_text()) + meta = decode((run / "run.meta.json").read_text(encoding="utf-8")) validate_environment(meta) environment = (run / "environment.json").read_bytes() if digest(environment) != meta["env_sha256"]: diff --git a/tests/benchmarks/test_public231.py b/tests/benchmarks/test_public231.py index 0f5720e..16da0e4 100644 --- a/tests/benchmarks/test_public231.py +++ b/tests/benchmarks/test_public231.py @@ -3,6 +3,8 @@ import copy import json import os +import subprocess +import sys from pathlib import Path from types import SimpleNamespace @@ -518,3 +520,50 @@ def test_environment_stop_reason_rejected_before_request(collect, stop_reason): with pytest.raises(ValueError, match="stop_reason"): cli.predict(args) assert len(calls) == 1 and not args.out.exists() + + +def test_cli_uses_utf8_independent_of_locale(reference, tmp_path): + script = r""" +import json +import sys +from pathlib import Path +import httpx +from benchmarks.client import DecisionClient +from benchmarks.evidence import write_json +from benchmarks.legs import leg1_public231 as cli + +root = Path(sys.argv[1]) +environment = root / "environment.json" +write_json(environment, dict( + checkpoint=dict(repo="fixture", revision="fixture"), + backend=dict(name="MockTransport", version=httpx.__version__), + sampling=dict(temperature=1), hardware="CPU \u6d4b\u8bd5", +)) +class Client(DecisionClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, transport=httpx.MockTransport( + lambda _: httpx.Response(422, text="refused\u2028request") + )) +cli.DecisionClient = Client +run = root / "run" +sys.argv = ["public231", "predict", "--endpoint", "http://fixture.invalid", + "--path", "/v1/decisions", "--run-meta", str(environment), + "--out", str(run), "--warmup", "0", "--rounds", "1", "--limit", "2"] +cli.main() +sys.argv = ["public231", "aggregate", "--run", str(run), "--out", str(root / "summary")] +cli.main() +with (run / "round-1/results.jsonl").open(encoding="utf-8") as stream: + records = [json.loads(line) for line in stream] +assert len(records) == 2 and all(row["error"] == "refused\u2028request" for row in records) +summary = json.loads((root / "summary/summary.json").read_text(encoding="utf-8")) +assert summary["pooled"]["attempted"] == 2 and summary["pooled"]["planned"] == 231 +""" + result = subprocess.run( + [sys.executable, "-c", script, str(tmp_path)], + env={**os.environ, "PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0", "LC_ALL": "C", "PYTHONIOENCODING": "utf-8"}, + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + ) + assert result.returncode == 0, result.stderr From 5c59dc4229c47d2deea31bd3fdb70ce844eaa3d5 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Sun, 11 Oct 2026 18:42:15 +0800 Subject: [PATCH 3/6] docs: describe benchmark transport tests accurately --- benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 19f462e..d1d02cd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -79,5 +79,5 @@ ruff check benchmarks tests/benchmarks ruff format --check benchmarks tests/benchmarks ``` -Reference-dependent tests skip if `JEVBENCH_ROOT` is absent. Mock and loopback tests +Reference-dependent tests skip if `JEVBENCH_ROOT` is absent. Mock tests check collection and scoring behavior; model quality and latency come from the linked real runs. From f51b82d37da6540738430dac7d8876755ccadddc Mon Sep 17 00:00:00 2001 From: linear3735 Date: Fri, 9 Oct 2026 10:34:30 +0800 Subject: [PATCH 4/6] Add changed-image benchmark replay --- benchmarks/README.md | 53 ++++ benchmarks/legs/changed_image_replay.py | 226 ++++++++++++++ tests/benchmarks/test_changed_image_replay.py | 285 ++++++++++++++++++ 3 files changed, 564 insertions(+) create mode 100644 benchmarks/legs/changed_image_replay.py create mode 100644 tests/benchmarks/test_changed_image_replay.py diff --git a/benchmarks/README.md b/benchmarks/README.md index d1d02cd..01ee729 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -71,6 +71,59 @@ consistency before re-scoring. Orphan or missing raw files are errors. It never an existing summary; use a new output directory for another recomputation. These checks detect inconsistent evidence, not coordinated rewriting of an unsigned archive. +## Changed-image replay + +Replay a frozen screenshot sequence against official JEV `/v1/decide` or native +Omni `/v1/systemone`, using the same `DecisionClient` without a model override: + +```bash +python -m benchmarks.legs.changed_image_replay \ + --manifest /absolute/path/to/sequence.jsonl \ + --endpoint http://127.0.0.1:8000 --path /v1/systemone \ + --run-meta /absolute/path/to/environment.json \ + --cache-state unknown --out /absolute/path/to/results/image-replay +``` + +Each JSONL line has a unique nonempty `id`, a nonempty `group`, and the original +`request` object. For example (replace the placeholder with actual base64 bytes): + +```json +{"id":"frame-1","group":"game-over","request":{"kind":"choice","state":["Inspect the screenshot.",{"image":"data:image/png;base64,..."}],"question":"Game over?","options":["No","Yes"]}} +``` + +`state` is a list with at least one inline PNG/JPEG image, either `{"image": URI}` +or `{"type":"image_url","image_url":{"url": URI}}`. Preflight checks every row, +base64 encoding and MIME signature before any request; image decoding stays with +the service. Remote URLs are rejected. `choice` requires 2–256 string options; +`noul` and `score` omit options. The original request, image order and option order +are preserved. Repeated requests with distinct IDs are sent again. There are no +warmups, retries, deduplication, service restarts or cache resets. + +Run metadata requires `checkpoint.repo/revision` and `backend.name/version` from +the actual service. Include source revisions, hardware and cache configuration +as available. `--cache-state` records only the caller's declaration: the caller +must arrange a cold service, and resetting counters does not empty its caches. +Keep any canonical-ID alias map alongside the external input evidence. + +The fresh output directory must be outside this repository. It retains the exact +`manifest.jsonl`, `environment.json`, `run.meta.json`, one `raw/NNNNNN.json` per +attempt and `results.jsonl`. Records include sequence/id/group/kind, hashes of +the request serialized by `benchmarks.evidence.dump`, image URIs in order, raw +evidence and original response text. Raw evidence retains request, HTTP status, +text, decoded response, client error, timestamp and latency. `completion.json` +counts planned/attempted/remaining/succeeded/failed and records the stop reason. +HTTP 400/422 refusals and malformed answers remain failed records; access errors, +other non-2xx responses, transport failures and interruptions stop the sequence. + +Response validation checks kind/effective_kind, ordered options, probability +count/range and choice/index consistency. `noul` uses `false,true`; `score` uses +`0` through `5`. This is structural validation, not reference agreement or task +quality, and does not add three-kind support to the Agents `DecisionModel` API. +Use the independent Omni harness for numerical reference comparison. Client +`latency_s` ends after `response.text` is read, before JSON decoding; it excludes +service startup. Short replays do not establish stable p95 latency, and replaying +2048 screenshots does not establish autonomous game completion. + ## Tests ```bash diff --git a/benchmarks/legs/changed_image_replay.py b/benchmarks/legs/changed_image_replay.py new file mode 100644 index 0000000..ef3de0e --- /dev/null +++ b/benchmarks/legs/changed_image_replay.py @@ -0,0 +1,226 @@ +"""Replay a frozen inline-image JSONL sequence and retain every HTTP attempt.""" + +import argparse +import base64 +from contextlib import closing +import json +import os +from pathlib import Path +import sys +import time + +from benchmarks.client import DecisionClient, decode +from benchmarks.evidence import digest, dump, outside_repo, write_json + + +def image_hashes(request): + state = request.get("state") + if not isinstance(state, list): + raise ValueError("state must be a list containing inline PNG/JPEG images") + hashes = [] + for part in state: + if not isinstance(part, dict): + continue + if "image" in part: + uri = part["image"] + elif part.get("type") == "image_url": + image = part.get("image_url") + uri = image.get("url") if isinstance(image, dict) else None + else: + continue + if not isinstance(uri, str): + raise ValueError("image URI must be a string") + header, separator, payload = uri.partition(",") + signatures = {"data:image/png;base64": b"\x89PNG\r\n\x1a\n", "data:image/jpeg;base64": b"\xff\xd8\xff"} + if not separator or header not in signatures: + raise ValueError("Only inline base64 PNG/JPEG images are allowed") + data = base64.b64decode(payload, validate=True) + if not data.startswith(signatures[header]): + raise ValueError("Image bytes do not match their PNG/JPEG MIME signature") + hashes.append(digest(uri.encode("utf-8"))) + if not hashes: + raise ValueError("Each request must contain at least one inline image") + return hashes + + +def load_manifest(source): + rows, seen = [], set() + for number, line in enumerate(source.splitlines(), 1): + try: + row = decode(line) + if not isinstance(row, dict) or any( + not isinstance(row.get(key), str) or not row[key] for key in ("id", "group") + ): + raise ValueError("Each row requires nonempty id and group strings") + if row["id"] in seen: + raise ValueError("Duplicate input id") + request = row.get("request") + if not isinstance(request, dict) or request.get("kind") not in ("choice", "noul", "score"): + raise ValueError("request.kind must be choice, noul or score") + if not isinstance(request.get("question"), str): + raise ValueError("request.question must be a string") + if request["kind"] == "choice": + options = request.get("options") + if ( + not isinstance(options, list) + or not 2 <= len(options) <= 256 + or any(not isinstance(option, str) for option in options) + ): + raise ValueError("choice requires 2–256 string options") + elif "options" in request: + raise ValueError("options is only allowed for choice") + hashes = image_hashes(request) + dump(row).encode("utf-8") + except ValueError as exc: + raise ValueError(f"Manifest line {number}: {exc}") from exc + seen.add(row["id"]) + rows.append((row, hashes)) + if not rows: + raise ValueError("Manifest is empty") + return rows + + +def response_error(request, response): + if not isinstance(response, dict): + return "Expected a JSON response object" + kind = request["kind"] + options = request.get("options", ["false", "true"] if kind == "noul" else list("012345")) + if response.get("kind") != kind or response.get("effective_kind") != kind: + return "Missing or mismatched kind/effective_kind" + if response.get("options") != options: + return "Response options differ from the requested kind or option order" + probabilities = response.get("probabilities") + if ( + not isinstance(probabilities, list) + or len(probabilities) != len(options) + or any(isinstance(p, bool) or not isinstance(p, (int, float)) or not 0 <= p <= 1 for p in probabilities) + ): + return "Expected one finite probability in [0, 1] per option" + index = response.get("choice_index") + if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(options): + return "choice_index is outside the options" + if response.get("choice") != options[index]: + return "choice differs from options[choice_index]" + return None + + +def record_for(row, hashes, sequence, raw, raw_path, output): + code = raw["http_status"] + if raw["client_error"] is not None: + status, error = "client_error", raw["client_error"] + elif code is None or not 200 <= code < 300: + status, error = ("refused" if code in (400, 422) else "http_error"), raw["response_body"] + else: + error = response_error(row["request"], raw["response"]) + status = "parse_error" if error else "ok" + text = raw["response_body"] + return dict( + sequence=sequence, + id=row["id"], + group=row["group"], + kind=row["request"]["kind"], + request_sha256=digest(dump(raw["request"]).encode("utf-8")), + image_uri_sha256=hashes, + response_body_sha256=digest(text.encode("utf-8")) if text is not None else None, + raw_file=str(raw_path.relative_to(output)), + raw_sha256=digest(raw_path.read_bytes()), + ts=raw["ts"], + latency_s=raw["latency_s"], + status_code=code, + status=status, + ok=status == "ok", + error=error, + ) + + +def predict(args): + source = args.manifest.read_bytes() + rows = load_manifest(source) + environment = args.run_meta.read_bytes() + meta = decode(environment) + for group, fields in (("checkpoint", ("repo", "revision")), ("backend", ("name", "version"))): + values = meta.get(group) if isinstance(meta, dict) else None + if not isinstance(values, dict) or any(not isinstance(values.get(k), str) or not values[k] for k in fields): + raise ValueError(f"run-meta requires actual {group}: {fields}") + if args.path not in ("/v1/decide", "/v1/systemone"): + raise ValueError("Choose /v1/decide or /v1/systemone explicitly") + if args.cache_state not in ("unknown", "cold", "warm"): + raise ValueError("cache-state must be unknown, cold or warm") + output = outside_repo(args.out) + with closing(DecisionClient(args.endpoint, path=args.path, model=None, timeout_s=args.timeout)) as client: + output.mkdir(parents=True, exist_ok=False) + (output / "raw").mkdir() + (output / "manifest.jsonl").write_bytes(source) + (output / "environment.json").write_bytes(environment) + write_json( + output / "run.meta.json", + dict( + argv=sys.argv, + endpoint=args.endpoint, + path=args.path, + environment=meta, + environment_file=str(args.run_meta.resolve()), + environment_sha256=digest(environment), + manifest_file=str(args.manifest.resolve()), + manifest_sha256=digest(source), + planned=len(rows), + timeout_s=args.timeout, + cache_state_declared=args.cache_state, + order="manifest sequential; no warmup, retries or deduplication", + latency_basis="client HTTP POST through response.text; excludes JSON decode and service startup", + ts=time.time(), + ), + ) + attempted, succeeded, stop = 0, 0, None + try: + with (output / "results.jsonl").open("x", encoding="utf-8") as stream: + for sequence, (row, hashes) in enumerate(rows, 1): + attempted += 1 + raw = client.request(row["request"]) + raw_path = output / "raw" / f"{sequence:06d}.json" + # Escaping preserves decoded lone surrogates from malformed service responses. + with raw_path.open("x", encoding="utf-8") as raw_stream: + raw_stream.write(json.dumps(raw, ensure_ascii=True, allow_nan=False) + "\n") + record = record_for(row, hashes, sequence, raw, raw_path, output) + stream.write(dump(record) + "\n") + stream.flush() + os.fsync(stream.fileno()) + succeeded += record["ok"] + if record["status"] in ("client_error", "http_error"): + stop = raw["client_error"] or f"HTTP {raw['http_status']}" + break + except KeyboardInterrupt: + stop = "interrupted" + except Exception as exc: + stop = f"{type(exc).__name__}: {exc}" + raise + finally: + write_json( + output / "completion.json", + dict( + planned=len(rows), + attempted=attempted, + remaining=len(rows) - attempted, + succeeded=succeeded, + failed=attempted - succeeded, + stop_reason=stop, + ts=time.time(), + ), + ) + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--endpoint", required=True) + parser.add_argument("--path", required=True, choices=("/v1/decide", "/v1/systemone")) + parser.add_argument("--run-meta", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=120) + parser.add_argument("--cache-state", choices=("unknown", "cold", "warm"), default="unknown") + print(predict(parser.parse_args())) + + +if __name__ == "__main__": + main() diff --git a/tests/benchmarks/test_changed_image_replay.py b/tests/benchmarks/test_changed_image_replay.py new file mode 100644 index 0000000..b639c8b --- /dev/null +++ b/tests/benchmarks/test_changed_image_replay.py @@ -0,0 +1,285 @@ +"""Offline replay contract checks; no model inference or external HTTP traffic.""" + +import copy +import json +from types import SimpleNamespace + +import httpx +import pytest + +from benchmarks.client import DecisionClient +from benchmarks.evidence import digest, dump +from benchmarks.legs import changed_image_replay as replay + + +PNG = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wl6nH0AAAAASUVORK5CYII=" +) +JPEG = "data:image/jpeg;base64,/9j/2Q==" + + +def item(identity="frame-1", kind="choice", image=PNG): + request = {"kind": kind, "state": ["Inspect this frame.", {"image": image}], "question": "Game over?"} + if kind == "choice": + request["options"] = ["No", "Yes"] + return {"id": identity, "group": "screenshots", "request": request} + + +def answer(request): + kind = request["kind"] + options = request.get("options", ["false", "true"] if kind == "noul" else list("012345")) + return { + "kind": kind, + "effective_kind": kind, + "options": options, + "probabilities": [1.0] + [0.0] * (len(options) - 1), + "choice_index": 0, + "choice": options[0], + "model": "CPU fixture, no inference", + } + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + calls = [] + rows = [item()] + manifest = tmp_path / "input.jsonl" + environment = tmp_path / "actual-environment.json" + environment.write_text( + dump({"checkpoint": {"repo": "fixture", "revision": "test"}, "backend": {"name": "mock", "version": "1"}}) + ) + args = SimpleNamespace( + manifest=manifest, + run_meta=environment, + endpoint="http://fixture.invalid", + path="/v1/decide", + timeout=1, + cache_state="unknown", + out=tmp_path / "output", + ) + + def configure(new_rows=None, handler=None): + if new_rows is not None: + rows[:] = new_rows + manifest.write_text("".join(dump(row) + "\n" for row in rows), encoding="utf-8") + + def respond(request): + calls.append(request) + return handler(request) if handler else httpx.Response(200, json=answer(json.loads(request.content))) + + class FixtureClient(DecisionClient): + def __init__(self, *positional, **kwargs): + assert kwargs.get("model") is None + super().__init__(*positional, **kwargs, transport=httpx.MockTransport(respond)) + + monkeypatch.setattr(replay, "DecisionClient", FixtureClient) + return args, calls + + return configure + + +def evidence(output): + rows = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + raw = [json.loads((output / row["raw_file"]).read_text()) for row in rows] + completion = json.loads((output / "completion.json").read_text()) + return rows, raw, completion + + +@pytest.mark.parametrize("path", ["/v1/decide", "/v1/systemone"]) +def test_serial_three_kinds_preserve_repeated_images_options_and_evidence(setup, path): + inputs = [item("first"), item("second", "noul", JPEG), item("third", "score"), item("reversed"), item("again")] + inputs[3]["request"]["options"].reverse() + inputs[2]["request"]["state"][1] = {"type": "image_url", "image_url": {"url": PNG}} + before = copy.deepcopy(inputs) + args, calls = setup(inputs) + args.path, args.cache_state = path, "cold" + output = replay.predict(args) + records, raw, complete = evidence(output) + assert inputs == before + assert [json.loads(call.content) for call in calls] == [row["request"] for row in inputs] + assert all(str(call.url) == args.endpoint + path for call in calls) + assert [row["sequence"] for row in records] == [1, 2, 3, 4, 5] + assert [row["id"] for row in records] == [row["id"] for row in inputs] + assert [row["kind"] for row in records] == ["choice", "noul", "score", "choice", "choice"] + assert all(row["ok"] and row["status"] == "ok" for row in records) + assert all(row["latency_s"] > 0 for row in records) + for row, original, response in zip(records, inputs, raw): + assert row["request_sha256"] == digest(dump(original["request"]).encode()) + assert row["raw_sha256"] == digest((output / row["raw_file"]).read_bytes()) + assert row["response_body_sha256"] == digest(response["response_body"].encode()) + assert response["request"] == original["request"] + assert json.loads(response["response_body"]) == response["response"] + assert records[0]["image_uri_sha256"] == records[2]["image_uri_sha256"] == records[3]["image_uri_sha256"] + assert (output / "manifest.jsonl").read_bytes() == args.manifest.read_bytes() + assert (output / "environment.json").read_bytes() == args.run_meta.read_bytes() + meta = json.loads((output / "run.meta.json").read_text()) + assert meta["cache_state_declared"] == "cold" + assert meta["manifest_sha256"] == digest(args.manifest.read_bytes()) + assert meta["environment_sha256"] == digest(args.run_meta.read_bytes()) + assert complete["planned"] == complete["attempted"] == complete["succeeded"] == 5 + assert complete["remaining"] == complete["failed"] == 0 and complete["stop_reason"] is None + + +def test_multiple_images_retain_uri_hash_order(setup): + row = item() + row["request"]["state"].extend([{"image": JPEG}, {"image": PNG}]) + args, calls = setup([row]) + records, _, _ = evidence(replay.predict(args)) + assert json.loads(calls[0].content) == row["request"] + assert records[0]["image_uri_sha256"] == [digest(uri.encode()) for uri in (PNG, JPEG, PNG)] + + +@pytest.mark.parametrize( + "broken", + [ + {}, + {"id": "later", "group": "x", "request": []}, + item("frame-1"), + item("later", "other"), + item("later", image="https://example.invalid/image.png"), + item("later", image="data:image/png;base64,not-base64"), + item("later", image="data:image/png;base64,dGV4dA=="), + item("later", image="data:image/gif;base64,R0lGODlh"), + ], +) +def test_bad_later_input_prevents_every_request(setup, broken): + args, calls = setup([item(), broken]) + with pytest.raises(ValueError): + replay.predict(args) + assert not calls and not args.out.exists() + + +@pytest.mark.parametrize("change", ["options", "state", "question", "noul_options"]) +def test_bad_request_fields_preflight(setup, change): + row = item() + if change == "options": + row["request"]["options"] = ["Only one"] + elif change == "state": + row["request"]["state"] = ["No screenshot"] + elif change == "question": + row["request"]["question"] = 3 + else: + row["request"]["kind"] = "noul" + args, calls = setup([row]) + with pytest.raises(ValueError): + replay.predict(args) + assert not calls and not args.out.exists() + + +@pytest.mark.parametrize("source", ["", "not json\n", "{}\n\n"]) +def test_empty_or_invalid_manifest_preflight(setup, source): + args, calls = setup() + args.manifest.write_text(source) + with pytest.raises(ValueError): + replay.predict(args) + assert not calls and not args.out.exists() + + +def test_invalid_unicode_in_later_request_is_rejected_before_http(setup): + args, calls = setup() + row = item("later") + row["request"]["question"] = "\ud800" + args.manifest.write_text(json.dumps(item()) + "\n" + json.dumps(row) + "\n") + with pytest.raises(ValueError): + replay.predict(args) + assert not calls and not args.out.exists() + + +@pytest.mark.parametrize("status,attempted", [(400, 2), (422, 2), (401, 1), (403, 1), (429, 1), (500, 1)]) +def test_http_rejection_and_access_errors_keep_raw_text(setup, status, attempted): + text = ' {"error":{"message":"rejected"}}\n' + args, calls = setup([item(), item("next")], lambda _: httpx.Response(status, text=text)) + output = replay.predict(args) + records, raw, complete = evidence(output) + assert len(calls) == complete["attempted"] == complete["failed"] == attempted + assert complete["remaining"] == 2 - attempted + assert bool(complete["stop_reason"]) is (attempted == 1) + assert all(r["response_body"] == text and r["http_status"] == status for r in raw) + assert all(not r["ok"] and r["error"] for r in records) + + +@pytest.mark.parametrize("error", [httpx.ReadTimeout("fixture timeout"), KeyboardInterrupt()]) +def test_transport_failure_and_interruption_save_completion(setup, error): + def handler(_): + raise error + + args, calls = setup([item(), item("next")], handler) + records, raw, complete = evidence(replay.predict(args)) + assert len(calls) == complete["attempted"] == 1 and complete["remaining"] == 1 + assert raw[0]["client_error"] and raw[0]["response_body"] is None + assert records[0]["status"] == "client_error" and records[0]["latency_s"] > 0 + assert complete["stop_reason"] + + +@pytest.mark.parametrize( + "patch", + [ + {"kind": "noul"}, + {"effective_kind": "noul"}, + {"options": ["Yes", "No"]}, + {"probabilities": [1]}, + {"probabilities": [True, False]}, + {"probabilities": [-1, 2]}, + {"choice_index": True}, + {"choice_index": 2}, + {"choice": "missing"}, + ], +) +def test_malformed_response_structure_is_retained(setup, patch): + body = answer(item()["request"]) | patch + args, _ = setup(handler=lambda _: httpx.Response(200, json=body)) + records, raw, complete = evidence(replay.predict(args)) + assert records[0]["status"] == "parse_error" and not records[0]["ok"] + assert raw[0]["response"] == body and complete["failed"] == 1 + + +@pytest.mark.parametrize("text", ["broken json", '{"probabilities":[NaN]}', "[]"]) +def test_unparseable_response_keeps_original_text(setup, text): + args, _ = setup(handler=lambda _: httpx.Response(200, text=text)) + records, raw, _ = evidence(replay.predict(args)) + assert records[0]["status"] == "parse_error" and raw[0]["response_body"] == text + + +@pytest.mark.parametrize("status", [200, 400, 422]) +def test_escaped_surrogate_response_is_retained_and_sequence_continues(setup, status): + text = '{"error":"\\ud800"}' + + def handler(request): + if len(calls) == 1: + return httpx.Response(status, text=text) + return httpx.Response(200, json=answer(json.loads(request.content))) + + args, calls = setup([item(), item("next")], handler) + records, raw, complete = evidence(replay.predict(args)) + assert len(calls) == complete["attempted"] == 2 and complete["remaining"] == 0 + assert complete["failed"] == complete["succeeded"] == 1 and complete["stop_reason"] is None + assert records[0]["status"] == ("parse_error" if status == 200 else "refused") + assert records[0]["latency_s"] > 0 and not records[0]["ok"] and records[1]["ok"] + assert raw[0]["http_status"] == status and raw[0]["response_body"] == text + assert raw[0]["response"] == {"error": "\ud800"} + assert records[0]["response_body_sha256"] == digest(text.encode()) + assert records[0]["raw_sha256"] == digest((args.out / records[0]["raw_file"]).read_bytes()) + + +def test_structure_check_does_not_compare_reference_values(setup): + body = answer(item()["request"]) | {"probabilities": [0.2, 0.2]} + args, _ = setup(handler=lambda _: httpx.Response(200, json=body)) + records, raw, _ = evidence(replay.predict(args)) + assert records[0]["ok"] and raw[0]["response"] == body + + +def test_output_never_overwrites_existing_evidence(setup): + args, calls = setup() + replay.predict(args) + before = (args.out / "completion.json").read_bytes() + with pytest.raises(FileExistsError): + replay.predict(args) + assert len(calls) == 1 and (args.out / "completion.json").read_bytes() == before + + +def test_output_must_be_outside_repository(setup): + args, calls = setup() + args.out = replay.Path(replay.__file__).resolve().parents[2] / "rejected-evidence" + with pytest.raises(ValueError, match="outside"): + replay.predict(args) + assert not calls and not args.out.exists() From f09748b967792d52014abb3b0c3b1acbd0ba069c Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 17:10:15 +0800 Subject: [PATCH 5/6] [Fix] Validate replay probabilities and require all benchmark tests --- .github/workflows/test.yml | 2 +- benchmarks/README.md | 51 ++++++++++++------- benchmarks/legs/changed_image_replay.py | 3 ++ tests/benchmarks/test_changed_image_replay.py | 21 +++++++- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 872a15a..7702647 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,7 +129,7 @@ jobs: - run: uv sync --extra dev - name: Required benchmark tests run: | - uv run pytest tests/benchmarks/test_public231.py -q --junitxml="${RUNNER_TEMP}/benchmark-tests.xml" + uv run pytest tests/benchmarks -q --junitxml="${RUNNER_TEMP}/benchmark-tests.xml" uv run python - <<'PY' import os import xml.etree.ElementTree as ET diff --git a/benchmarks/README.md b/benchmarks/README.md index 01ee729..254f5a7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -26,9 +26,13 @@ python -m benchmarks.legs.leg1_public231 aggregate \ --out /absolute/path/to/results/open-jev-9b ``` -Omni native Open-Jev uses `/v1/systemone`; Intern's official service uses -`/v1/decisions`. `--model` is optional. The collector preserves state, questions and -option order, sends `thinking.enabled=false`, and never sends targets or provenance. +The public231 collector uses the typed `state` / `questions` / `thinking` request +and reads `answers.decision`. Omni native text Open-Jev exposes this contract at +`/v1/systemone`; Intern's official service uses `/v1/decisions`. Omni's request +and response schema depends on the selected worker: native JEV-27B-VL instead +uses the flat contract described below at the same `/v1/systemone` path. +`--model` is optional. The collector preserves state, questions and option order, +sends `thinking.enabled=false`, and never sends targets or provenance. It makes one request at a time, without retries. It does not start services or choose GPUs. `environment.json` describes the **actual service**, without credentials: @@ -73,8 +77,10 @@ detect inconsistent evidence, not coordinated rewriting of an unsigned archive. ## Changed-image replay -Replay a frozen screenshot sequence against official JEV `/v1/decide` or native -Omni `/v1/systemone`, using the same `DecisionClient` without a model override: +Replay a frozen screenshot sequence using the flat `kind` / `state` / `question` / +`options` contract of official JEV `/v1/decide` or Omni native JEV-27B-VL +`/v1/systemone`, using the same `DecisionClient` without a model override. This +collector does not send the public231 typed contract to Intern `/v1/decisions`: ```bash python -m benchmarks.legs.changed_image_replay \ @@ -107,22 +113,33 @@ Keep any canonical-ID alias map alongside the external input evidence. The fresh output directory must be outside this repository. It retains the exact `manifest.jsonl`, `environment.json`, `run.meta.json`, one `raw/NNNNNN.json` per -attempt and `results.jsonl`. Records include sequence/id/group/kind, hashes of -the request serialized by `benchmarks.evidence.dump`, image URIs in order, raw -evidence and original response text. Raw evidence retains request, HTTP status, -text, decoded response, client error, timestamp and latency. `completion.json` -counts planned/attempted/remaining/succeeded/failed and records the stop reason. +attempt and `results.jsonl`. Result records include sequence/id/group/kind, +`request_sha256` (of the request serialized by `benchmarks.evidence.dump`), ordered +`image_uri_sha256`, `response_body_sha256`, and the raw file's path and hash. +The full request, image URIs and original response text are in raw evidence, +together with HTTP status, decoded response, client error, timestamp and latency. +`completion.json` counts planned/attempted/remaining/succeeded/failed and records +the stop reason. HTTP 400/422 refusals and malformed answers remain failed records; access errors, other non-2xx responses, transport failures and interruptions stop the sequence. Response validation checks kind/effective_kind, ordered options, probability -count/range and choice/index consistency. `noul` uses `false,true`; `score` uses -`0` through `5`. This is structural validation, not reference agreement or task -quality, and does not add three-kind support to the Agents `DecisionModel` API. -Use the independent Omni harness for numerical reference comparison. Client -`latency_s` ends after `response.text` is read, before JSON decoding; it excludes -service startup. Short replays do not establish stable p95 latency, and replaying -2048 screenshots does not establish autonomous game completion. +count/range, a probability sum within `1e-6` of 1, and choice/index consistency. +The sum tolerance only allows floating-point rounding, including float32 output; +it is not a reference-comparison tolerance. Unnormalized answers remain +`parse_error` records with their raw evidence and count as failures. `noul` uses +`false,true`; `score` uses `0` through `5`. `ok` and `succeeded` mean only a valid +response; they do not establish reference agreement, cache correctness or task +completion, and do not add three-kind support to the Agents `DecisionModel` API. +Use the independent Omni harness for numerical reference comparison. Comparing +repeated images requires the complete request to match, including all state text, +image order, question, options and inference settings. Compare the cache path +with the reference or cache-disabled path under the same checkpoint and runtime; +require deterministic inference settings or account for expected nondeterminism +before attributing differences to caching. Matching image URI hashes alone is +insufficient. Client `latency_s` ends after `response.text` is read, before JSON +decoding; it excludes service startup. Short replays do not establish stable p95 +latency, and replaying 2048 screenshots does not establish autonomous game completion. ## Tests diff --git a/benchmarks/legs/changed_image_replay.py b/benchmarks/legs/changed_image_replay.py index ef3de0e..2c84244 100644 --- a/benchmarks/legs/changed_image_replay.py +++ b/benchmarks/legs/changed_image_replay.py @@ -4,6 +4,7 @@ import base64 from contextlib import closing import json +import math import os from pathlib import Path import sys @@ -96,6 +97,8 @@ def response_error(request, response): or any(isinstance(p, bool) or not isinstance(p, (int, float)) or not 0 <= p <= 1 for p in probabilities) ): return "Expected one finite probability in [0, 1] per option" + if not math.isclose(math.fsum(probabilities), 1.0, rel_tol=0, abs_tol=1e-6): + return "Expected probabilities to sum to 1" index = response.get("choice_index") if isinstance(index, bool) or not isinstance(index, int) or not 0 <= index < len(options): return "choice_index is outside the options" diff --git a/tests/benchmarks/test_changed_image_replay.py b/tests/benchmarks/test_changed_image_replay.py index b639c8b..fc04e1e 100644 --- a/tests/benchmarks/test_changed_image_replay.py +++ b/tests/benchmarks/test_changed_image_replay.py @@ -261,13 +261,30 @@ def handler(request): assert records[0]["raw_sha256"] == digest((args.out / records[0]["raw_file"]).read_bytes()) -def test_structure_check_does_not_compare_reference_values(setup): - body = answer(item()["request"]) | {"probabilities": [0.2, 0.2]} +@pytest.mark.parametrize("probabilities", [[0.2, 0.8], [0.20000000298023224, 0.800000011920929]]) +def test_structure_check_does_not_compare_reference_values(setup, probabilities): + body = answer(item()["request"]) | {"probabilities": probabilities} args, _ = setup(handler=lambda _: httpx.Response(200, json=body)) records, raw, _ = evidence(replay.predict(args)) assert records[0]["ok"] and raw[0]["response"] == body +def test_unnormalized_probabilities_are_failed_and_sequence_continues(setup): + body = answer(item()["request"]) | {"probabilities": [0.2, 0.2]} + + def handler(request): + response = body if len(calls) == 1 else answer(json.loads(request.content)) + return httpx.Response(200, json=response) + + args, calls = setup([item(), item("next")], handler) + records, raw, complete = evidence(replay.predict(args)) + assert len(calls) == complete["attempted"] == 2 and complete["remaining"] == 0 + assert records[0]["status"] == "parse_error" and not records[0]["ok"] + assert "sum to 1" in records[0]["error"] and records[1]["ok"] + assert raw[0]["response"] == body and json.loads(raw[0]["response_body"]) == body + assert complete["failed"] == complete["succeeded"] == 1 and complete["stop_reason"] is None + + def test_output_never_overwrites_existing_evidence(setup): args, calls = setup() replay.predict(args) From da2bc9355e0bb4c94851c3e89c57c4e94a8a5e55 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 19:31:01 +0800 Subject: [PATCH 6/6] [Refactor] Trim redundant replay metadata and assertion --- benchmarks/legs/changed_image_replay.py | 5 ----- tests/benchmarks/test_changed_image_replay.py | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/benchmarks/legs/changed_image_replay.py b/benchmarks/legs/changed_image_replay.py index 2c84244..8889f2b 100644 --- a/benchmarks/legs/changed_image_replay.py +++ b/benchmarks/legs/changed_image_replay.py @@ -7,7 +7,6 @@ import math import os from pathlib import Path -import sys import time from benchmarks.client import DecisionClient, decode @@ -158,13 +157,9 @@ def predict(args): write_json( output / "run.meta.json", dict( - argv=sys.argv, endpoint=args.endpoint, path=args.path, - environment=meta, - environment_file=str(args.run_meta.resolve()), environment_sha256=digest(environment), - manifest_file=str(args.manifest.resolve()), manifest_sha256=digest(source), planned=len(rows), timeout_s=args.timeout, diff --git a/tests/benchmarks/test_changed_image_replay.py b/tests/benchmarks/test_changed_image_replay.py index fc04e1e..1b1dd74 100644 --- a/tests/benchmarks/test_changed_image_replay.py +++ b/tests/benchmarks/test_changed_image_replay.py @@ -1,6 +1,5 @@ """Offline replay contract checks; no model inference or external HTTP traffic.""" -import copy import json from types import SimpleNamespace @@ -90,12 +89,10 @@ def test_serial_three_kinds_preserve_repeated_images_options_and_evidence(setup, inputs = [item("first"), item("second", "noul", JPEG), item("third", "score"), item("reversed"), item("again")] inputs[3]["request"]["options"].reverse() inputs[2]["request"]["state"][1] = {"type": "image_url", "image_url": {"url": PNG}} - before = copy.deepcopy(inputs) args, calls = setup(inputs) args.path, args.cache_state = path, "cold" output = replay.predict(args) records, raw, complete = evidence(output) - assert inputs == before assert [json.loads(call.content) for call in calls] == [row["request"] for row in inputs] assert all(str(call.url) == args.endpoint + path for call in calls) assert [row["sequence"] for row in records] == [1, 2, 3, 4, 5] @@ -113,6 +110,7 @@ def test_serial_three_kinds_preserve_repeated_images_options_and_evidence(setup, assert (output / "manifest.jsonl").read_bytes() == args.manifest.read_bytes() assert (output / "environment.json").read_bytes() == args.run_meta.read_bytes() meta = json.loads((output / "run.meta.json").read_text()) + assert not {"argv", "environment", "environment_file", "manifest_file"} & meta.keys() assert meta["cache_state_declared"] == "cold" assert meta["manifest_sha256"] == digest(args.manifest.read_bytes()) assert meta["environment_sha256"] == digest(args.run_meta.read_bytes())