From b72985493044e465e7f917265207600a786f5d97 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Sat, 10 Oct 2026 12:54:24 -0400 Subject: [PATCH 1/3] feat(bench): add parity capture and compare tooling for the v0.6.0 gates parity.py capture ingests the pinned COCO set into a fresh project, trains clusters once and stores per-stage outputs (detector boxes, crop rectangles, PE crop and whole-frame vectors, cluster fields, labels, optional core-route and VLM probes) as capture.json plus a float16 NPZ with the environment, engine and manifest record. parity.py compare measures a candidate against a reference (box matching, cosine, kNN overlap, cluster agreement and purity against COCO ground truth, label agreement), applies the thresholds of parity_gates.json and exits non-zero on failure (#208). --- scripts/bench/parity.py | 278 +++++++++++++++++++ scripts/bench/parity_assemble.py | 195 +++++++++++++ scripts/bench/parity_capture.py | 228 ++++++++++++++++ scripts/bench/parity_env.py | 136 +++++++++ scripts/bench/parity_gates.json | 164 +++++++++++ scripts/bench/parity_gates.py | 297 ++++++++++++++++++++ scripts/bench/parity_metrics.py | 410 ++++++++++++++++++++++++++++ scripts/bench/parity_store.py | 157 +++++++++++ tests/bench/conftest.py | 102 +++++++ tests/bench/test_parity_assemble.py | 135 +++++++++ tests/bench/test_parity_capture.py | 355 ++++++++++++++++++++++++ tests/bench/test_parity_cli.py | 159 +++++++++++ tests/bench/test_parity_gates.py | 221 +++++++++++++++ tests/bench/test_parity_metrics.py | 289 ++++++++++++++++++++ tests/bench/test_parity_store.py | 92 +++++++ 15 files changed, 3218 insertions(+) create mode 100644 scripts/bench/parity.py create mode 100644 scripts/bench/parity_assemble.py create mode 100644 scripts/bench/parity_capture.py create mode 100644 scripts/bench/parity_env.py create mode 100644 scripts/bench/parity_gates.json create mode 100644 scripts/bench/parity_gates.py create mode 100644 scripts/bench/parity_metrics.py create mode 100644 scripts/bench/parity_store.py create mode 100644 tests/bench/conftest.py create mode 100644 tests/bench/test_parity_assemble.py create mode 100644 tests/bench/test_parity_capture.py create mode 100644 tests/bench/test_parity_cli.py create mode 100644 tests/bench/test_parity_gates.py create mode 100644 tests/bench/test_parity_metrics.py create mode 100644 tests/bench/test_parity_store.py diff --git a/scripts/bench/parity.py b/scripts/bench/parity.py new file mode 100644 index 00000000..73a476cb --- /dev/null +++ b/scripts/bench/parity.py @@ -0,0 +1,278 @@ +"""Parity capture and compare: the gate every v0.6.0 wave passes before it merges. + + parity.py capture --tag fp32 --images DIR --api-url URL --triton-url URL \\ + --opensearch-url URL --compose-project NAME --engine-label fp32 \\ + [--engine-file PLAN] [--annotations COCO_DIR] [--probe-images 200] [--vlm-crops N \\ + --vlm-classes a,b,c] + parity.py compare REF_DIR CAND_DIR [--profile NAME] [--gates FILE] [--out DIR] [--strict] + +``capture`` ingests the pinned dataset into a fresh, project-scoped ingest, trains clusters once, +and stores every stage output (detector boxes, crop rectangles, PE crop and whole-frame vectors, +cluster fields, labels, optional MobileCLIP / face / OCR probes and VLM labels) under +``artifacts_local/bench/parity//`` as ``capture.json`` plus a float16 ``arrays.npz``. +``compare`` measures a candidate capture against a reference and applies the thresholds of +``parity_gates.json``; it exits 0 on PASS, 1 on any FAIL (or any SKIP with ``--strict``), 2 when +the inputs cannot be compared. Large capture files stay out of git; commit only the summary. + +Preconditions of ``capture``: a stack whose ingest embedding policy is ``all``, with the +auto-label worker running (compose profile ``curation``; cluster training is its job), and the +pinned images on the local disk. The curation API serves no embedding vectors, so the PE vectors +are read by document id from the project's OpenSearch indexes through ``--opensearch-url``. +Detector output varies slightly between identical runs (about 1 percent of crop rectangles +differ, mostly by a pixel), so capture the reference configuration twice and ``compare`` those two as the noise +floor before reading a candidate's numbers. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import httpx + + +if TYPE_CHECKING: + from collections.abc import Sequence + +_REPO_ROOT = Path(__file__).resolve().parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +# ruff: noqa: E402 +from scripts.bench import parity_capture as cap, parity_gates as gates, parity_store as store +from scripts.bench.parity_assemble import build_arrays, load_gt +from scripts.bench.parity_env import engine_record, environment_record +from scripts.curation._project_worker_utils import curation_api_prefix, scoped_url +from scripts.datasets.bench_set import load_pin + + +DEFAULT_MANIFEST = _REPO_ROOT / 'scripts/datasets/manifests/coco_bench_2000.json' +DEFAULT_OUT = Path('artifacts_local/bench/parity') +DEFAULT_GATES = Path(__file__).with_name('parity_gates.json') + + +def _slug(tag: str) -> str: + return f'par-{re.sub(r"[^a-z0-9]+", "-", tag.lower()).strip("-")}-{datetime.now(UTC):%H%M%S}' + + +def run_capture( + args: argparse.Namespace, *, transport: httpx.BaseTransport | None = None +) -> store.Capture: + """Ingest the pinned set into a fresh project and collect every stage output.""" + pin = load_pin(args.manifest) + rows = pin['images'][: args.limit or None] + files = [args.images / r['file_name'] for r in rows] + missing = [f.name for f in files if not f.is_file()] + if missing: + raise store.CaptureError(f'{len(missing)} pinned images are missing, first: {missing[0]}') + dims = {r['file_name']: (r['width'], r['height']) for r in rows} + slug = args.project or _slug(args.tag) + base = scoped_url(args.api_url, curation_api_prefix(), slug, '') + sections = ['items', 'frames', 'clusters'] + timings: dict[str, float] = {} + + def lap(name: str, since: float) -> None: + timings[name] = time.monotonic() - since + print(f'{name}: {timings[name]:.0f}s', flush=True) + + with httpx.Client(timeout=httpx.Timeout(3600.0), transport=transport) as client: + client.post( + f'{args.api_url}{curation_api_prefix()}/projects', + json={'slug': slug, 'display_name': slug}, + ).raise_for_status() + try: + environment = environment_record( + client, + gpu=args.gpu, + compose_project=args.compose_project, + api_url=args.api_url, + triton_url=args.triton_url, + opensearch_url=args.opensearch_url, + manifest=args.manifest, + ) + policy = cap.require_embed_all(client, base) + started = time.monotonic() + image_names = cap.ingest_images( + client, base, files, batch_size=args.batch_size, concurrency=args.concurrency + ) + lap('ingest', started) + started = time.monotonic() + cluster_job = cap.run_clustering(client, base, args.cluster_timeout) + lap('cluster', started) + items = cap.fetch_items(client, base) + items_index, images_index = cap.project_indexes(client, base) + os_url = args.opensearch_url.rstrip('/') + vectors = cap.mget_vectors(client, os_url, items_index, [i['id'] for i in items]) + frames = cap.mget_vectors(client, os_url, images_index, list(image_names)) + vlm: dict[str, str] = {} + if args.vlm_crops: + sections.append('vlm') + ids = cap.vlm_sample(items, image_names, args.vlm_crops) + vlm = cap.label_with_vlm( + client, base, ids, args.vlm_classes.split(','), args.batch_size + ) + probes: dict[str, list[tuple[Any, ...]]] = {} + if args.probe_images: + sections += ['clip', 'faces', 'ocr'] + probes = cap.probe_core( + client, args.api_url, files[: args.probe_images], concurrency=args.concurrency + ) + finally: + if not args.keep_project: + client.delete(base, params={'confirm': slug}).raise_for_status() + + gt = load_gt(args.annotations, pin) if args.annotations else None + arrays = build_arrays( + items, + image_names=image_names, + image_dims=dims, + vectors=vectors, + frame_vectors=frames, + gt=gt, + vlm=vlm, + probes=probes, + ) + meta = { + 'tag': args.tag, + 'created_at': datetime.now(UTC).isoformat(), + 'project': slug, + 'project_kept': args.keep_project, + 'manifest': { + 'name': args.manifest.name, + 'sha256': pin['manifest_sha256'], + 'count': len(rows), + 'seed': pin['seed'], + 'annotations_sha256': pin['annotations_sha256'], + }, + 'engine': engine_record(args.engine_label, args.engine_file, args.engine_record), + 'environment': environment, + 'sections': sections, + 'settings': { + 'ingest_embedding_policy': policy, + 'batch_size': args.batch_size, + 'concurrency': args.concurrency, + 'probe_images': args.probe_images, + 'vlm_crops': args.vlm_crops, + 'ground_truth': bool(gt), + }, + 'cluster_job': cluster_job, + 'phase_seconds': timings, + } + return store.Capture(meta, arrays) + + +def cmd_capture(args: argparse.Namespace) -> int: + out = args.out / args.tag + if out.exists() and any(out.iterdir()): + print(f'{out} already holds a capture; choose another --tag or remove it', file=sys.stderr) + return 2 + capture = run_capture(args) + store.save_capture(out, capture) + counts = json.loads((out / store.META_FILE).read_text(encoding='utf-8'))['counts'] + print(f'captured {counts} into {out}') + return 0 + + +def cmd_compare(args: argparse.Namespace) -> int: + ref, cand = store.load_capture(args.ref), store.load_capture(args.cand) + spec = gates.load_gates(args.gates) + metrics = gates.compute_metrics(ref, cand, spec['settings']) + result = gates.evaluate(metrics, spec, args.profile) + text = gates.render_markdown(ref.meta, cand.meta, metrics, result) + print(text) + if args.out: + args.out.mkdir(parents=True, exist_ok=True) + (args.out / 'parity_summary.md').write_text(text, encoding='utf-8') + summary = { + 'schema': 1, + 'reference': _brief(ref.meta), + 'candidate': _brief(cand.meta), + 'metrics': metrics, + 'evaluation': result, + } + (args.out / 'parity_summary.json').write_text( + json.dumps(summary, indent=2, sort_keys=True, default=float) + '\n', encoding='utf-8' + ) + return gates.exit_code(result, strict=args.strict) + + +def _brief(meta: dict[str, Any]) -> dict[str, Any]: + env = meta['environment'] + return { + 'tag': meta['tag'], + 'created_at': meta['created_at'], + 'engine': meta['engine'], + 'manifest': meta['manifest'], + 'git_sha': env.get('git_sha'), + 'git_dirty': env.get('git_dirty'), + 'gpu': env.get('hardware', {}).get('gpu', {}).get('model'), + 'image_digests': env.get('image_digests'), + 'sections': meta['sections'], + } + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + sub = p.add_subparsers(dest='command', required=True) + c = sub.add_parser('capture', help='ingest the pinned set and store every stage output') + c.add_argument('--tag', required=True, help='capture name; the output directory') + c.add_argument('--manifest', type=Path, default=DEFAULT_MANIFEST) + c.add_argument('--images', type=Path, required=True, help='directory with the pinned images') + c.add_argument('--api-url', required=True) + c.add_argument('--triton-url', required=True) + c.add_argument('--opensearch-url', required=True) + c.add_argument('--compose-project', required=True, help='compose project of the stack') + c.add_argument('--gpu', default='0') + c.add_argument('--out', type=Path, default=DEFAULT_OUT) + c.add_argument('--engine-label', required=True, help='precision or build label, e.g. fp16') + c.add_argument('--engine-file', type=Path, help='the PE plan file: size and sha256 recorded') + c.add_argument('--engine-record', action='append', default=[], metavar='KEY=VALUE') + c.add_argument('--annotations', type=Path, help='COCO annotation directory (purity gate)') + c.add_argument('--limit', type=int, default=0, help='first N pinned images (smoke runs)') + c.add_argument('--batch-size', type=int, default=32) + c.add_argument('--concurrency', type=int, default=4, help='parallel upload requests') + c.add_argument('--cluster-timeout', type=float, default=3600.0, help='seconds') + c.add_argument('--probe-images', type=int, default=200, help='core-route sample; 0 skips') + c.add_argument('--vlm-crops', type=int, default=0, help='label about N crops with the VLM') + c.add_argument('--vlm-classes', default='', help='comma list of class names for the VLM') + c.add_argument('--project', help='project slug (default: generated)') + c.add_argument('--keep-project', action='store_true') + c.set_defaults(func=cmd_capture) + d = sub.add_parser('compare', help='gate a candidate capture against a reference') + d.add_argument('ref', type=Path) + d.add_argument('cand', type=Path) + d.add_argument('--gates', type=Path, default=DEFAULT_GATES) + d.add_argument('--profile', default='precision') + d.add_argument('--out', type=Path, help='write parity_summary.json and .md here') + d.add_argument('--strict', action='store_true', help='a SKIP counts as a failure') + d.set_defaults(func=cmd_compare) + return p + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == 'capture' and args.vlm_crops and not args.vlm_classes: + print('--vlm-crops needs --vlm-classes', file=sys.stderr) + return 2 + try: + return int(args.func(args)) + except (store.CaptureError, FileNotFoundError, ValueError) as exc: + print(f'parity: {exc}', file=sys.stderr) + return 2 + except httpx.HTTPStatusError as exc: + body = exc.response.text[:500] + print( + f'parity: {exc.request.method} {exc.request.url} -> {exc.response.status_code}: {body}', + file=sys.stderr, + ) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/bench/parity_assemble.py b/scripts/bench/parity_assemble.py new file mode 100644 index 00000000..256a8db7 --- /dev/null +++ b/scripts/bench/parity_assemble.py @@ -0,0 +1,195 @@ +"""Turn the rows a capture collected from the stack into the fixed-schema arrays. + +Pure functions over plain Python rows and numpy: ground-truth class assignment from COCO +annotations, the integer crop rectangle, and the sorted, dtype-checked arrays of +``parity_store``. The HTTP side lives in ``parity_capture.py``. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import numpy as np + +from scripts.bench import parity_metrics as pm, parity_store as ps + + +if TYPE_CHECKING: + from pathlib import Path + +GT_MIN_IOU = 0.5 +GT_NONE = 'none' +GroundTruth = dict[str, tuple[np.ndarray, list[str]]] + + +def crop_rect(box_norm: list[float], width: int, height: int) -> list[int]: + """The integer pixel rectangle a crop is cut with (the ingest crop rule: rounded, >= 1 px).""" + x1 = max(0, round(box_norm[0] * width)) + y1 = max(0, round(box_norm[1] * height)) + return [ + x1, + y1, + max(x1 + 1, round(box_norm[2] * width)), + max(y1 + 1, round(box_norm[3] * height)), + ] + + +def _annotation_file(root: Path, split: str) -> Path: + for candidate in ( + root / f'instances_{split}.json', + root / 'annotations' / f'instances_{split}.json', + ): + if candidate.is_file(): + return candidate + raise FileNotFoundError(f'instances_{split}.json not found under {root}') + + +def load_gt(annotations_dir: Path, pin: dict[str, Any]) -> GroundTruth: + """``{file_name: (boxes_norm (n, 4), class names)}`` for the pinned images, crowd boxes dropped.""" + rows = {(r['split'], r['image_id']): r for r in pin['images']} + out: GroundTruth = {r['file_name']: (np.zeros((0, 4), np.float32), []) for r in pin['images']} + for split in sorted({r['split'] for r in pin['images']}): + doc = json.loads(_annotation_file(annotations_dir, split).read_text(encoding='utf-8')) + names = {c['id']: c['name'] for c in doc['categories']} + found: dict[str, list[tuple[list[float], str]]] = {} + for ann in doc['annotations']: + row = rows.get((split, ann['image_id'])) + if row is None or ann.get('iscrowd'): + continue + x, y, w, h = ann['bbox'] + box = [ + x / row['width'], + y / row['height'], + (x + w) / row['width'], + (y + h) / row['height'], + ] + found.setdefault(row['file_name'], []).append((box, names[ann['category_id']])) + for file_name, anns in found.items(): + out[file_name] = (np.array([a[0] for a in anns], np.float32), [a[1] for a in anns]) + return out + + +def assign_gt(boxes_norm: np.ndarray, gt: tuple[np.ndarray, list[str]]) -> list[str]: + """Per box, the class of the best-IoU ground-truth annotation (IoU >= 0.5), else ``none``.""" + gt_boxes, gt_names = gt + if len(boxes_norm) == 0 or len(gt_boxes) == 0: + return [GT_NONE] * len(boxes_norm) + iou = pm.box_iou(boxes_norm, gt_boxes) + best = iou.argmax(axis=1) + return [ + gt_names[j] if iou[i, j] >= GT_MIN_IOU else GT_NONE for i, j in enumerate(best.tolist()) + ] + + +def _str_array(values: list[str]) -> np.ndarray: + return np.array(values, dtype=str) if values else np.zeros(0, dtype='U1') + + +def _vec_array(rows: list[list[float]], dim: int) -> np.ndarray: + return np.array(rows, dtype=np.float16).reshape(len(rows), dim) + + +def build_arrays( + items: list[dict[str, Any]], + *, + image_names: dict[str, str], + image_dims: dict[str, tuple[int, int]], + vectors: dict[str, list[float]], + frame_vectors: dict[str, list[float]], + gt: GroundTruth | None, + vlm: dict[str, str], + probes: dict[str, list[tuple[Any, ...]]], +) -> dict[str, np.ndarray]: + """The schema arrays from API item rows, vectors read per id, ground truth and core probes. + + ``image_names`` maps an image id to the pinned file name. Items sort by file name then box, + so two captures of the same set line up row for row where they agree. + """ + dim = len(next(iter(vectors.values()), [])) or len(next(iter(frame_vectors.values()), [])) + keyed = sorted( + ((image_names[it['image_id']], tuple(it['bbox_norm']), it) for it in items), + key=lambda t: (t[0], t[1], t[2]['id']), + ) + arrays = ps.empty_arrays(dim) + if keyed: + files = [k[0] for k in keyed] + rows = [k[2] for k in keyed] + boxes = np.array([k[1] for k in keyed], np.float32) + gt_names = [''] * len(files) + if gt: + groups: dict[str, list[int]] = {} + for i, name in enumerate(files): + groups.setdefault(name, []).append(i) + for name, idx in groups.items(): + for i, label in zip(idx, assign_gt(boxes[idx], gt[name]), strict=True): + gt_names[i] = label + has_vec = [r['id'] in vectors for r in rows] + arrays.update( + item_id=_str_array([r['id'] for r in rows]), + item_image=_str_array(files), + item_box=boxes, + item_score=np.array( + [ + r['detector_confidence'] + if r.get('detector_confidence') is not None + else r['confidence'] + for r in rows + ], + np.float32, + ), + item_class=_str_array( + [r.get('detector_class_name') or r.get('class_name') or '' for r in rows] + ), + item_crop_rect=np.array( + [crop_rect(list(k[1]), *image_dims[k[0]]) for k in keyed], np.int32 + ), + item_cluster=np.array( + [-1 if r.get('cluster_id') is None else r['cluster_id'] for r in rows], np.int32 + ), + item_cluster_distance=np.array( + [ + np.nan if r.get('cluster_distance') is None else r['cluster_distance'] + for r in rows + ], + np.float32, + ), + item_cluster_similarity=np.array( + [ + np.nan if r.get('cluster_similarity') is None else r['cluster_similarity'] + for r in rows + ], + np.float32, + ), + item_cluster_core=np.array( + [ + -1 if r.get('cluster_is_core') is None else int(r['cluster_is_core']) + for r in rows + ], + np.int8, + ), + item_label=_str_array([r.get('class_name') or '' for r in rows]), + item_label_source=_str_array([r.get('class_source') or '' for r in rows]), + item_gt=_str_array(gt_names), + item_vlm=_str_array([vlm.get(r['id'], '') for r in rows]), + item_has_vec=np.array(has_vec, dtype=bool), + item_vec=_vec_array([vectors.get(r['id']) or [0.0] * dim for r in rows], dim), + ) + frame_files = sorted(set(image_names.values())) + by_file = {image_names[i]: v for i, v in frame_vectors.items() if i in image_names} + kept = [f for f in frame_files if f in by_file] + arrays['frame_image'] = _str_array(kept) + arrays['frame_vec'] = _vec_array([by_file[f] for f in kept], dim) + clip = sorted(probes.get('clip', []), key=lambda r: r[0]) + arrays['clip_image'] = _str_array([r[0] for r in clip]) + arrays['clip_vec'] = _vec_array([r[1] for r in clip], len(clip[0][1]) if clip else 0) + faces = sorted(probes.get('faces', []), key=lambda r: (r[0], tuple(r[1]))) + arrays['face_image'] = _str_array([r[0] for r in faces]) + arrays['face_box'] = np.array([r[1] for r in faces], np.float32).reshape(len(faces), 4) + arrays['face_score'] = np.array([r[2] for r in faces], np.float32) + arrays['face_vec'] = _vec_array([r[3] for r in faces], len(faces[0][3]) if faces else 0) + ocr = sorted(probes.get('ocr', []), key=lambda r: (r[0], tuple(r[1]))) + arrays['ocr_image'] = _str_array([r[0] for r in ocr]) + arrays['ocr_box'] = np.array([r[1] for r in ocr], np.float32).reshape(len(ocr), 4) + arrays['ocr_text'] = _str_array([r[2] for r in ocr]) + return arrays diff --git a/scripts/bench/parity_capture.py b/scripts/bench/parity_capture.py new file mode 100644 index 00000000..7f89cd85 --- /dev/null +++ b/scripts/bench/parity_capture.py @@ -0,0 +1,228 @@ +"""Collect the per-stage outputs of one ingest of the pinned set from a running stack. + +Public API only, project scoped: images go in through ``POST .../ingest/upload``, items and their +cluster fields come back through ``GET .../crops``, and the core routes (MobileCLIP, faces, OCR) +are probed on a sample. The curation API deliberately never serves embedding vectors, so the PE +vectors are read by document id from the project's own OpenSearch indexes (the datastore of the +stack under test, never an index of another project). +""" + +from __future__ import annotations + +import hashlib +import math +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, Any + +from scripts.bench.baseline_suite import run_autolabel +from scripts.bench.parity_store import CaptureError +from src.config.curation import ITEM_EMBEDDING_FIELD + + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + import httpx + +# The list routes refuse to page past this many rows, so wider result sets are split in bands. +RESULT_WINDOW = 10_000 +PAGE_SIZE = 500 +MGET_CHUNK = 200 +EMBED_ROUTE = '/embed/image' +FACES_ROUTE = '/faces/recognize' +OCR_ROUTE = '/ocr/predict' + + +def chunks(items: Sequence[Any], size: int) -> list[Sequence[Any]]: + return [items[i : i + size] for i in range(0, len(items), size)] + + +def project_indexes(client: httpx.Client, base: str) -> tuple[str, str]: + """The project's ``(items, images)`` index names, as the API reports them.""" + names = client.get(base).raise_for_status().json()['resources']['indexes'].values() + items = [n for n in names if n.endswith('__items')] + images = [n for n in names if n.endswith('__images')] + if len(items) != 1 or len(images) != 1: + raise CaptureError(f'expected one items and one images index, got {items} and {images}') + return items[0], images[0] + + +def require_embed_all(client: httpx.Client, base: str) -> str: + """Fail unless the project embeds every crop; a parity set needs a vector per item.""" + policy = client.get(f'{base}/ingest/policy').raise_for_status().json() + mode = policy['embedding']['mode'] + if mode != 'all': + raise CaptureError(f'ingest embedding policy is {mode!r}; a parity capture needs "all"') + return mode + + +def ingest_images( + client: httpx.Client, base: str, files: Sequence[Path], *, batch_size: int, concurrency: int +) -> dict[str, str]: + """Upload every file; returns ``{image_id: file name}``. Any failed image aborts.""" + + def post(batch: Sequence[Path]) -> dict[str, Any]: + upload = [('images', (p.name, p.read_bytes(), 'image/jpeg')) for p in batch] + return client.post(f'{base}/ingest/upload', files=upload).raise_for_status().json() + + with ThreadPoolExecutor(concurrency) as pool: + bodies = list(pool.map(post, chunks(list(files), batch_size))) + names: dict[str, str] = {} + failed: list[str] = [] + for body in bodies: + for res in body['results']: + if res['status'] == 'failed': + failed.append( + f'{res.get("source_identifier") or res["image_path"]}: {res.get("error")}' + ) + else: + names[res['image_id']] = (res.get('source_identifier') or res['image_path']).rsplit( + '/', 1 + )[-1] + if failed: + raise CaptureError(f'{len(failed)} images failed to ingest, first: {failed[0]}') + return names + + +def _crops_page(client: httpx.Client, base: str, params: dict[str, Any]) -> dict[str, Any]: + query = {'include_test': 'true', 'include_excluded': 'true', **params} + return client.get(f'{base}/crops', params=query).raise_for_status().json() + + +def fetch_items( + client: httpx.Client, base: str, *, window: int = RESULT_WINDOW, page_size: int = PAGE_SIZE +) -> list[dict[str, Any]]: + """Every item of the project through ``GET .../crops``. + + A result set wider than the API's paging window is split by confidence band, recursively; + items on a band edge appear in both and are de-duplicated by id. The union must equal the + unfiltered total, otherwise the capture aborts (unstable paging or a missed band). + """ + total = int(_crops_page(client, base, {'page_size': 1})['total']) + items: dict[str, dict[str, Any]] = {} + bands = [(0.0, 1.0)] + while bands: + low, high = bands.pop() + band = {'conf_min': low, 'conf_max': high} + count = int(_crops_page(client, base, {**band, 'page_size': 1})['total']) + if count > window and high - low > 1e-6: + mid = (low + high) / 2 + bands += [(low, mid), (mid, high)] + continue + for page in range(1, math.ceil(count / page_size) + 1): + body = _crops_page(client, base, {**band, 'page': page, 'page_size': page_size}) + items.update({c['id']: c for c in body['crops']}) + if len(items) != total: + raise CaptureError(f'read {len(items)} items but the project reports {total}') + return list(items.values()) + + +def mget_vectors( + client: httpx.Client, + os_url: str, + index: str, + ids: Sequence[str], + field: str = ITEM_EMBEDDING_FIELD, +) -> dict[str, list[float]]: + """``{doc id: vector}`` read by id from one index; documents without a vector are absent.""" + out: dict[str, list[float]] = {} + for part in chunks(list(ids), MGET_CHUNK): + resp = client.post( + f'{os_url}/{index}/_mget', params={'_source_includes': field}, json={'ids': list(part)} + ) + for doc in resp.raise_for_status().json()['docs']: + vec = (doc.get('_source') or {}).get(field) if doc.get('found') else None + if vec: + out[doc['_id']] = vec + return out + + +def run_clustering(client: httpx.Client, base: str, timeout: float) -> dict[str, Any]: + """One cluster training pass (no VLM) through the auto-label job. + + The job is claimed by the auto-label worker (compose profile ``curation``); without it + the job stays queued until ``timeout``. + """ + result = run_autolabel( + client, base, {'train_clusters': 'true', 'run_vlm': 'false'}, timeout=timeout + ) + if result['status'] != 'completed': + raise CaptureError(f'cluster training ended {result["status"]}') + return result + + +def probe_core( + client: httpx.Client, api_url: str, files: Sequence[Path], *, concurrency: int +) -> dict[str, list[tuple[Any, ...]]]: + """MobileCLIP, face (box, score, ArcFace vector) and OCR (box, text) outputs per file. + + A 422 from a route means "nothing usable in this image" and yields no rows; any other + error aborts. + """ + + def post(route: str, path: Path) -> dict[str, Any] | None: + resp = client.post( + f'{api_url}{route}', files={'image': (path.name, path.read_bytes(), 'image/jpeg')} + ) + if resp.status_code == 422: + return None + return resp.raise_for_status().json() + + def one(path: Path) -> dict[str, list[tuple[Any, ...]]]: + out: dict[str, list[tuple[Any, ...]]] = {'clip': [], 'faces': [], 'ocr': []} + if clip := post(EMBED_ROUTE, path): + out['clip'].append((path.name, clip['embedding'])) + if faces := post(FACES_ROUTE, path): + out['faces'] += [ + (path.name, f['box'], f['score'], vec) + for f, vec in zip(faces['faces'], faces.get('embeddings') or [], strict=False) + ] + if ocr := post(OCR_ROUTE, path): + out['ocr'] += [(path.name, r['box_normalized'], r['text']) for r in ocr['regions']] + return out + + merged: dict[str, list[tuple[Any, ...]]] = {'clip': [], 'faces': [], 'ocr': []} + with ThreadPoolExecutor(concurrency) as pool: + for part in pool.map(one, files): + for key, rows in part.items(): + merged[key] += rows + return merged + + +def vlm_sample( + items: Sequence[dict[str, Any]], image_names: dict[str, str], count: int +) -> list[str]: + """Item ids of about ``count`` crops, whole images chosen by a hash of the file name. + + The choice depends only on file names, so two captures of the same set sample the same images. + """ + by_file: dict[str, list[str]] = {} + for item in items: + by_file.setdefault(image_names[item['image_id']], []).append(item['id']) + chosen: list[str] = [] + for name in sorted(by_file, key=lambda n: hashlib.sha256(n.encode()).hexdigest()): + if len(chosen) >= count: + break + chosen += sorted(by_file[name]) + return chosen + + +def label_with_vlm( + client: httpx.Client, base: str, ids: Sequence[str], classes: Sequence[str], batch: int +) -> dict[str, str]: + """Run ``vlm/label_batch`` on ``ids`` and return ``{item id: proposed class name}``.""" + for name in classes: + client.post(f'{base}/classes', json={'name': name, 'group': 'parity'}) + for part in chunks(list(ids), batch): + client.post(f'{base}/vlm/label_batch', json={'crop_ids': list(part)}).raise_for_status() + out: dict[str, str] = {} + for part in chunks(list(ids), 100): + rows = _crops_page(client, base, {'ids': ','.join(part)})['crops'] + out.update( + { + r['id']: r.get('vlm_proposed_class_name') or r.get('vlm_raw_class') or '' + for r in rows + } + ) + return out diff --git a/scripts/bench/parity_env.py b/scripts/bench/parity_env.py new file mode 100644 index 00000000..9ddfb0aa --- /dev/null +++ b/scripts/bench/parity_env.py @@ -0,0 +1,136 @@ +"""The provenance block of a parity capture: hardware, git sha, image digests, engine record. + +Reuses the baseline suite's environment probe (GPU, CPU, RAM, Triton and OpenSearch versions, +dataset pin, foreign GPU load) and adds what a precision comparison needs: the image digests of +the compose project, whether the working tree was clean, the Triton model configuration of every +model the capture touches, and the operator's statement of the engine under test together with the +hash of the plan file when one is given. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from scripts.bench.parity_store import sha256_file +from scripts.bench.suite_probes import phase_env, run_cmd +from src.config.settings import TritonModelConfig + + +if TYPE_CHECKING: + import httpx + +MODELS = { + 'detector': TritonModelConfig.YOLO_MODEL, + 'pe_image': TritonModelConfig.PE_IMAGE_MODEL, + 'clip_image': TritonModelConfig.CLIP_IMAGE_MODEL, + 'face_detect': TritonModelConfig.FACE_DETECT_MODEL, + 'face_embed': TritonModelConfig.ARCFACE_MODEL, +} +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def parse_inspect(text: str) -> dict[str, dict[str, str]]: + """``docker inspect --format '{{.Name}} {{.Config.Image}} {{.Image}}'`` -> name -> image, id.""" + out: dict[str, dict[str, str]] = {} + for line in text.splitlines(): + parts = line.split() + if len(parts) == 3: + out[parts[0].lstrip('/')] = {'image': parts[1], 'image_id': parts[2]} + return out + + +def image_digests(compose_project: str) -> dict[str, dict[str, str]]: + """Image reference and image id of every container of one compose project.""" + names = run_cmd( + [ + 'docker', + 'ps', + '--filter', + f'label=com.docker.compose.project={compose_project}', + '--format', + '{{.Names}}', + ] + ).split() + if not names: + return {} + return parse_inspect( + run_cmd(['docker', 'inspect', '--format', '{{.Name}} {{.Config.Image}} {{.Image}}', *names]) + ) + + +def git_state() -> dict[str, Any]: + status = run_cmd(['git', '-C', str(REPO_ROOT), 'status', '--porcelain']) + return { + 'git_sha': run_cmd(['git', '-C', str(REPO_ROOT), 'rev-parse', 'HEAD']).strip(), + 'git_branch': run_cmd( + ['git', '-C', str(REPO_ROOT), 'rev-parse', '--abbrev-ref', 'HEAD'] + ).strip(), + 'git_dirty': bool(status.strip()), + } + + +def triton_models(client: httpx.Client, triton_url: str) -> dict[str, Any]: + """Platform, batch limit and tensor types of each model the capture touches.""" + out: dict[str, Any] = {} + for role, model in MODELS.items(): + resp = client.get(f'{triton_url}/v2/models/{model}/config') + if resp.status_code != 200: + out[role] = {'name': model, 'available': False} + continue + cfg = resp.json() + out[role] = { + 'name': model, + 'available': True, + 'platform': cfg.get('platform') or cfg.get('backend'), + 'max_batch_size': cfg.get('max_batch_size'), + 'input': [(t['name'], t['data_type']) for t in cfg.get('input', [])], + 'output': [(t['name'], t['data_type']) for t in cfg.get('output', [])], + } + return out + + +def engine_record(label: str, plan: Path | None, extra: list[str]) -> dict[str, Any]: + """The operator's statement of the engine under test plus the plan file's size and hash.""" + record: dict[str, Any] = {'label': label} + for pair in extra: + key, sep, value = pair.partition('=') + if not sep: + raise ValueError(f'--engine-record expects KEY=VALUE, got {pair!r}') + record[key] = value + if plan is not None: + record['plan_file'] = plan.name + record['plan_bytes'] = plan.stat().st_size + record['plan_sha256'] = sha256_file(plan) + return record + + +def environment_record( + client: httpx.Client, + *, + gpu: str, + compose_project: str, + api_url: str, + triton_url: str, + opensearch_url: str, + manifest: Path, + idle_samples: int = 3, +) -> dict[str, Any]: + probe_args = argparse.Namespace( + gpu=gpu, + project=compose_project, + manifest=manifest, + idle_samples=idle_samples, + api_url=api_url, + triton_url=triton_url, + opensearch_url=opensearch_url, + ) + env = phase_env(probe_args, client) + return { + **env, + **git_state(), + 'compose_project': compose_project, + 'image_digests': image_digests(compose_project), + 'triton_models': triton_models(client, triton_url), + } diff --git a/scripts/bench/parity_gates.json b/scripts/bench/parity_gates.json new file mode 100644 index 00000000..8eb9f941 --- /dev/null +++ b/scripts/bench/parity_gates.json @@ -0,0 +1,164 @@ +{ + "schema": 1, + "source": "docs/design/triton_pipeline_optimization_plan.md section 7; docs/design/v060_wave_plan.md sections 2 and 2.1", + "settings": { + "knn_k": 10, + "knn_queries": 1000, + "near_duplicate_cosine": 0.98, + "seed": 20261009 + }, + "profiles": { + "precision": { + "description": "A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent.", + "checks": [ + { + "id": "P-box", + "metric": "boxes.images_identical_share", + "op": ">=", + "value": 0.98, + "note": "images with every box matched at IoU >= 0.95 and confidence delta <= 0.02" + }, + { + "id": "P-box-confident", + "metric": "boxes.unmatched_confident", + "op": "<=", + "value": 0, + "note": "unmatched boxes with confidence >= 0.5" + }, + { + "id": "P-item", + "metric": "boxes.item_count_class_identical_share", + "op": ">=", + "value": 0.99, + "note": "images with the same item count and class names" + }, + { + "id": "P-pe-p01", + "metric": "pe_crop.p01", + "op": ">=", + "value": 0.995, + "note": "cosine, crop vectors of matched items" + }, + { + "id": "P-pe-median", + "metric": "pe_crop.median", + "op": ">=", + "value": 0.999 + }, + { + "id": "P-pew-p01", + "metric": "pe_frame.p01", + "op": ">=", + "value": 0.995, + "note": "cosine, whole-frame vectors" + }, + { + "id": "P-pew-median", + "metric": "pe_frame.median", + "op": ">=", + "value": 0.999 + }, + { + "id": "P-knn", + "metric": "knn.overlap_at_k", + "op": ">=", + "value": 0.95, + "note": "neighbour overlap at k over the query items" + }, + { + "id": "P-dup", + "metric": "knn.near_dup_agreement", + "op": ">=", + "value": 0.995, + "note": "near-duplicate decisions at the configured cosine, identical on this share of pairs" + }, + { + "id": "P-cluster-assign", + "metric": "cluster.assignment_agreement", + "op": ">=", + "value": 0.98, + "note": "nearest reference centroid unchanged when candidate vectors are assigned to it" + }, + { + "id": "P-cluster-purity", + "metric": "cluster.purity_drop_points", + "op": "<=", + "value": 0.5, + "note": "purity against COCO ground-truth classes, percentage points lost" + }, + { + "id": "P-label-vote", + "metric": "knn.label_vote_agreement", + "op": ">=", + "value": 0.97, + "note": "majority detector class among the top-k neighbours, identical in both spaces (label propagation); the VLM agreement bar" + }, + { + "id": "P-knn-precision", + "metric": "knn.precision_drop_points", + "op": "<=", + "value": 0.5, + "note": "kNN precision against COCO ground-truth classes, percentage points lost" + }, + { + "id": "P-label", + "metric": "labels.class_agreement", + "op": ">=", + "value": 0.99, + "note": "class name per matched item (the auto-promote outcome)" + }, + { + "id": "P-vlm", + "metric": "labels.vlm_agreement", + "op": ">=", + "value": 0.97, + "note": "VLM label on the sampled crops" + }, + { + "id": "P-clip", + "metric": "clip.p01", + "op": ">=", + "value": 0.995, + "note": "core-route image embedding, cosine" + }, + { + "id": "P-face", + "metric": "face.p01", + "op": ">=", + "value": 0.995, + "note": "face embedding of matched faces, cosine" + }, + { + "id": "P-ocr", + "metric": "ocr.exact_share", + "op": ">=", + "value": 0.98, + "note": "OCR lines matched at IoU >= 0.5, exact string" + } + ], + "record": [ + "boxes.crop_rect_identical_share", + "pe_crop.identical_share", + "pe_frame.identical_share", + "cluster.ari", + "cluster.nmi", + "cluster.objective_rel_diff", + "cluster.purity_ref", + "cluster.similarity_abs_delta_mean", + "cluster.similarity_abs_delta_p99", + "cluster.core_agreement", + "knn.precision_ref", + "knn.precision_cand", + "cluster.purity_cand", + "knn.top1_recall_at_k", + "pe_crop.min", + "pe_crop.mean", + "pe_frame.min", + "pe_frame.mean", + "boxes.iou_min", + "boxes.score_delta_max", + "ocr.char_error_rate" + ] + } + } +} diff --git a/scripts/bench/parity_gates.py b/scripts/bench/parity_gates.py new file mode 100644 index 00000000..5534d3e4 --- /dev/null +++ b/scripts/bench/parity_gates.py @@ -0,0 +1,297 @@ +"""Compare two parity captures: metrics, gate evaluation, exit code and the markdown summary. + +``compute_metrics`` pairs the captures item by item (image, box overlap and class name, never by +stored id) and measures each stage in the downstream order: detection, crops, embeddings, +core-route models, clustering, labels. ``evaluate`` applies the thresholds of one profile of +``parity_gates.json``. A gate whose metric could not be measured (a section missing from either +capture) is reported as SKIP and fails only under ``--strict``. +""" + +from __future__ import annotations + +import json +import operator +from typing import TYPE_CHECKING, Any + +import numpy as np + +from scripts.bench import parity_metrics as pm +from scripts.bench.parity_store import Capture, CaptureError + + +if TYPE_CHECKING: + from pathlib import Path + +OPS = {'>=': operator.ge, '<=': operator.le, '==': operator.eq} +GATE_SCHEMA = 1 + + +def load_gates(path: Path) -> dict[str, Any]: + doc = json.loads(path.read_text(encoding='utf-8')) + if doc.get('schema') != GATE_SCHEMA: + raise ValueError(f'{path}: gate schema {doc.get("schema")}, this tool reads {GATE_SCHEMA}') + for name, profile in doc['profiles'].items(): + for check in profile['checks']: + if check['op'] not in OPS: + raise ValueError(f'{path}: profile {name} check {check["id"]}: bad operator') + return doc + + +def _pair_by_name(ref: np.ndarray, cand: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Index arrays of the names present in both (one row per name, first occurrence).""" + cand_index: dict[str, int] = {} + for i, name in enumerate(cand.tolist()): + cand_index.setdefault(name, i) + seen: set[str] = set() + ri, ci = [], [] + for i, name in enumerate(ref.tolist()): + if name in cand_index and name not in seen: + seen.add(name) + ri.append(i) + ci.append(cand_index[name]) + return np.array(ri, dtype=np.int64), np.array(ci, dtype=np.int64) + + +def _vector_section(ref: np.ndarray, cand: np.ndarray, expected: int) -> dict[str, Any] | None: + """Cosine summary of row-aligned vectors; ``expected`` rows existed in the reference. + + No expected rows means nothing to compare (``None``). Expected rows with no counterpart + score the worst case, so a vanished section fails its gates instead of skipping them. + """ + if expected == 0: + return None + if len(ref) == 0: + return { + 'n': 0, + 'min': -1.0, + 'mean': -1.0, + 'p01': -1.0, + 'median': -1.0, + 'identical_share': 0.0, + } + if ref.shape[1] != cand.shape[1]: + raise CaptureError(f'vector widths differ: {ref.shape[1]} against {cand.shape[1]}') + return { + **pm.cosine_summary(pm.cosine_rows(ref, cand)), + 'identical_share': float(np.mean(np.all(ref == cand, axis=1))), + } + + +def _cluster_section( + ref: Capture, cand: Capture, pairs: np.ndarray, has_vec: np.ndarray +) -> dict[str, Any] | None: + ri, ci = pairs[has_vec, 0], pairs[has_vec, 1] + a, b = ref.arrays['item_cluster'][ri], cand.arrays['item_cluster'][ci] + ok = (a >= 0) & (b >= 0) + if ok.sum() < 2: + return None + ri, ci, a, b = ri[ok], ci[ok], a[ok], b[ok] + rv, cv = ref.arrays['item_vec'][ri], cand.arrays['item_vec'][ci] + obj_ref, obj_cand = pm.kmeans_objective(rv, a), pm.kmeans_objective(cv, b) + out: dict[str, Any] = { + 'items': int(ok.sum()), + 'clusters_ref': len(np.unique(a)), + 'clusters_cand': len(np.unique(b)), + 'ari': pm.adjusted_rand_index(a, b), + 'nmi': pm.normalized_mutual_info(a, b), + 'assignment_agreement': pm.assignment_agreement(rv, cv, a), + 'objective_ref': obj_ref, + 'objective_cand': obj_cand, + 'objective_rel_diff': (obj_cand - obj_ref) / obj_ref if obj_ref else 0.0, + } + sim = np.abs( + ref.arrays['item_cluster_similarity'][ri] - cand.arrays['item_cluster_similarity'][ci] + ) + if not np.isnan(sim).all(): + out['similarity_abs_delta_mean'] = float(np.nanmean(sim)) + out['similarity_abs_delta_p99'] = float(np.nanpercentile(sim, 99)) + out['core_agreement'] = float( + np.mean(ref.arrays['item_cluster_core'][ri] == cand.arrays['item_cluster_core'][ci]) + ) + gt_ref, gt_cand = ref.arrays['item_gt'][ri], cand.arrays['item_gt'][ci] + if (gt_ref != '').any() and (gt_cand != '').any(): + out['purity_ref'] = pm.purity(a, gt_ref) + out['purity_cand'] = pm.purity(b, gt_cand) + out['purity_drop_points'] = (out['purity_ref'] - out['purity_cand']) * 100.0 + return out + + +def _label_section( + ref: Capture, cand: Capture, pairs: np.ndarray, *, with_vlm: bool +) -> dict[str, Any]: + ri, ci = pairs[:, 0], pairs[:, 1] + out: dict[str, Any] = { + 'class_agreement': pm.agreement_share( + ref.arrays['item_label'][ri].tolist(), cand.arrays['item_label'][ci].tolist() + ), + 'pairs': len(pairs), + } + if not with_vlm: + return out + rv, cv = ref.arrays['item_vlm'][ri], cand.arrays['item_vlm'][ci] + both = (rv != '') | (cv != '') + if both.any(): + out['vlm_agreement'] = pm.agreement_share(rv[both].tolist(), cv[both].tolist()) + out['vlm_pairs'] = int(both.sum()) + return out + + +def compute_metrics(ref: Capture, cand: Capture, settings: dict[str, Any]) -> dict[str, Any]: + """Every measurable stage metric of ``cand`` against ``ref`` as nested plain dicts. + + A stage not captured in both (``sections`` in the metadata) is left out, so its gates SKIP. + """ + for key in ('sha256', 'count'): + if ref.meta['manifest'][key] != cand.meta['manifest'][key]: + raise CaptureError(f'the captures differ in dataset manifest {key}') + ra, ca = ref.arrays, cand.arrays + both = set(ref.meta['sections']) & set(cand.meta['sections']) + metrics: dict[str, Any] = {} + pairs = pm.pair_items( + (ra['item_image'], ra['item_box'], ra['item_class']), + (ca['item_image'], ca['item_box'], ca['item_class']), + ) + boxes = pm.box_metrics( + (ra['item_image'], ra['item_box'], ra['item_class'], ra['item_score']), + (ca['item_image'], ca['item_box'], ca['item_class'], ca['item_score']), + ) + same_rect = np.all( + ra['item_crop_rect'][pairs[:, 0]] == ca['item_crop_rect'][pairs[:, 1]], axis=1 + ) + boxes['crop_rect_identical_share'] = float(same_rect.mean()) if len(pairs) else 1.0 + metrics['boxes'] = boxes + if len(pairs) == 0: + return metrics + has_vec = ra['item_has_vec'][pairs[:, 0]] & ca['item_has_vec'][pairs[:, 1]] + ri, ci = pairs[has_vec, 0], pairs[has_vec, 1] + expected = int(ra['item_has_vec'].sum()) + if section := _vector_section(ra['item_vec'][ri], ca['item_vec'][ci], expected): + metrics['pe_crop'] = {**section, 'matched': len(ri)} + if len(ri) > 2: + metrics['knn'] = pm.knn_agreement( + ra['item_vec'][ri], + ca['item_vec'][ci], + k=settings['knn_k'], + n_queries=settings['knn_queries'], + seed=settings['seed'], + dup_threshold=settings['near_duplicate_cosine'], + labels=ra['item_class'][ri], + truth=(ra['item_gt'][ri], ca['item_gt'][ci]), + ) + if 'frames' in both: + fr, fc = _pair_by_name(ra['frame_image'], ca['frame_image']) + if section := _vector_section( + ra['frame_vec'][fr], ca['frame_vec'][fc], len(ra['frame_image']) + ): + metrics['pe_frame'] = {**section, 'matched': len(fr)} + if 'clip' in both: + cr, cc = _pair_by_name(ra['clip_image'], ca['clip_image']) + if section := _vector_section( + ra['clip_vec'][cr], ca['clip_vec'][cc], len(ra['clip_image']) + ): + metrics['clip'] = section + if 'faces' in both: + face_pairs = pm.pair_items( + (ra['face_image'], ra['face_box'], np.full(len(ra['face_image']), 'face')), + (ca['face_image'], ca['face_box'], np.full(len(ca['face_image']), 'face')), + ) + if section := _vector_section( + ra['face_vec'][face_pairs[:, 0]], + ca['face_vec'][face_pairs[:, 1]], + len(ra['face_image']), + ): + metrics['face'] = { + **section, + 'ref': len(ra['face_image']), + 'cand': len(ca['face_image']), + } + if 'ocr' in both and (len(ra['ocr_image']) or len(ca['ocr_image'])): + metrics['ocr'] = pm.text_agreement( + (ra['ocr_image'], ra['ocr_box'], ra['ocr_text']), + (ca['ocr_image'], ca['ocr_box'], ca['ocr_text']), + ) + if 'clusters' in both and (cluster := _cluster_section(ref, cand, pairs, has_vec)): + metrics['cluster'] = cluster + metrics['labels'] = _label_section(ref, cand, pairs, with_vlm='vlm' in both) + return metrics + + +def _lookup(metrics: dict[str, Any], dotted: str) -> Any: + node: Any = metrics + for part in dotted.split('.'): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def evaluate(metrics: dict[str, Any], gates: dict[str, Any], profile: str) -> dict[str, Any]: + """Apply one profile of the gate file to the metrics.""" + spec = gates['profiles'][profile] + checks = [] + for check in spec['checks']: + observed = _lookup(metrics, check['metric']) + if observed is None: + status = 'SKIP' + else: + status = 'PASS' if OPS[check['op']](observed, check['value']) else 'FAIL' + checks.append({**check, 'observed': observed, 'status': status}) + count = lambda s: sum(1 for c in checks if c['status'] == s) # noqa: E731 + return { + 'profile': profile, + 'description': spec['description'], + 'checks': checks, + 'recorded': {m: _lookup(metrics, m) for m in spec['record']}, + 'failed': count('FAIL'), + 'skipped': count('SKIP'), + 'passed': count('FAIL') == 0, + } + + +def exit_code(result: dict[str, Any], *, strict: bool) -> int: + return 1 if result['failed'] or (strict and result['skipped']) else 0 + + +def _fmt(value: Any) -> str: + if value is None: + return 'n/a' + return f'{value:.5f}' if isinstance(value, float) else str(value) + + +def render_markdown( + ref_meta: dict[str, Any], + cand_meta: dict[str, Any], + metrics: dict[str, Any], + result: dict[str, Any], +) -> str: + verdict = 'PASS' if result['passed'] else 'FAIL' + lines = [ + f'# Parity: {cand_meta["tag"]} against {ref_meta["tag"]} ({verdict})', + '', + f'Profile `{result["profile"]}`: {result["description"]}', + '', + '| | reference | candidate |', + '|---|---|---|', + f'| tag | {ref_meta["tag"]} | {cand_meta["tag"]} |', + f'| engine | {ref_meta["engine"].get("label", "")} | {cand_meta["engine"].get("label", "")} |', + f'| git sha | {ref_meta["environment"].get("git_sha", "")[:12]} | ' + f'{cand_meta["environment"].get("git_sha", "")[:12]} |', + f'| manifest sha256 | {ref_meta["manifest"]["sha256"][:16]} | {cand_meta["manifest"]["sha256"][:16]} |', + '', + '| Gate | Metric | Observed | Rule | Status |', + '|---|---|---:|---|---|', + ] + lines += [ + f'| {c["id"]} | `{c["metric"]}` | {_fmt(c["observed"])} | {c["op"]} {c["value"]} | {c["status"]} |' + for c in result['checks'] + ] + if result['recorded']: + lines += ['', 'Recorded, not gated:', '', '| Metric | Value |', '|---|---:|'] + lines += [f'| `{k}` | {_fmt(v)} |' for k, v in result['recorded'].items()] + lines += [ + '', + f'Items: {metrics["boxes"]["items_ref"]} reference, {metrics["boxes"]["items_cand"]} candidate, ' + f'{metrics["boxes"]["matched"]} matched.', + '', + ] + return '\n'.join(lines) diff --git a/scripts/bench/parity_metrics.py b/scripts/bench/parity_metrics.py new file mode 100644 index 00000000..74e97893 --- /dev/null +++ b/scripts/bench/parity_metrics.py @@ -0,0 +1,410 @@ +"""Pure parity metrics: box matching, vector agreement, clustering agreement, strings. + +Every function takes plain numpy arrays or lists and returns plain numbers, so a +synthetic fixture pins each one down. Nothing here touches the network or disk. +The gate thresholds live in ``parity_gates.json``; this module only measures. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from typing import TYPE_CHECKING, Any + +import numpy as np + + +if TYPE_CHECKING: + from collections.abc import Sequence + +MATCH_IOU = 0.5 +IDENTICAL_IOU = 0.95 +SCORE_TOLERANCE = 0.02 +CONFIDENT_SCORE = 0.5 + + +# ---------------------------------------------------------------- boxes + + +def box_iou(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """IoU matrix of two ``(n, 4)`` / ``(m, 4)`` arrays of ``x1, y1, x2, y2`` boxes.""" + a64, b64 = a.astype(np.float64), b.astype(np.float64) + lt = np.maximum(a64[:, None, :2], b64[None, :, :2]) + rb = np.minimum(a64[:, None, 2:], b64[None, :, 2:]) + inter = np.clip(rb - lt, 0.0, None).prod(axis=2) + area_a = np.clip(a64[:, 2:] - a64[:, :2], 0.0, None).prod(axis=1) + area_b = np.clip(b64[:, 2:] - b64[:, :2], 0.0, None).prod(axis=1) + union = area_a[:, None] + area_b[None, :] - inter + return np.divide(inter, union, out=np.zeros_like(inter), where=union > 0) + + +def match_boxes( + ref_boxes: np.ndarray, + ref_cls: Sequence[str], + cand_boxes: np.ndarray, + cand_cls: Sequence[str], + min_iou: float = MATCH_IOU, +) -> tuple[list[tuple[int, int]], list[int], list[int]]: + """Greedy one-to-one matching, best IoU first, only within the same class name. + + Returns ``(pairs, unmatched_ref, unmatched_cand)`` as indices into the inputs. + """ + if len(ref_boxes) == 0 or len(cand_boxes) == 0: + return [], list(range(len(ref_boxes))), list(range(len(cand_boxes))) + iou = box_iou(ref_boxes, cand_boxes) + same = np.asarray(ref_cls)[:, None] == np.asarray(cand_cls)[None, :] + iou = np.where(same, iou, 0.0) + ri, ci = np.nonzero(iou >= min_iou) + order = np.lexsort((ci, ri, -iou[ri, ci])) + used_r: set[int] = set() + used_c: set[int] = set() + pairs: list[tuple[int, int]] = [] + for k in order: + r, c = int(ri[k]), int(ci[k]) + if r in used_r or c in used_c: + continue + used_r.add(r) + used_c.add(c) + pairs.append((r, c)) + miss_ref = [i for i in range(len(ref_boxes)) if i not in used_r] + miss_cand = [i for i in range(len(cand_boxes)) if i not in used_c] + return sorted(pairs), miss_ref, miss_cand + + +def _group(images: np.ndarray) -> dict[str, list[int]]: + groups: dict[str, list[int]] = defaultdict(list) + for i, name in enumerate(images.tolist()): + groups[name].append(i) + return groups + + +def _match_by_image( + ref: tuple[Any, ...], cand: tuple[Any, ...], min_iou: float +) -> list[tuple[str, list[tuple[int, int]], list[int], list[int]]]: + """Per image (sorted by name): matched pairs and the unmatched ref / cand indices. + + ``ref`` and ``cand`` are ``(image, box, cls, ...)``; indices are global. + """ + ref_groups, cand_groups = _group(ref[0]), _group(cand[0]) + out = [] + for name in sorted(set(ref_groups) | set(cand_groups)): + ri, ci = ref_groups.get(name, []), cand_groups.get(name, []) + pairs, miss_r, miss_c = match_boxes( + ref[1][ri].reshape(-1, 4), ref[2][ri], cand[1][ci].reshape(-1, 4), cand[2][ci], min_iou + ) + out.append( + ( + name, + [(ri[a], ci[b]) for a, b in pairs], + [ri[a] for a in miss_r], + [ci[b] for b in miss_c], + ) + ) + return out + + +def pair_items( + ref: tuple[np.ndarray, np.ndarray, np.ndarray], + cand: tuple[np.ndarray, np.ndarray, np.ndarray], + min_iou: float = MATCH_IOU, +) -> np.ndarray: + """``(P, 2)`` global index pairs ``(ref_i, cand_i)`` of items matched by image, box and class.""" + rows = [p for _, pairs, _, _ in _match_by_image(ref, cand, min_iou) for p in pairs] + return np.array(sorted(rows), dtype=np.int64).reshape(-1, 2) + + +def box_metrics( + ref: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], + cand: tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray], + *, + min_iou: float = MATCH_IOU, + identical_iou: float = IDENTICAL_IOU, + score_tolerance: float = SCORE_TOLERANCE, + confident: float = CONFIDENT_SCORE, +) -> dict[str, Any]: + """P-box and P-item numbers from ``(image, box, class, score)`` tuples of two captures.""" + ref_score, cand_score = ref[3], cand[3] + per_image = _match_by_image(ref, cand, min_iou) + ref_cls_by_image = {k: sorted(ref[2][v].tolist()) for k, v in _group(ref[0]).items()} + cand_cls_by_image = {k: sorted(cand[2][v].tolist()) for k, v in _group(cand[0]).items()} + identical = same_items = unmatched_confident = matched = 0 + ious: list[float] = [] + deltas: list[float] = [] + for name, pairs, miss_r, miss_c in per_image: + pair_iou = [float(box_iou(ref[1][r : r + 1], cand[1][c : c + 1])[0, 0]) for r, c in pairs] + pair_delta = [abs(float(ref_score[r]) - float(cand_score[c])) for r, c in pairs] + ious.extend(pair_iou) + deltas.extend(pair_delta) + matched += len(pairs) + unmatched_confident += sum(1 for r in miss_r if ref_score[r] >= confident) + unmatched_confident += sum(1 for c in miss_c if cand_score[c] >= confident) + if ref_cls_by_image.get(name, []) == cand_cls_by_image.get(name, []): + same_items += 1 + if ( + not miss_r + and not miss_c + and all(v >= identical_iou for v in pair_iou) + and all(d <= score_tolerance for d in pair_delta) + ): + identical += 1 + n_images = len(per_image) + return { + 'images': n_images, + 'items_ref': len(ref[0]), + 'items_cand': len(cand[0]), + 'matched': matched, + 'images_identical_share': identical / n_images if n_images else 1.0, + 'item_count_class_identical_share': same_items / n_images if n_images else 1.0, + 'unmatched_confident': unmatched_confident, + 'iou_min': min(ious) if ious else None, + 'iou_mean': float(np.mean(ious)) if ious else None, + 'score_delta_max': max(deltas) if deltas else None, + } + + +# ---------------------------------------------------------------- vectors + + +def cosine_rows(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Row-wise cosine similarity of two ``(n, d)`` arrays (computed in float64).""" + a64, b64 = a.astype(np.float64), b.astype(np.float64) + denom = np.linalg.norm(a64, axis=1) * np.linalg.norm(b64, axis=1) + return np.divide((a64 * b64).sum(axis=1), denom, out=np.zeros(len(a64)), where=denom > 0) + + +def cosine_summary(cos: np.ndarray) -> dict[str, Any]: + if len(cos) == 0: + return {'n': 0} + return { + 'n': len(cos), + 'min': float(cos.min()), + 'mean': float(cos.mean()), + 'p01': float(np.percentile(cos, 1)), + 'median': float(np.median(cos)), + } + + +def _unit_rows(x: np.ndarray) -> np.ndarray: + x32 = x.astype(np.float32) + norm = np.linalg.norm(x32, axis=1, keepdims=True) + return x32 / np.where(norm > 0, norm, 1.0) + + +def knn_indices(x: np.ndarray, queries: np.ndarray, k: int) -> np.ndarray: + """Top-``k`` cosine neighbours (self excluded) of each query row, best first.""" + xn = _unit_rows(x) + sims = xn[queries] @ xn.T + sims[np.arange(len(queries)), queries] = -np.inf + top = np.argpartition(-sims, k - 1, axis=1)[:, :k] + order = np.argsort(-np.take_along_axis(sims, top, axis=1), axis=1, kind='stable') + return np.take_along_axis(top, order, axis=1) + + +def knn_agreement( + ref: np.ndarray, + cand: np.ndarray, + *, + k: int, + n_queries: int, + seed: int, + dup_threshold: float, + labels: np.ndarray | None = None, + truth: tuple[np.ndarray, np.ndarray] | None = None, +) -> dict[str, Any]: + """Neighbour-set agreement of two row-aligned vector sets. + + ``overlap_at_k`` is the mean share of the reference top-k found in the candidate top-k; + ``top1_recall_at_k`` the share of queries whose reference nearest neighbour is in the + candidate top-k; ``near_dup_agreement`` the share of query/neighbour pairs (from either + list) that sit on the same side of ``dup_threshold`` in both spaces. + + With ``labels`` (one per row) it adds ``label_vote_agreement``: the share of queries whose + majority label among the top-k is the same in both spaces (what label propagation would + do). With ``truth`` (reference and candidate ground-truth class per row) it adds the kNN + class precision of each space and ``precision_drop_points``; queries without a ground-truth + class are left out. + """ + n = len(ref) + k = min(k, n - 1) + if k < 1: + return {'queries': 0, 'k': 0} + queries = np.sort(np.random.default_rng(seed).choice(n, size=min(n_queries, n), replace=False)) + ref_nn, cand_nn = knn_indices(ref, queries, k), knn_indices(cand, queries, k) + overlap = [ + len(set(r) & set(c)) / k for r, c in zip(ref_nn.tolist(), cand_nn.tolist(), strict=True) + ] + top1 = [int(r[0] in set(c)) for r, c in zip(ref_nn.tolist(), cand_nn.tolist(), strict=True)] + ref_n, cand_n = _unit_rows(ref), _unit_rows(cand) + qs: list[int] = [] + ns: list[int] = [] + for q, r, c in zip(queries.tolist(), ref_nn.tolist(), cand_nn.tolist(), strict=True): + for m in sorted(set(r) | set(c)): + qs.append(q) + ns.append(m) + q_idx, n_idx = np.array(qs), np.array(ns) + in_ref = (ref_n[q_idx] * ref_n[n_idx]).sum(axis=1) >= dup_threshold + in_cand = (cand_n[q_idx] * cand_n[n_idx]).sum(axis=1) >= dup_threshold + out: dict[str, Any] = { + 'queries': len(queries), + 'k': k, + 'overlap_at_k': float(np.mean(overlap)), + 'top1_recall_at_k': float(np.mean(top1)), + 'near_dup_pairs': len(q_idx), + 'near_dup_agreement': float(np.mean(in_ref == in_cand)), + } + if labels is not None: + votes = [(_vote(labels[r]), _vote(labels[c])) for r, c in zip(ref_nn, cand_nn, strict=True)] + out['label_vote_agreement'] = float(np.mean([a == b for a, b in votes])) + if truth is not None: + p_ref = _neighbour_precision(truth[0], queries, ref_nn) + p_cand = _neighbour_precision(truth[1], queries, cand_nn) + if p_ref is not None and p_cand is not None: + out['precision_ref'] = p_ref + out['precision_cand'] = p_cand + out['precision_drop_points'] = (p_ref - p_cand) * 100.0 + return out + + +def _vote(row: np.ndarray) -> str: + """The most common label of a neighbour list; ties go to the alphabetically first.""" + counts = Counter(row.tolist()) + return min(counts, key=lambda label: (-counts[label], label)) + + +def _neighbour_precision(truth: np.ndarray, queries: np.ndarray, nn: np.ndarray) -> float | None: + """Mean share of a query's neighbours that share its ground-truth class (classed queries only).""" + known = [i for i, q in enumerate(queries.tolist()) if truth[q] not in ('', 'none')] + if not known: + return None + return float(np.mean([np.mean(truth[nn[i]] == truth[queries[i]]) for i in known])) + + +# ---------------------------------------------------------------- clusters + + +def _contingency(a: np.ndarray, b: np.ndarray) -> np.ndarray: + _, ai = np.unique(a, return_inverse=True) + _, bi = np.unique(b, return_inverse=True) + table = np.zeros((ai.max() + 1, bi.max() + 1), dtype=np.int64) + np.add.at(table, (ai, bi), 1) + return table + + +def _comb2(x: np.ndarray | int) -> Any: + return x * (x - 1) / 2 + + +def adjusted_rand_index(a: np.ndarray, b: np.ndarray) -> float: + """Adjusted Rand index of two labellings (invariant to label renaming).""" + table = _contingency(a, b) + n = int(table.sum()) + sum_cells = float(_comb2(table).sum()) + sum_a = float(_comb2(table.sum(axis=1)).sum()) + sum_b = float(_comb2(table.sum(axis=0)).sum()) + expected = sum_a * sum_b / _comb2(n) if n > 1 else 0.0 + maximum = (sum_a + sum_b) / 2 + return 1.0 if maximum == expected else (sum_cells - expected) / (maximum - expected) + + +def _entropy(counts: np.ndarray) -> float: + p = counts[counts > 0] / counts.sum() + return float(-(p * np.log(p)).sum()) + + +def normalized_mutual_info(a: np.ndarray, b: np.ndarray) -> float: + """NMI with the arithmetic-mean normalisation.""" + table = _contingency(a, b).astype(np.float64) + n = table.sum() + pa, pb = table.sum(axis=1), table.sum(axis=0) + nz = table > 0 + outer = np.outer(pa, pb) + mi = float((table[nz] / n * np.log(table[nz] * n / outer[nz])).sum()) + denom = (_entropy(pa) + _entropy(pb)) / 2 + return 1.0 if denom == 0 else mi / denom + + +def purity(labels: np.ndarray, truth: np.ndarray) -> float: + """Size-weighted share of each cluster's modal ground-truth class.""" + if len(labels) == 0: + return 0.0 + table = _contingency(labels, truth) + return float(table.max(axis=1).sum() / table.sum()) + + +def _centroids(vec: np.ndarray, labels: np.ndarray) -> np.ndarray: + unit = _unit_rows(vec) + ids = np.unique(labels) + cents = np.stack([unit[labels == i].mean(axis=0) for i in ids]) + return _unit_rows(cents) + + +def assignment_agreement( + ref_vec: np.ndarray, cand_vec: np.ndarray, ref_labels: np.ndarray +) -> float: + """Share of items whose nearest reference centroid is the same for ref and candidate vectors. + + The reference centroids are the normalised mean of each reference cluster's vectors + (the API does not serve centroids), so this isolates the effect of the vector change + from k-means' own run-to-run variation. + """ + cents = _centroids(ref_vec, ref_labels) + ref_near = np.argmax(_unit_rows(ref_vec) @ cents.T, axis=1) + cand_near = np.argmax(_unit_rows(cand_vec) @ cents.T, axis=1) + return float(np.mean(ref_near == cand_near)) + + +def kmeans_objective(vec: np.ndarray, labels: np.ndarray) -> float: + """Mean cosine distance of each vector to the normalised mean of its own cluster.""" + unit = _unit_rows(vec) + _, inverse = np.unique(labels, return_inverse=True) + cents = _centroids(vec, labels) + return float(np.mean(1.0 - (unit * cents[inverse]).sum(axis=1))) + + +# ---------------------------------------------------------------- strings + + +def agreement_share(a: Sequence[str], b: Sequence[str]) -> float: + """Share of positions where two equal-length string lists agree (1.0 when empty).""" + if not a: + return 1.0 + return sum(x == y for x, y in zip(a, b, strict=True)) / len(a) + + +def _edit_distance(a: str, b: str) -> int: + prev = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + cur = [i] + for j, cb in enumerate(b, 1): + cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) + prev = cur + return prev[-1] + + +def text_agreement( + ref: tuple[np.ndarray, np.ndarray, np.ndarray], + cand: tuple[np.ndarray, np.ndarray, np.ndarray], + min_iou: float = MATCH_IOU, +) -> dict[str, Any]: + """Exact-match share and character error rate of OCR lines matched by box overlap. + + A line present in only one capture counts against ``exact_share``. + + ``ref`` and ``cand`` are ``(image, box, text)``. Lines are matched per image at + ``min_iou`` regardless of their text. + """ + same_kind = np.array(['text'] * len(ref[0])), np.array(['text'] * len(cand[0])) + matches = _match_by_image( + (ref[0], ref[1], same_kind[0]), (cand[0], cand[1], same_kind[1]), min_iou + ) + pairs = [p for _, ps, _, _ in matches for p in ps] + exact = sum(ref[2][r] == cand[2][c] for r, c in pairs) + chars = sum(len(str(ref[2][r])) for r, _ in pairs) + errors = sum(_edit_distance(str(ref[2][r]), str(cand[2][c])) for r, c in pairs) + lines = max(len(ref[0]), len(cand[0])) + return { + 'lines_ref': len(ref[0]), + 'lines_cand': len(cand[0]), + 'matched': len(pairs), + 'exact_share': exact / lines if lines else 1.0, + 'char_error_rate': errors / chars if chars else 0.0, + } diff --git a/scripts/bench/parity_store.py b/scripts/bench/parity_store.py new file mode 100644 index 00000000..da633de1 --- /dev/null +++ b/scripts/bench/parity_store.py @@ -0,0 +1,157 @@ +"""The parity capture file format: one JSON document and one compressed NPZ per capture. + +``capture.json`` holds the schema number, the run metadata (hardware, git sha, image digests, +engine and build record, dataset manifest hash) and the shape and dtype of every array, plus the +SHA-256 of ``arrays.npz``. ``arrays.npz`` holds the per-stage outputs, every array always present +(zero rows when a section was not captured). Vectors are float16 and boxes float32; strings are +unicode. Large files stay outside git; only the small comparison summaries are committed. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy as np + + +if TYPE_CHECKING: + from pathlib import Path + +SCHEMA = 1 +META_FILE = 'capture.json' +ARRAYS_FILE = 'arrays.npz' +REQUIRED_META = ('tag', 'created_at', 'manifest', 'engine', 'environment', 'sections') +SECTIONS = ('items', 'frames', 'clusters', 'vlm', 'clip', 'faces', 'ocr') + +# name -> (dtype, ndim). 'U' means any unicode string dtype. The prefix is the row group. +ARRAY_SPEC: dict[str, tuple[str, int]] = { + # one row per stored item (a detector box with its crop) + 'item_id': ('U', 1), + 'item_image': ('U', 1), + 'item_box': ('float32', 2), + 'item_score': ('float32', 1), + 'item_class': ('U', 1), + 'item_crop_rect': ('int32', 2), + 'item_cluster': ('int32', 1), + 'item_cluster_distance': ('float32', 1), + 'item_cluster_similarity': ('float32', 1), + 'item_cluster_core': ('int8', 1), + 'item_label': ('U', 1), + 'item_label_source': ('U', 1), + 'item_gt': ('U', 1), + 'item_vlm': ('U', 1), + 'item_has_vec': ('bool', 1), + 'item_vec': ('float16', 2), + # one row per image: the whole-frame vector + 'frame_image': ('U', 1), + 'frame_vec': ('float16', 2), + # core-route probes on a sample of images + 'clip_image': ('U', 1), + 'clip_vec': ('float16', 2), + 'face_image': ('U', 1), + 'face_box': ('float32', 2), + 'face_score': ('float32', 1), + 'face_vec': ('float16', 2), + 'ocr_image': ('U', 1), + 'ocr_box': ('float32', 2), + 'ocr_text': ('U', 1), +} + + +class CaptureError(ValueError): + """A capture is malformed, tampered with, or from another schema.""" + + +@dataclass +class Capture: + meta: dict[str, Any] + arrays: dict[str, np.ndarray] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open('rb') as fh: + for block in iter(lambda: fh.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def empty_arrays(dim: int, clip_dim: int = 0, face_dim: int = 0) -> dict[str, np.ndarray]: + """Every array with zero rows; vector widths are ``dim`` (PE), ``clip_dim`` and ``face_dim``.""" + widths = {'item_vec': dim, 'frame_vec': dim, 'clip_vec': clip_dim, 'face_vec': face_dim} + out: dict[str, np.ndarray] = {} + for name, (dtype, ndim) in ARRAY_SPEC.items(): + shape = (0,) if ndim == 1 else (0, widths.get(name, 4)) + out[name] = np.zeros(shape, dtype='U1' if dtype == 'U' else dtype) + return out + + +def validate(arrays: dict[str, np.ndarray]) -> None: + missing = sorted(set(ARRAY_SPEC) - set(arrays)) + extra = sorted(set(arrays) - set(ARRAY_SPEC)) + if missing or extra: + raise CaptureError(f'array set differs from the schema: missing {missing}, extra {extra}') + rows: dict[str, set[int]] = {} + for name, (dtype, ndim) in ARRAY_SPEC.items(): + arr = arrays[name] + kind_ok = arr.dtype.kind == 'U' if dtype == 'U' else arr.dtype == np.dtype(dtype) + if not kind_ok or arr.ndim != ndim: + raise CaptureError( + f'{name}: expected {dtype} with {ndim} dims, got {arr.dtype} {arr.shape}' + ) + rows.setdefault(name.split('_', 1)[0], set()).add(len(arr)) + ragged = sorted(group for group, sizes in rows.items() if len(sizes) > 1) + if ragged: + raise CaptureError(f'row counts differ inside the {ragged} array group') + + +def _counts(arrays: dict[str, np.ndarray]) -> dict[str, int]: + return { + 'items': len(arrays['item_id']), + 'frames': len(arrays['frame_image']), + 'clip': len(arrays['clip_image']), + 'faces': len(arrays['face_image']), + 'ocr': len(arrays['ocr_image']), + } + + +def save_capture(out_dir: Path, capture: Capture) -> None: + """Write ``capture.json`` and ``arrays.npz`` into ``out_dir`` (created if needed).""" + absent = [k for k in REQUIRED_META if k not in capture.meta] + if absent: + raise CaptureError(f'capture metadata is missing {absent}') + validate(capture.arrays) + out_dir.mkdir(parents=True, exist_ok=True) + npz = out_dir / ARRAYS_FILE + np.savez_compressed(npz, **capture.arrays) + doc = { + **capture.meta, + 'schema': SCHEMA, + 'counts': _counts(capture.arrays), + 'arrays': { + k: {'dtype': str(v.dtype), 'shape': list(v.shape)} for k, v in capture.arrays.items() + }, + 'npz_sha256': sha256_file(npz), + } + (out_dir / META_FILE).write_text( + json.dumps(doc, indent=2, sort_keys=True) + '\n', encoding='utf-8' + ) + + +def load_capture(directory: Path) -> Capture: + """Read and verify a capture; raises :class:`CaptureError` on any inconsistency.""" + meta_path, npz_path = directory / META_FILE, directory / ARRAYS_FILE + if not meta_path.is_file() or not npz_path.is_file(): + raise CaptureError(f'{directory}: capture.json or arrays.npz is missing') + meta = json.loads(meta_path.read_text(encoding='utf-8')) + if meta.get('schema') != SCHEMA: + raise CaptureError(f'{directory}: schema {meta.get("schema")}, this tool reads {SCHEMA}') + if meta.get('npz_sha256') != sha256_file(npz_path): + raise CaptureError(f'{directory}: arrays.npz sha256 differs from capture.json') + with np.load(npz_path, allow_pickle=False) as data: + arrays = {name: data[name] for name in data.files} + validate(arrays) + return Capture(meta, arrays) diff --git a/tests/bench/conftest.py b/tests/bench/conftest.py new file mode 100644 index 00000000..ee04fbe9 --- /dev/null +++ b/tests/bench/conftest.py @@ -0,0 +1,102 @@ +"""Synthetic parity captures for the bench tests: planted structure, fixed seed, no stack.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numpy as np +import pytest + +from scripts.bench import parity_store as ps + + +if TYPE_CHECKING: + from collections.abc import Callable + +DIM = 16 +N_IMAGES = 60 +PER_IMAGE = 3 +N_CLUSTERS = 4 + + +def _synthetic(seed: int = 0) -> dict[str, np.ndarray]: + """``N_IMAGES`` images x ``PER_IMAGE`` boxes; vectors follow ``N_CLUSTERS`` planted centres.""" + rng = np.random.default_rng(seed) + n = N_IMAGES * PER_IMAGE + arrays = ps.empty_arrays(DIM) + centres = rng.normal(size=(N_CLUSTERS, DIM)) + centres /= np.linalg.norm(centres, axis=1, keepdims=True) + cluster = rng.integers(0, N_CLUSTERS, size=n) + vec = centres[cluster] + rng.normal(scale=0.02, size=(n, DIM)) + vec /= np.linalg.norm(vec, axis=1, keepdims=True) + slot = np.tile(np.arange(PER_IMAGE), N_IMAGES) + x1 = 0.05 + 0.3 * slot + box = np.stack([x1, np.full(n, 0.1), x1 + 0.25, np.full(n, 0.6)], axis=1) + classes = np.array(['cat', 'dog', 'car', 'bus']) + clip_n, text_n = 10, 12 + clip = rng.normal(size=(clip_n, 8)) + clip /= np.linalg.norm(clip, axis=1, keepdims=True) + face_vec = rng.normal(size=(text_n, 8)) + face_vec /= np.linalg.norm(face_vec, axis=1, keepdims=True) + vlm = np.array([''] * n, dtype='U8') + vlm[:30] = classes[cluster[:30]] + arrays.update( + item_id=np.array([f'it{i}' for i in range(n)]), + item_image=np.repeat([f'img{i:03d}.jpg' for i in range(N_IMAGES)], PER_IMAGE), + item_box=box.astype(np.float32), + item_score=rng.uniform(0.5, 0.99, size=n).astype(np.float32), + item_class=classes[cluster], + item_crop_rect=(box * 100).astype(np.int32), + item_cluster=cluster.astype(np.int32), + item_cluster_distance=rng.uniform(0, 1, size=n).astype(np.float32), + item_cluster_similarity=rng.uniform(0, 1, size=n).astype(np.float32), + item_cluster_core=np.ones(n, dtype=np.int8), + item_label=classes[cluster], + item_label_source=np.array(['detector'] * n), + item_gt=classes[cluster], + item_vlm=vlm, + item_has_vec=np.ones(n, dtype=bool), + item_vec=vec.astype(np.float16), + frame_image=np.array([f'img{i:03d}.jpg' for i in range(N_IMAGES)]), + frame_vec=vec[::PER_IMAGE].astype(np.float16), + clip_image=np.array([f'img{i:03d}.jpg' for i in range(clip_n)]), + clip_vec=clip.astype(np.float16), + face_image=np.array([f'img{i:03d}.jpg' for i in range(text_n)]), + face_box=np.tile(np.array([[0.2, 0.2, 0.4, 0.5]], np.float32), (text_n, 1)), + face_score=np.full(text_n, 0.9, np.float32), + face_vec=face_vec.astype(np.float16), + ocr_image=np.array([f'img{i:03d}.jpg' for i in range(text_n)]), + ocr_box=np.tile(np.array([[0.1, 0.7, 0.9, 0.8]], np.float32), (text_n, 1)), + ocr_text=np.array([f'TEXT {i}' for i in range(text_n)]), + ) + return arrays + + +def _meta(tag: str, engine: str, manifest: str = 'abc123') -> dict[str, Any]: + return { + 'tag': tag, + 'created_at': '2026-10-10T00:00:00+00:00', + 'manifest': {'name': 'm.json', 'sha256': manifest, 'count': N_IMAGES, 'seed': 1}, + 'engine': {'label': engine}, + 'environment': {'git_sha': 'deadbeef'}, + 'project': 'p', + 'sections': list(ps.SECTIONS), + } + + +@pytest.fixture +def make_capture() -> Callable[..., ps.Capture]: + def build( + tag: str = 'ref', + *, + engine: str = 'fp32', + seed: int = 0, + manifest: str = 'abc123', + mutate: Callable[[dict[str, np.ndarray]], None] | None = None, + ) -> ps.Capture: + arrays = _synthetic(seed) + if mutate: + mutate(arrays) + return ps.Capture(_meta(tag, engine, manifest), arrays) + + return build diff --git a/tests/bench/test_parity_assemble.py b/tests/bench/test_parity_assemble.py new file mode 100644 index 00000000..ed3c2b59 --- /dev/null +++ b/tests/bench/test_parity_assemble.py @@ -0,0 +1,135 @@ +"""Ground truth assignment, crop rectangles and the array assembly.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +from scripts.bench import parity_assemble as pa, parity_store as ps + + +if TYPE_CHECKING: + from pathlib import Path + + +def test_crop_rect_rounds_and_keeps_at_least_one_pixel() -> None: + assert pa.crop_rect([0.1, 0.2, 0.5, 0.9], 100, 50) == [10, 10, 50, 45] + assert pa.crop_rect([0.5, 0.5, 0.5, 0.5], 10, 10) == [5, 5, 6, 6] + assert pa.crop_rect([-0.2, 0.0, 0.3, 1.0], 100, 100)[0] == 0 + + +def test_assign_gt_takes_best_overlap_and_none_below_half() -> None: + gt = ( + np.array([[0.0, 0.0, 0.5, 0.5], [0.45, 0.0, 1.0, 0.5]], np.float32), + ['person', 'dog'], + ) + boxes = np.array([[0.0, 0.0, 0.5, 0.5], [0.5, 0.0, 1.0, 0.5], [0.0, 0.6, 0.2, 0.9]], np.float32) + assert pa.assign_gt(boxes, gt) == ['person', 'dog', 'none'] + assert pa.assign_gt(boxes, (np.zeros((0, 4), np.float32), [])) == ['none'] * 3 + + +def _write_coco(root: Path) -> dict: + ann_dir = root / 'annotations' + ann_dir.mkdir() + cats = [{'id': 1, 'name': 'person'}, {'id': 3, 'name': 'car'}] + for split, image_id, cat in (('val2017', 7, 1), ('train2017', 9, 3)): + doc = { + 'categories': cats, + 'annotations': [ + {'image_id': image_id, 'category_id': cat, 'bbox': [10, 20, 30, 40], 'iscrowd': 0}, + {'image_id': image_id, 'category_id': cat, 'bbox': [0, 0, 5, 5], 'iscrowd': 1}, + {'image_id': 999, 'category_id': cat, 'bbox': [0, 0, 5, 5], 'iscrowd': 0}, + ], + } + (ann_dir / f'instances_{split}.json').write_text(json.dumps(doc)) + return { + 'images': [ + {'image_id': 7, 'file_name': 'a.jpg', 'split': 'val2017', 'width': 100, 'height': 200}, + {'image_id': 9, 'file_name': 'b.jpg', 'split': 'train2017', 'width': 50, 'height': 100}, + {'image_id': 11, 'file_name': 'c.jpg', 'split': 'val2017', 'width': 10, 'height': 10}, + ] + } + + +def test_load_gt_normalises_boxes_and_drops_crowd(tmp_path: Path) -> None: + pin = _write_coco(tmp_path) + gt = pa.load_gt(tmp_path, pin) + boxes, names = gt['a.jpg'] + assert names == ['person'] + assert boxes[0].tolist() == pytest.approx([0.1, 0.1, 0.4, 0.3]) + assert gt['b.jpg'][1] == ['car'] + assert gt['c.jpg'][1] == [] + + +def test_load_gt_names_the_missing_file(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match='instances_val2017'): + pa.load_gt(tmp_path, {'images': [{'image_id': 1, 'file_name': 'a', 'split': 'val2017'}]}) + + +def _item(item_id: str, image_id: str, box: list[float], **extra) -> dict: + return { + 'id': item_id, + 'image_id': image_id, + 'bbox_norm': box, + 'confidence': 0.5, + 'detector_confidence': 0.9, + 'detector_class_name': 'cat', + 'class_name': 'cat', + 'cluster_id': None, + **extra, + } + + +def test_build_arrays_sorts_by_file_then_box_and_marks_missing_vectors() -> None: + items = [ + _item('b1', 'ib', [0.5, 0.1, 0.9, 0.5]), + _item('a2', 'ia', [0.5, 0.1, 0.9, 0.5], cluster_id=3, cluster_is_core=True), + _item('a1', 'ia', [0.1, 0.1, 0.4, 0.5], cluster_distance=0.2), + ] + arrays = pa.build_arrays( + items, + image_names={'ia': 'a.jpg', 'ib': 'b.jpg'}, + image_dims={'a.jpg': (100, 100), 'b.jpg': (200, 100)}, + vectors={'a1': [1.0, 0.0], 'a2': [0.0, 1.0]}, + frame_vectors={'ia': [1.0, 1.0]}, + gt=None, + vlm={'a2': 'dog'}, + probes={'ocr': [('b.jpg', [0, 0, 1, 1], 'HI')]}, + ) + ps.validate(arrays) + assert arrays['item_id'].tolist() == ['a1', 'a2', 'b1'] + assert arrays['item_has_vec'].tolist() == [True, True, False] + assert arrays['item_cluster'].tolist() == [-1, 3, -1] + assert arrays['item_cluster_core'].tolist() == [-1, 1, -1] + assert np.isnan(arrays['item_cluster_distance'][1]) + assert arrays['item_vlm'].tolist() == ['', 'dog', ''] + assert arrays['item_gt'].tolist() == ['', '', ''] + assert arrays['frame_image'].tolist() == ['a.jpg'] + assert arrays['ocr_text'].tolist() == ['HI'] + assert arrays['item_vec'].dtype == np.float16 + + +def test_build_arrays_records_ground_truth_per_item() -> None: + gt = {'a.jpg': (np.array([[0.1, 0.1, 0.4, 0.5]], np.float32), ['cat'])} + arrays = pa.build_arrays( + [_item('a1', 'ia', [0.1, 0.1, 0.4, 0.5]), _item('a2', 'ia', [0.6, 0.6, 0.9, 0.9])], + image_names={'ia': 'a.jpg'}, + image_dims={'a.jpg': (10, 10)}, + vectors={}, + frame_vectors={}, + gt=gt, + vlm={}, + probes={}, + ) + assert arrays['item_gt'].tolist() == ['cat', 'none'] + + +def test_build_arrays_empty_capture_is_valid() -> None: + arrays = pa.build_arrays( + [], image_names={}, image_dims={}, vectors={}, frame_vectors={}, gt=None, vlm={}, probes={} + ) + ps.validate(arrays) + assert len(arrays['item_id']) == 0 diff --git a/tests/bench/test_parity_capture.py b/tests/bench/test_parity_capture.py new file mode 100644 index 00000000..dc22c420 --- /dev/null +++ b/tests/bench/test_parity_capture.py @@ -0,0 +1,355 @@ +"""Capture against a fake stack (httpx.MockTransport): routes, paging, vectors, the whole run.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import httpx +import numpy as np +import pytest + +from scripts.bench import parity, parity_capture as cap, parity_store as ps +from scripts.datasets.bench_set import build_pin + + +if TYPE_CHECKING: + from pathlib import Path + +DIM = 4 +OS_URL = 'http://os.test' +API = 'http://api.test' + + +class FakeStack: + """Two items per image on ``n_images`` images, vectors derived from the item index.""" + + def __init__( + self, n_images: int = 5, policy: str = 'all', fail_image: str | None = None + ) -> None: + self.names = [f'{i:012d}.jpg' for i in range(n_images)] + self.policy = policy + self.fail_image = fail_image + self.items: dict[str, dict[str, Any]] = {} + self.vectors: dict[str, list[float]] = {} + self.frames: dict[str, list[float]] = {} + self.calls: list[tuple[str, str]] = [] + self.deleted = False + self.polls = 0 + + def _ingest(self, filename: str) -> dict[str, Any]: + n = self.names.index(filename) + image_id = f'img{n}' + self.frames[image_id] = [float(n), 1.0, 0.0, 0.0] + for k in range(2): + item_id = f'{image_id}_{k}' + self.items[item_id] = { + 'id': item_id, + 'image_id': image_id, + 'bbox_norm': [0.1 + 0.4 * k, 0.1, 0.4 + 0.4 * k, 0.6], + 'confidence': 0.5 + 0.1 * k, + 'detector_confidence': 0.5 + 0.1 * k, + 'detector_class_name': 'cat' if k == 0 else 'dog', + 'class_name': 'cat' if k == 0 else 'dog', + 'class_source': 'detector', + 'cluster_id': 10000 + k, + 'cluster_distance': 0.1, + 'cluster_similarity': 0.9, + 'cluster_is_core': True, + } + self.vectors[item_id] = [float(n), float(k), 1.0, 0.5] + return { + 'status': 'success', + 'image_id': image_id, + 'image_path': f'/data/{n}', + 'source_identifier': filename, + } + + def handler(self, request: httpx.Request) -> httpx.Response: # noqa: PLR0911 + path = request.url.path + self.calls.append((request.method, path)) + if request.method == 'DELETE': + self.deleted = True + return httpx.Response(200, json={}) + if path.endswith('/projects'): + return httpx.Response(200, json={}) + if path.endswith('/ingest/policy'): + return httpx.Response(200, json={'embedding': {'mode': self.policy}}) + if path.endswith('/ingest/upload'): + names = request.content.decode('latin-1').split('filename="')[1:] + results = [] + for chunk in names: + filename = chunk.split('"', 1)[0] + if filename == self.fail_image: + results.append({'status': 'failed', 'image_path': filename, 'error': 'decode'}) + else: + results.append(self._ingest(filename)) + return httpx.Response(200, json={'results': results}) + if path.endswith('/pipeline/auto_label/start'): + return httpx.Response(200, json={'job_id': 'j1'}) + if '/pipeline/auto_label/status/' in path: + return httpx.Response( + 200, json={'status': 'completed', 'stage_durations': {'train': 1.0}} + ) + if path.endswith('/crops'): + return self._crops(request) + if path.endswith('/_mget'): + ids = json.loads(request.content)['ids'] + pool = self.vectors if '__items' in path else self.frames + docs = [ + { + '_id': i, + 'found': i in pool, + '_source': {'pe_embedding': pool[i]} if i in pool else {}, + } + for i in ids + ] + return httpx.Response(200, json={'docs': docs}) + if path.endswith('/embed/image'): + return httpx.Response(200, json={'embedding': [0.0, 1.0]}) + if path.endswith('/faces/recognize'): + return httpx.Response( + 200, + json={ + 'faces': [{'box': [0.1, 0.1, 0.2, 0.3], 'score': 0.9}], + 'embeddings': [[0.5, 0.5]], + }, + ) + if path.endswith('/ocr/predict'): + return httpx.Response(422, json={'detail': 'no text'}) + if path.endswith(('/classes', '/vlm/label_batch')): + return httpx.Response(200, json={}) + # GET project: index names + return httpx.Response( + 200, + json={'resources': {'indexes': {'items': 'op_p__items', 'images': 'op_p__images'}}}, + ) + + def _crops(self, request: httpx.Request) -> httpx.Response: + q = request.url.params + rows = list(self.items.values()) + if q.get('ids'): + wanted = q['ids'].split(',') + rows = [dict(r, vlm_proposed_class_name='cat') for r in rows if r['id'] in wanted] + return httpx.Response(200, json={'total': len(rows), 'crops': rows}) + if 'conf_min' in q: + low, high = float(q['conf_min']), float(q['conf_max']) + rows = [r for r in rows if low <= r['confidence'] <= high] + size, page = int(q.get('page_size', 50)), int(q.get('page', 1)) + return httpx.Response( + 200, json={'total': len(rows), 'crops': rows[(page - 1) * size : page * size]} + ) + + +def _client(stack: FakeStack) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(stack.handler)) + + +def test_fetch_items_pages_through_a_single_band() -> None: + stack = FakeStack() + for n in range(5): + stack._ingest(stack.names[n]) + with _client(stack) as client: + items = cap.fetch_items(client, f'{API}/p', page_size=3) + assert len(items) == 10 + + +def test_fetch_items_splits_bands_when_the_window_is_exceeded() -> None: + stack = FakeStack() + for n in range(5): + stack._ingest(stack.names[n]) + with _client(stack) as client: + items = cap.fetch_items(client, f'{API}/p', window=6, page_size=4) + assert sorted(i['id'] for i in items) == sorted(stack.items) + + +def test_fetch_items_aborts_when_the_union_misses_rows() -> None: + stack = FakeStack() + for n in range(5): + stack._ingest(stack.names[n]) + stack.items['ghost'] = {**stack.items['img0_0'], 'id': 'ghost', 'confidence': 1.7} + with _client(stack) as client, pytest.raises(ps.CaptureError, match='reports'): + cap.fetch_items(client, f'{API}/p') + + +def test_mget_vectors_skips_documents_without_one() -> None: + stack = FakeStack() + stack._ingest(stack.names[0]) + with _client(stack) as client: + got = cap.mget_vectors(client, OS_URL, 'op_p__items', ['img0_0', 'missing']) + assert list(got) == ['img0_0'] + + +def test_ingest_images_maps_ids_to_file_names(tmp_path: Path) -> None: + stack = FakeStack(n_images=3) + files = [] + for name in stack.names: + (tmp_path / name).write_bytes(b'x') + files.append(tmp_path / name) + with _client(stack) as client: + names = cap.ingest_images(client, f'{API}/p', files, batch_size=2, concurrency=2) + assert names == {f'img{i}': stack.names[i] for i in range(3)} + + +def test_ingest_images_aborts_on_a_failed_image(tmp_path: Path) -> None: + stack = FakeStack(n_images=2, fail_image='000000000001.jpg') + files = [] + for name in stack.names: + (tmp_path / name).write_bytes(b'x') + files.append(tmp_path / name) + with _client(stack) as client, pytest.raises(ps.CaptureError, match='failed to ingest'): + cap.ingest_images(client, f'{API}/p', files, batch_size=8, concurrency=1) + + +def test_policy_other_than_all_is_refused() -> None: + with _client(FakeStack(policy='lazy')) as client, pytest.raises(ps.CaptureError, match='all'): + cap.require_embed_all(client, f'{API}/p') + + +def test_probe_core_treats_422_as_no_rows(tmp_path: Path) -> None: + path = tmp_path / 'a.jpg' + path.write_bytes(b'x') + with _client(FakeStack()) as client: + out = cap.probe_core(client, API, [path], concurrency=1) + assert out['clip'] == [('a.jpg', [0.0, 1.0])] + assert out['faces'] == [('a.jpg', [0.1, 0.1, 0.2, 0.3], 0.9, [0.5, 0.5])] + assert out['ocr'] == [] + + +def test_vlm_sample_depends_only_on_file_names() -> None: + items = [{'id': f'{n}_{k}', 'image_id': f'i{n}'} for n in range(20) for k in range(2)] + a = cap.vlm_sample(items, {f'i{n}': f'{n}.jpg' for n in range(20)}, 10) + shuffled = list(reversed(items)) + b = cap.vlm_sample(shuffled, {f'i{n}': f'{n}.jpg' for n in range(20)}, 10) + assert a == b + assert len(a) == 10 + + +def _write_pin(tmp_path: Path, stack: FakeStack) -> tuple[Path, Path]: + images = tmp_path / 'images' + images.mkdir() + rows = [] + for i, name in enumerate(stack.names): + (images / name).write_bytes(b'jpeg') + rows.append( + { + 'image_id': i, + 'file_name': name, + 'split': 'val2017', + 'license_name': 'x', + 'flickr_url': '', + 'width': 100, + 'height': 80, + 'bytes': 4, + 'sha256': 'x', + } + ) + manifest = tmp_path / 'pin.json' + manifest.write_text( + json.dumps(build_pin(rows, seed=1, licenses=['x'], annotations_sha256='abc')) + ) + return manifest, images + + +def _args(manifest: Path, images: Path, out: Path, **extra: Any) -> Any: + ns = parity.build_parser().parse_args( + [ + 'capture', + '--tag', + 'unit', + '--manifest', + str(manifest), + '--images', + str(images), + '--api-url', + API, + '--triton-url', + 'http://triton.test', + '--opensearch-url', + OS_URL, + '--compose-project', + 'w0par', + '--engine-label', + 'fp16', + '--engine-record', + 'precision=fp16', + '--out', + str(out), + '--probe-images', + '2', + ] + ) + for key, value in extra.items(): + setattr(ns, key, value) + return ns + + +@pytest.fixture +def no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr('scripts.bench.baseline_suite.time.sleep', lambda _s: None) + monkeypatch.setattr( + parity, 'environment_record', lambda *_a, **_k: {'git_sha': 'abc', 'hardware': {}} + ) + + +def test_run_capture_end_to_end(tmp_path: Path, no_sleep: None) -> None: + stack = FakeStack() + manifest, images = _write_pin(tmp_path, stack) + capture = parity.run_capture( + _args(manifest, images, tmp_path / 'out'), transport=httpx.MockTransport(stack.handler) + ) + a = capture.arrays + assert len(a['item_id']) == 10 + assert a['item_has_vec'].all() + assert a['item_vec'].shape == (10, DIM) + assert a['item_vec'].dtype == np.float16 + assert a['frame_vec'].shape == (5, DIM) + assert a['item_image'][0] == '000000000000.jpg' + assert a['item_crop_rect'][0].tolist() == [10, 8, 40, 48] + assert capture.meta['sections'] == ['items', 'frames', 'clusters', 'clip', 'faces', 'ocr'] + assert capture.meta['manifest']['count'] == 5 + assert capture.meta['engine'] == {'label': 'fp16', 'precision': 'fp16'} + assert stack.deleted is True + ps.validate(a) + + +def test_run_capture_keeps_the_project_on_request(tmp_path: Path, no_sleep: None) -> None: + stack = FakeStack() + manifest, images = _write_pin(tmp_path, stack) + args = _args(manifest, images, tmp_path / 'out', keep_project=True) + parity.run_capture(args, transport=httpx.MockTransport(stack.handler)) + assert stack.deleted is False + + +def test_run_capture_removes_the_project_when_ingest_fails(tmp_path: Path, no_sleep: None) -> None: + stack = FakeStack(fail_image='000000000002.jpg') + manifest, images = _write_pin(tmp_path, stack) + with pytest.raises(ps.CaptureError): + parity.run_capture( + _args(manifest, images, tmp_path / 'out'), transport=httpx.MockTransport(stack.handler) + ) + assert stack.deleted is True + + +def test_run_capture_refuses_missing_images(tmp_path: Path, no_sleep: None) -> None: + stack = FakeStack() + manifest, images = _write_pin(tmp_path, stack) + (images / stack.names[0]).unlink() + with pytest.raises(ps.CaptureError, match='missing'): + parity.run_capture( + _args(manifest, images, tmp_path / 'out'), transport=httpx.MockTransport(stack.handler) + ) + + +def test_urls_use_the_configured_api_prefix(tmp_path: Path, no_sleep: None) -> None: + from scripts.curation._project_worker_utils import curation_api_prefix + + stack = FakeStack() + manifest, images = _write_pin(tmp_path, stack) + parity.run_capture( + _args(manifest, images, tmp_path / 'out'), transport=httpx.MockTransport(stack.handler) + ) + prefix = curation_api_prefix() + curation_calls = [p for _, p in stack.calls if '/projects' in p] + assert curation_calls + assert all(p.startswith(prefix + '/projects') for p in curation_calls) diff --git a/tests/bench/test_parity_cli.py b/tests/bench/test_parity_cli.py new file mode 100644 index 00000000..d4e39587 --- /dev/null +++ b/tests/bench/test_parity_cli.py @@ -0,0 +1,159 @@ +"""The compare CLI end to end on stored synthetic captures, and the environment record helpers.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +from scripts.bench import parity, parity_env, parity_store as ps + + +if TYPE_CHECKING: + from pathlib import Path + + +def _store(tmp_path: Path, make_capture, tag: str, **kw) -> Path: + path = tmp_path / tag + ps.save_capture(path, make_capture(tag, **kw)) + return path + + +def test_compare_passes_and_writes_the_summary(make_capture, tmp_path: Path, capsys) -> None: + ref = _store(tmp_path, make_capture, 'ref', engine='fp32') + cand = _store(tmp_path, make_capture, 'cand', engine='fp16') + out = tmp_path / 'summary' + code = parity.main(['compare', str(ref), str(cand), '--out', str(out)]) + assert code == 0 + assert 'PASS' in capsys.readouterr().out + doc = json.loads((out / 'parity_summary.json').read_text()) + assert doc['evaluation']['passed'] is True + assert doc['reference']['engine']['label'] == 'fp32' + assert doc['metrics']['boxes']['images_identical_share'] == 1.0 + assert (out / 'parity_summary.md').is_file() + + +def test_compare_exits_one_on_a_failed_gate(make_capture, tmp_path: Path) -> None: + def scramble(a: dict[str, np.ndarray]) -> None: + a['item_vec'] = np.random.default_rng(0).normal(size=a['item_vec'].shape).astype(np.float16) + + ref = _store(tmp_path, make_capture, 'ref') + cand = _store(tmp_path, make_capture, 'cand', mutate=scramble) + assert parity.main(['compare', str(ref), str(cand)]) == 1 + + +def test_compare_exits_two_when_inputs_cannot_be_compared( + make_capture, tmp_path: Path, capsys +) -> None: + ref = _store(tmp_path, make_capture, 'ref') + other = _store(tmp_path, make_capture, 'other', manifest='different') + assert parity.main(['compare', str(ref), str(other)]) == 2 + assert 'manifest' in capsys.readouterr().err + assert parity.main(['compare', str(ref), str(tmp_path / 'nope')]) == 2 + + +def test_compare_refuses_captures_of_a_different_image_count(make_capture, tmp_path: Path) -> None: + ref = _store(tmp_path, make_capture, 'ref') + cand_dir = _store(tmp_path, make_capture, 'cand') + meta = json.loads((cand_dir / 'capture.json').read_text()) + meta['manifest']['count'] = 10 + (cand_dir / 'capture.json').write_text(json.dumps(meta)) + assert parity.main(['compare', str(ref), str(cand_dir)]) == 2 + + +def test_compare_strict_fails_on_skips(make_capture, tmp_path: Path) -> None: + ref = _store(tmp_path, make_capture, 'ref') + cand = make_capture('cand') + cand.meta['sections'] = ['items', 'frames'] + ps.save_capture(tmp_path / 'cand', cand) + assert parity.main(['compare', str(ref), str(tmp_path / 'cand')]) == 0 + assert parity.main(['compare', str(ref), str(tmp_path / 'cand'), '--strict']) == 1 + + +def test_capture_refuses_to_overwrite_a_tag(tmp_path: Path, capsys) -> None: + (tmp_path / 'fp16').mkdir() + (tmp_path / 'fp16' / 'capture.json').write_text('{}') + code = parity.main( + [ + 'capture', '--tag', 'fp16', '--images', str(tmp_path), '--api-url', 'http://x', + '--triton-url', 'http://x', '--opensearch-url', 'http://x', '--compose-project', 'p', + '--engine-label', 'fp16', '--out', str(tmp_path), + ] + ) # fmt: skip + assert code == 2 + assert 'already holds a capture' in capsys.readouterr().err + + +def test_vlm_crops_need_classes(tmp_path: Path, capsys) -> None: + code = parity.main( + [ + 'capture', '--tag', 't', '--images', str(tmp_path), '--api-url', 'http://x', + '--triton-url', 'http://x', '--opensearch-url', 'http://x', '--compose-project', 'p', + '--engine-label', 'fp16', '--vlm-crops', '5', + ] + ) # fmt: skip + assert code == 2 + assert '--vlm-classes' in capsys.readouterr().err + + +def test_engine_record_hashes_the_plan_and_parses_pairs(tmp_path: Path) -> None: + plan = tmp_path / 'model.plan' + plan.write_bytes(b'abc') + rec = parity_env.engine_record('fp16', plan, ['precision=fp16', 'trt=11.1']) + assert rec['plan_bytes'] == 3 + assert rec['plan_sha256'] == 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad' + assert rec['trt'] == '11.1' + assert parity_env.engine_record('fp32', None, []) == {'label': 'fp32'} + + +def test_engine_record_rejects_a_bare_word(tmp_path: Path) -> None: + with pytest.raises(ValueError, match='KEY=VALUE'): + parity_env.engine_record('x', None, ['oops']) + + +def test_parse_inspect_reads_name_image_and_id() -> None: + text = '/w0par-api repo/api:1 sha256:aa\n/w0par-triton triton:2 sha256:bb\nnoise\n' + assert parity_env.parse_inspect(text) == { + 'w0par-api': {'image': 'repo/api:1', 'image_id': 'sha256:aa'}, + 'w0par-triton': {'image': 'triton:2', 'image_id': 'sha256:bb'}, + } + + +def test_image_digests_filters_by_compose_label(monkeypatch: pytest.MonkeyPatch) -> None: + seen: list[list[str]] = [] + + def fake(cmd: list[str], timeout: float = 30.0) -> str: + seen.append(cmd) + return 'w0par-api\n' if cmd[1] == 'ps' else '/w0par-api img sha256:cc\n' + + monkeypatch.setattr(parity_env, 'run_cmd', fake) + out = parity_env.image_digests('w0par') + assert 'label=com.docker.compose.project=w0par' in seen[0] + assert out == {'w0par-api': {'image': 'img', 'image_id': 'sha256:cc'}} + + +def test_http_errors_are_reported_with_the_response_body( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys +) -> None: + import httpx + + def boom(_args) -> int: + request = httpx.Request('POST', 'http://x/ingest') + raise httpx.HTTPStatusError( + 'bad', + request=request, + response=httpx.Response(503, text='opensearch unavailable', request=request), + ) + + monkeypatch.setattr(parity, 'cmd_capture', boom) + code = parity.main( + [ + 'capture', '--tag', 't', '--images', str(tmp_path), '--api-url', 'http://x', + '--triton-url', 'http://x', '--opensearch-url', 'http://x', '--compose-project', 'p', + '--engine-label', 'fp16', + ] + ) # fmt: skip + assert code == 2 + assert '503: opensearch unavailable' in capsys.readouterr().err diff --git a/tests/bench/test_parity_gates.py b/tests/bench/test_parity_gates.py new file mode 100644 index 00000000..5fffcd47 --- /dev/null +++ b/tests/bench/test_parity_gates.py @@ -0,0 +1,221 @@ +"""compare: metrics between two synthetic captures, gate evaluation, exit-code semantics.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest + +from scripts.bench import parity_gates as pg, parity_store as ps + + +GATES = Path(pg.__file__).with_name('parity_gates.json') + + +def _evaluate(ref, cand, profile: str = 'precision') -> dict: + gates = pg.load_gates(GATES) + metrics = pg.compute_metrics(ref, cand, gates['settings']) + return pg.evaluate(metrics, gates, profile) + + +def _status(result: dict) -> dict[str, str]: + return {c['id']: c['status'] for c in result['checks']} + + +def _noise(scale: float, seed: int = 9): + def apply(a: dict[str, np.ndarray]) -> None: + rng = np.random.default_rng(seed) + v = a['item_vec'].astype(np.float32) + v += rng.normal(scale=scale, size=v.shape).astype(np.float32) + a['item_vec'] = v.astype(np.float16) + f = a['frame_vec'].astype(np.float32) + f += rng.normal(scale=scale, size=f.shape).astype(np.float32) + a['frame_vec'] = f.astype(np.float16) + + return apply + + +def test_a_capture_against_itself_passes_every_gate(make_capture) -> None: + result = _evaluate(make_capture('ref'), make_capture('cand')) + assert result['failed'] == 0 + assert result['skipped'] == 0 + assert result['passed'] is True + assert result['recorded']['pe_crop.identical_share'] == 1.0 + + +def test_half_precision_noise_passes_and_is_visible_in_the_recorded_metrics(make_capture) -> None: + ref = make_capture('ref') + cand = make_capture('cand', mutate=_noise(0.0005)) + result = _evaluate(ref, cand) + assert result['passed'] is True + assert result['recorded']['pe_crop.identical_share'] < 0.5 + + +def test_heavy_vector_noise_fails_the_cosine_and_knn_gates(make_capture) -> None: + ref = make_capture('ref') + cand = make_capture('cand', mutate=_noise(0.5)) + status = _status(_evaluate(ref, cand)) + assert status['P-pe-p01'] == 'FAIL' + assert status['P-knn'] == 'FAIL' + assert status['P-cluster-assign'] == 'FAIL' + + +def test_shifted_boxes_fail_the_box_gate_and_change_the_crop_rectangles(make_capture) -> None: + def shift(a: dict[str, np.ndarray]) -> None: + a['item_box'] = a['item_box'] + np.float32([0.04, 0, 0.04, 0]) + a['item_crop_rect'] = a['item_crop_rect'] + 4 + + result = _evaluate(make_capture('ref'), make_capture('cand', mutate=shift)) + assert _status(result)['P-box'] == 'FAIL' + assert _status(result)['P-item'] == 'PASS' + assert result['recorded']['boxes.crop_rect_identical_share'] == 0.0 + + +def test_a_class_flip_is_not_hidden_by_box_overlap(make_capture) -> None: + def flip(a: dict[str, np.ndarray]) -> None: + a['item_class'] = a['item_class'].copy() + a['item_class'][:20] = 'zebra' + + status = _status(_evaluate(make_capture('ref'), make_capture('cand', mutate=flip))) + assert status['P-item'] == 'FAIL' + assert status['P-box-confident'] == 'FAIL' + + +def test_a_permuted_cluster_labelling_still_passes(make_capture) -> None: + def permute(a: dict[str, np.ndarray]) -> None: + a['item_cluster'] = ((a['item_cluster'] + 2) % 4).astype(np.int32) + + result = _evaluate(make_capture('ref'), make_capture('cand', mutate=permute)) + status = _status(result) + assert status['P-cluster-assign'] == 'PASS' + assert status['P-cluster-purity'] == 'PASS' + assert result['recorded']['cluster.ari'] == pytest.approx(1.0) + + +def test_random_clusters_lose_purity(make_capture) -> None: + def scramble(a: dict[str, np.ndarray]) -> None: + a['item_cluster'] = ( + np.random.default_rng(3).integers(0, 4, size=len(a['item_id'])).astype(np.int32) + ) + + result = _evaluate(make_capture('ref'), make_capture('cand', mutate=scramble)) + assert _status(result)['P-cluster-purity'] == 'FAIL' + assert result['recorded']['cluster.ari'] < 0.2 + + +def test_label_and_vlm_disagreement_fail_their_gates(make_capture) -> None: + def relabel(a: dict[str, np.ndarray]) -> None: + a['item_label'] = a['item_label'].copy() + a['item_label'][:9] = 'x' + a['item_vlm'] = a['item_vlm'].copy() + a['item_vlm'][:5] = 'y' + + status = _status(_evaluate(make_capture('ref'), make_capture('cand', mutate=relabel))) + assert status['P-label'] == 'FAIL' + assert status['P-vlm'] == 'FAIL' + + +def test_ocr_and_face_changes_are_gated(make_capture) -> None: + def change(a: dict[str, np.ndarray]) -> None: + a['ocr_text'] = np.array(['changed'] * len(a['ocr_text'])) + a['face_vec'] = (-a['face_vec'].astype(np.float32)).astype(np.float16) + a['clip_vec'] = (-a['clip_vec'].astype(np.float32)).astype(np.float16) + + status = _status(_evaluate(make_capture('ref'), make_capture('cand', mutate=change))) + assert status['P-ocr'] == 'FAIL' + assert status['P-face'] == 'FAIL' + assert status['P-clip'] == 'FAIL' + + +def test_missing_sections_are_skipped_not_failed(make_capture) -> None: + ref = make_capture('ref') + cand = make_capture('cand') + cand.arrays.update({k: v[:0] for k, v in cand.arrays.items() if k.startswith(('clip', 'ocr'))}) + cand.meta['sections'] = [s for s in cand.meta['sections'] if s not in ('clip', 'ocr')] + result = _evaluate(ref, cand) + status = _status(result) + assert status['P-clip'] == 'SKIP' + assert status['P-ocr'] == 'SKIP' + assert result['passed'] is True + assert pg.exit_code(result, strict=True) == 1 + assert pg.exit_code(result, strict=False) == 0 + + +def test_a_section_that_vanished_from_a_captured_stage_fails(make_capture) -> None: + cand = make_capture('cand') + cand.arrays.update({k: v[:0] for k, v in cand.arrays.items() if k.startswith('ocr')}) + status = _status(_evaluate(make_capture('ref'), cand)) + assert status['P-ocr'] == 'FAIL' + + +def test_vlm_label_present_in_only_one_capture_disagrees(make_capture) -> None: + def drop(a: dict[str, np.ndarray]) -> None: + a['item_vlm'] = np.array([''] * len(a['item_vlm']), dtype='U8') + + status = _status(_evaluate(make_capture('ref'), make_capture('cand', mutate=drop))) + assert status['P-vlm'] == 'FAIL' + + +def test_exit_code_is_nonzero_on_failure(make_capture) -> None: + result = _evaluate(make_capture('ref'), make_capture('cand', mutate=_noise(0.5))) + assert pg.exit_code(result, strict=False) == 1 + + +def test_different_manifests_cannot_be_compared(make_capture) -> None: + with pytest.raises(ps.CaptureError, match='manifest'): + pg.compute_metrics(make_capture('a'), make_capture('b', manifest='other'), {}) + + +def test_unknown_operator_and_metric_shape_are_rejected(tmp_path: Path) -> None: + doc = json.loads(GATES.read_text()) + doc['profiles']['precision']['checks'][0]['op'] = '~=' + bad = tmp_path / 'g.json' + bad.write_text(json.dumps(doc)) + with pytest.raises(ValueError, match='operator'): + pg.load_gates(bad) + + +def test_shipped_gate_ids_are_unique_and_follow_the_plan() -> None: + gates = pg.load_gates(GATES) + ids = [c['id'] for c in gates['profiles']['precision']['checks']] + assert len(ids) == len(set(ids)) + by_id = {c['id']: c for c in gates['profiles']['precision']['checks']} + assert by_id['P-pe-p01']['value'] == 0.995 + assert by_id['P-pe-median']['value'] == 0.999 + assert by_id['P-cluster-assign']['value'] == 0.98 + assert by_id['P-cluster-purity']['value'] == 0.5 + assert by_id['P-knn']['value'] == 0.95 + + +def test_markdown_lists_every_check_and_the_verdict(make_capture) -> None: + ref, cand = make_capture('ref', engine='fp32'), make_capture('cand', engine='fp16') + gates = pg.load_gates(GATES) + metrics = pg.compute_metrics(ref, cand, gates['settings']) + result = pg.evaluate(metrics, gates, 'precision') + text = pg.render_markdown(ref.meta, cand.meta, metrics, result) + assert 'PASS' in text + assert 'fp32' in text + assert 'fp16' in text + assert all(c['id'] in text for c in result['checks']) + + +def test_objective_difference_is_relative_to_the_reference(make_capture) -> None: + ref = make_capture('ref') + cand = make_capture('cand', mutate=_noise(0.05)) + metrics = pg.compute_metrics(ref, cand, pg.load_gates(GATES)['settings'])['cluster'] + expected = (metrics['objective_cand'] - metrics['objective_ref']) / metrics['objective_ref'] + assert metrics['objective_rel_diff'] == pytest.approx(expected) + assert metrics['objective_rel_diff'] > 0.1 + + +def test_cluster_geometry_deltas_are_reported(make_capture) -> None: + def shift(a: dict[str, np.ndarray]) -> None: + a['item_cluster_similarity'] = a['item_cluster_similarity'] - np.float32(0.1) + a['item_cluster_core'] = np.zeros_like(a['item_cluster_core']) + + ref, cand = make_capture('ref'), make_capture('cand', mutate=shift) + cluster = pg.compute_metrics(ref, cand, pg.load_gates(GATES)['settings'])['cluster'] + assert cluster['similarity_abs_delta_mean'] == pytest.approx(0.1, abs=1e-5) + assert cluster['core_agreement'] == 0.0 diff --git a/tests/bench/test_parity_metrics.py b/tests/bench/test_parity_metrics.py new file mode 100644 index 00000000..ceb291c0 --- /dev/null +++ b/tests/bench/test_parity_metrics.py @@ -0,0 +1,289 @@ +"""Parity metrics on synthetic boxes, vectors, labellings and strings (pure logic, no stack).""" + +from __future__ import annotations + +import numpy as np +import pytest + +from scripts.bench import parity_metrics as pm + + +def _unit(rng: np.random.Generator, n: int, d: int) -> np.ndarray: + x = rng.normal(size=(n, d)).astype(np.float32) + return x / np.linalg.norm(x, axis=1, keepdims=True) + + +# ---------------------------------------------------------------- boxes + + +def test_iou_known_values() -> None: + a = np.array([[0.0, 0.0, 0.5, 0.5]], dtype=np.float32) + b = np.array([[0.0, 0.0, 0.5, 0.5], [0.25, 0.0, 0.75, 0.5], [0.6, 0.6, 0.9, 0.9]], np.float32) + iou = pm.box_iou(a, b) + assert iou[0, 0] == pytest.approx(1.0) + assert iou[0, 1] == pytest.approx(1 / 3) + assert iou[0, 2] == 0.0 + + +def test_match_boxes_is_greedy_within_class() -> None: + ref = np.array([[0.0, 0.0, 0.4, 0.4], [0.5, 0.5, 0.9, 0.9]], np.float32) + cand = np.array([[0.01, 0.0, 0.4, 0.4], [0.5, 0.5, 0.9, 0.9]], np.float32) + pairs, miss_ref, miss_cand = pm.match_boxes(ref, ['a', 'b'], cand, ['a', 'b']) + assert sorted(pairs) == [(0, 0), (1, 1)] + assert not miss_ref + assert not miss_cand + + +def test_match_boxes_never_crosses_classes() -> None: + box = np.array([[0.1, 0.1, 0.6, 0.6]], np.float32) + pairs, miss_ref, miss_cand = pm.match_boxes(box, ['cat'], box, ['dog']) + assert pairs == [] + assert miss_ref == [0] + assert miss_cand == [0] + + +def test_match_boxes_takes_the_best_overlap_first() -> None: + ref = np.array([[0.0, 0.0, 0.5, 0.5]], np.float32) + cand = np.array([[0.0, 0.0, 0.4, 0.5], [0.0, 0.0, 0.5, 0.5]], np.float32) + pairs, _, miss_cand = pm.match_boxes(ref, ['a'], cand, ['a', 'a']) + assert pairs == [(0, 1)] + assert miss_cand == [0] + + +def _items(rows: list[tuple[str, str, float, float, float, float, float]]) -> dict: + return { + 'image': np.array([r[0] for r in rows]), + 'cls': np.array([r[1] for r in rows]), + 'box': np.array([r[3:7] for r in rows], np.float32).reshape(-1, 4), + 'score': np.array([r[2] for r in rows], np.float32), + } + + +def _box_metrics(ref: dict, cand: dict) -> dict: + return pm.box_metrics( + (ref['image'], ref['box'], ref['cls'], ref['score']), + (cand['image'], cand['box'], cand['cls'], cand['score']), + ) + + +def test_box_metrics_identical_sets() -> None: + ref = _items([('a', 'car', 0.9, 0.1, 0.1, 0.5, 0.5), ('b', 'dog', 0.8, 0.2, 0.2, 0.9, 0.9)]) + out = _box_metrics(ref, ref) + assert out['images_identical_share'] == 1.0 + assert out['unmatched_confident'] == 0 + assert out['item_count_class_identical_share'] == 1.0 + + +def test_box_metrics_shifted_box_breaks_identity_but_still_matches() -> None: + ref = _items([('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5), ('b', 'dog', 0.8, 0.2, 0.2, 0.9, 0.9)]) + cand = _items([('a', 'car', 0.9, 0.05, 0.0, 0.55, 0.5), ('b', 'dog', 0.8, 0.2, 0.2, 0.9, 0.9)]) + out = _box_metrics(ref, cand) + assert out['images_identical_share'] == 0.5 + assert out['unmatched_confident'] == 0 + assert out['iou_min'] < 0.95 + + +def test_box_metrics_score_delta_breaks_identity() -> None: + ref = _items([('a', 'car', 0.90, 0.0, 0.0, 0.5, 0.5)]) + cand = _items([('a', 'car', 0.95, 0.0, 0.0, 0.5, 0.5)]) + assert _box_metrics(ref, cand)['images_identical_share'] == 0.0 + + +def test_box_metrics_counts_confident_unmatched_only() -> None: + ref = _items([('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5), ('a', 'dog', 0.3, 0.6, 0.6, 0.9, 0.9)]) + cand = _items([('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5)]) + out = _box_metrics(ref, cand) + assert out['unmatched_confident'] == 0 # the lost box scored 0.3 + cand_extra = _items( + [('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5), ('a', 'cat', 0.8, 0.6, 0.6, 0.9, 0.9)] + ) + assert _box_metrics(ref, cand_extra)['unmatched_confident'] == 1 + + +def test_box_metrics_class_change_is_a_class_mismatch() -> None: + ref = _items([('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5)]) + cand = _items([('a', 'truck', 0.9, 0.0, 0.0, 0.5, 0.5)]) + out = _box_metrics(ref, cand) + assert out['item_count_class_identical_share'] == 0.0 + assert out['unmatched_confident'] == 2 + + +def test_pair_images_pairs_are_global_indices() -> None: + ref = _items([('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5), ('b', 'dog', 0.8, 0.2, 0.2, 0.9, 0.9)]) + cand = _items([('b', 'dog', 0.8, 0.2, 0.2, 0.9, 0.9), ('a', 'car', 0.9, 0.0, 0.0, 0.5, 0.5)]) + pairs = pm.pair_items( + (ref['image'], ref['box'], ref['cls']), (cand['image'], cand['box'], cand['cls']) + ) + assert pairs.tolist() == [[0, 1], [1, 0]] + + +# ---------------------------------------------------------------- vectors + + +def test_cosine_summary_of_noisy_copy() -> None: + rng = np.random.default_rng(0) + a = _unit(rng, 500, 32) + b = a + rng.normal(scale=0.01, size=a.shape).astype(np.float32) + s = pm.cosine_summary(pm.cosine_rows(a, b)) + assert s['n'] == 500 + assert s['min'] <= s['p01'] <= s['median'] <= 1.0 + assert s['median'] > 0.99 + assert pm.cosine_summary(pm.cosine_rows(a, a))['min'] == pytest.approx(1.0, abs=1e-6) + + +def test_cosine_summary_detects_a_swapped_row() -> None: + rng = np.random.default_rng(1) + a = _unit(rng, 100, 32) + b = a.copy() + b[7] = -a[7] + assert pm.cosine_summary(pm.cosine_rows(a, b))['min'] == pytest.approx(-1.0, abs=1e-5) + + +def test_knn_agreement_identical_and_noisy() -> None: + rng = np.random.default_rng(2) + a = _unit(rng, 300, 24) + same = pm.knn_agreement(a, a, k=10, n_queries=100, seed=3, dup_threshold=0.98) + assert same['overlap_at_k'] == 1.0 + assert same['top1_recall_at_k'] == 1.0 + assert same['near_dup_agreement'] == 1.0 + b = _unit(rng, 300, 24) # unrelated space + other = pm.knn_agreement(a, b, k=10, n_queries=100, seed=3, dup_threshold=0.98) + assert other['overlap_at_k'] < 0.2 + + +def test_knn_agreement_excludes_the_query_itself() -> None: + a = np.eye(4, dtype=np.float32) + got = pm.knn_indices(a, np.array([0]), k=3) + assert 0 not in got[0] + + +def test_near_dup_agreement_sees_a_threshold_crossing() -> None: + base = np.array([[1.0, 0.0], [0.99, 0.141], [0.0, 1.0]], np.float32) + base /= np.linalg.norm(base, axis=1, keepdims=True) + moved = base.copy() + moved[1] = np.array([0.8, 0.6], np.float32) # no longer a near duplicate of row 0 + out = pm.knn_agreement(base, moved, k=2, n_queries=3, seed=0, dup_threshold=0.98) + assert out['near_dup_agreement'] < 1.0 + + +# ---------------------------------------------------------------- clusters + + +def test_ari_is_label_permutation_invariant() -> None: + a = np.array([0, 0, 1, 1, 2, 2, 2]) + perm = np.array([5, 5, 3, 3, 9, 9, 9]) + assert pm.adjusted_rand_index(a, perm) == pytest.approx(1.0) + assert pm.normalized_mutual_info(a, perm) == pytest.approx(1.0) + + +def test_ari_penalises_a_split_cluster() -> None: + a = np.array([0, 0, 0, 0, 1, 1, 1, 1]) + b = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + assert pm.adjusted_rand_index(a, b) < 0.5 + assert 0.0 < pm.normalized_mutual_info(a, b) < 1.0 + + +def test_ari_known_value() -> None: + a = np.array([0, 0, 1, 1]) + b = np.array([0, 0, 1, 2]) + assert pm.adjusted_rand_index(a, b) == pytest.approx(0.5714285714) + + +def test_purity_weighted_by_cluster_size() -> None: + labels = np.array([0, 0, 0, 0, 1, 1]) + truth = np.array(['a', 'a', 'a', 'b', 'c', 'd']) + assert pm.purity(labels, truth) == pytest.approx((3 + 1) / 6) + + +def test_assignment_agreement_is_one_for_identical_vectors() -> None: + rng = np.random.default_rng(4) + centres = _unit(rng, 4, 16) + labels = rng.integers(0, 4, size=200) + vec = centres[labels] + rng.normal(scale=0.02, size=(200, 16)).astype(np.float32) + assert pm.assignment_agreement(vec, vec, labels) == 1.0 + scrambled = _unit(rng, 200, 16) + assert pm.assignment_agreement(vec, scrambled, labels) < 0.6 + + +def test_kmeans_objective_is_lower_for_tight_clusters() -> None: + rng = np.random.default_rng(5) + centres = _unit(rng, 3, 16) + labels = rng.integers(0, 3, size=150) + tight = centres[labels] + rng.normal(scale=0.01, size=(150, 16)).astype(np.float32) + loose = centres[labels] + rng.normal(scale=0.3, size=(150, 16)).astype(np.float32) + assert pm.kmeans_objective(tight, labels) < pm.kmeans_objective(loose, labels) + + +# ---------------------------------------------------------------- strings and labels + + +def test_agreement_share_counts_equal_strings() -> None: + assert pm.agreement_share(['a', 'b', 'c'], ['a', 'x', 'c']) == pytest.approx(2 / 3) + assert pm.agreement_share([], []) == 1.0 + + +def test_text_agreement_matches_boxes_then_compares_exactly() -> None: + ref = ( + np.array(['a', 'a']), + np.array([[0.0, 0.0, 0.5, 0.2], [0.0, 0.5, 0.5, 0.7]], np.float32), + np.array(['STOP', 'EXIT']), + ) + cand = ( + np.array(['a', 'a']), + np.array([[0.0, 0.0, 0.5, 0.2], [0.0, 0.5, 0.5, 0.7]], np.float32), + np.array(['STOP', 'EXIT.']), + ) + out = pm.text_agreement(ref, cand) + assert out['matched'] == 2 + assert out['exact_share'] == 0.5 + assert out['char_error_rate'] > 0.0 + + +def test_cosine_ignores_vector_length() -> None: + a = np.array([[2.0, 0.0], [3.0, 4.0]]) + b = np.array([[0.5, 0.0], [4.0, 3.0]]) + assert pm.cosine_rows(a, b).tolist() == pytest.approx([1.0, 0.96]) + + +def test_knn_label_vote_and_precision_follow_the_vectors() -> None: + rng = np.random.default_rng(6) + centres = _unit(rng, 3, 16) + labels = np.repeat(np.array(['a', 'b', 'c']), 60) + vec = centres[np.repeat(np.arange(3), 60)] + rng.normal(scale=0.02, size=(180, 16)) + vec = vec.astype(np.float32) + same = pm.knn_agreement( + vec, + vec, + k=10, + n_queries=100, + seed=1, + dup_threshold=0.98, + labels=labels, + truth=(labels, labels), + ) + assert same['label_vote_agreement'] == 1.0 + assert same['precision_ref'] == 1.0 + assert same['precision_drop_points'] == 0.0 + scrambled = _unit(rng, 180, 16) + worse = pm.knn_agreement( + vec, + scrambled, + k=10, + n_queries=100, + seed=1, + dup_threshold=0.98, + labels=labels, + truth=(labels, labels), + ) + assert worse['label_vote_agreement'] < 0.6 + assert worse['precision_drop_points'] > 50.0 + + +def test_knn_precision_ignores_queries_without_ground_truth() -> None: + rng = np.random.default_rng(7) + vec = _unit(rng, 50, 8) + none = np.array(['none'] * 50) + out = pm.knn_agreement( + vec, vec, k=5, n_queries=20, seed=0, dup_threshold=0.98, truth=(none, none) + ) + assert 'precision_drop_points' not in out diff --git a/tests/bench/test_parity_store.py b/tests/bench/test_parity_store.py new file mode 100644 index 00000000..312451b7 --- /dev/null +++ b/tests/bench/test_parity_store.py @@ -0,0 +1,92 @@ +"""The capture file format: fixed schema, round trip, integrity checks.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +from scripts.bench import parity_store as ps + + +if TYPE_CHECKING: + from pathlib import Path + + +def test_round_trip_preserves_arrays_and_meta(make_capture, tmp_path: Path) -> None: + cap = make_capture('ref', engine='fp32') + ps.save_capture(tmp_path / 'ref', cap) + back = ps.load_capture(tmp_path / 'ref') + assert back.meta['tag'] == 'ref' + assert set(back.arrays) == set(ps.ARRAY_SPEC) + for name, arr in cap.arrays.items(): + assert back.arrays[name].dtype == arr.dtype + assert np.array_equal(back.arrays[name], arr) + + +def test_capture_json_records_schema_and_npz_digest(make_capture, tmp_path: Path) -> None: + ps.save_capture(tmp_path / 'c', make_capture()) + doc = json.loads((tmp_path / 'c' / 'capture.json').read_text()) + assert doc['schema'] == ps.SCHEMA + assert doc['npz_sha256'] == ps.sha256_file(tmp_path / 'c' / 'arrays.npz') + assert doc['arrays']['item_vec']['dtype'] == 'float16' + assert doc['counts']['items'] == 180 + + +def test_vectors_are_stored_as_float16(make_capture) -> None: + cap = make_capture() + assert cap.arrays['item_vec'].dtype == np.float16 + assert cap.arrays['item_box'].dtype == np.float32 + + +def test_load_rejects_a_tampered_npz(make_capture, tmp_path: Path) -> None: + ps.save_capture(tmp_path / 'c', make_capture()) + with (tmp_path / 'c' / 'arrays.npz').open('ab') as fh: + fh.write(b'x') + with pytest.raises(ps.CaptureError, match='sha256'): + ps.load_capture(tmp_path / 'c') + + +def test_load_rejects_another_schema(make_capture, tmp_path: Path) -> None: + ps.save_capture(tmp_path / 'c', make_capture()) + path = tmp_path / 'c' / 'capture.json' + doc = json.loads(path.read_text()) + doc['schema'] = 99 + path.write_text(json.dumps(doc)) + with pytest.raises(ps.CaptureError, match='schema'): + ps.load_capture(tmp_path / 'c') + + +def test_save_rejects_a_wrong_dtype(make_capture, tmp_path: Path) -> None: + cap = make_capture(mutate=lambda a: a.update(item_vec=a['item_vec'].astype(np.float32))) + with pytest.raises(ps.CaptureError, match='item_vec'): + ps.save_capture(tmp_path / 'c', cap) + + +def test_save_rejects_ragged_groups(make_capture, tmp_path: Path) -> None: + cap = make_capture(mutate=lambda a: a.update(item_score=a['item_score'][:-1])) + with pytest.raises(ps.CaptureError, match='item'): + ps.save_capture(tmp_path / 'c', cap) + + +def test_save_rejects_missing_meta(make_capture, tmp_path: Path) -> None: + cap = make_capture() + del cap.meta['manifest'] + with pytest.raises(ps.CaptureError, match='manifest'): + ps.save_capture(tmp_path / 'c', cap) + + +def test_empty_arrays_are_a_valid_capture(tmp_path: Path) -> None: + arrays = ps.empty_arrays(8) + meta = { + 'tag': 't', + 'created_at': 'now', + 'manifest': {'sha256': 'x'}, + 'engine': {}, + 'environment': {}, + 'sections': [], + } + ps.save_capture(tmp_path / 'e', ps.Capture(meta, arrays)) + assert ps.load_capture(tmp_path / 'e').arrays['item_vec'].shape == (0, 8) From 355dc0f8f4b29bdbe9c6c5694275e897c1f35197 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Sat, 10 Oct 2026 13:46:53 -0400 Subject: [PATCH 2/3] docs(perf): record the Wave 0 parity tool and the FP16 PE gate run The FP16 PE engine passes every parity gate against the FP32 plan in two independent runs, with a same-plan repeat as the noise floor; the wave plan notes what the first runs established (detector run-to-run variation, vectors read from the datastore, the cluster step needs the auto-label worker) (#208). --- CHANGELOG.md | 10 + docs/PERFORMANCE.md | 143 +++++++ .../fp32_repeat_run1/parity_summary.json | 386 ++++++++++++++++++ .../parity/fp32_repeat_run1/parity_summary.md | 59 +++ .../fp32_repeat_run2/parity_summary.json | 382 +++++++++++++++++ .../parity/fp32_repeat_run2/parity_summary.md | 59 +++ .../fp32_vs_fp16_run1/parity_summary.json | 386 ++++++++++++++++++ .../fp32_vs_fp16_run1/parity_summary.md | 59 +++ .../fp32_vs_fp16_run2/parity_summary.json | 382 +++++++++++++++++ .../fp32_vs_fp16_run2/parity_summary.md | 59 +++ docs/design/v060_wave_plan.md | 26 +- 11 files changed, 1950 insertions(+), 1 deletion(-) create mode 100644 docs/benchmarks/parity/fp32_repeat_run1/parity_summary.json create mode 100644 docs/benchmarks/parity/fp32_repeat_run1/parity_summary.md create mode 100644 docs/benchmarks/parity/fp32_repeat_run2/parity_summary.json create mode 100644 docs/benchmarks/parity/fp32_repeat_run2/parity_summary.md create mode 100644 docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.json create mode 100644 docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.md create mode 100644 docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.json create mode 100644 docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0286f85d..c38000be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,16 @@ history of this codebase and was never published. This release is `[0.5.0]`. environment, ingest through `/ingest/upload` and `/ingest/batch`, Triton statistics, perf_analyzer points, endpoints and VLM throughput as one JSON; the v0.5.0 numbers are in `docs/PERFORMANCE.md` and `docs/benchmarks/` (#45, #40). +- `scripts/bench/parity.py` with `capture` and `compare` subcommands: the accuracy gate every + v0.6.0 change passes. `capture` ingests the pinned COCO set into a fresh project and stores the + per-stage outputs (detector boxes, crop rectangles, PE crop and whole-frame vectors, cluster + assignments and geometry, labels, optional MobileCLIP, face, OCR and VLM probes) as + `capture.json` plus a float16 NPZ, with hardware, git sha, image digests, engine record and + dataset manifest hash. `compare` measures a candidate against a reference (box matching, cosine, + neighbour overlap, cluster agreement and purity against the COCO ground truth, label agreement), + applies the thresholds of `scripts/bench/parity_gates.json` and exits non-zero on failure. The + FP32 against FP16 PE engine results are in `docs/PERFORMANCE.md` (#208). + ### Fixed - The `curation-vlm-worker` no longer polls every project each cycle and takes a 409 diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 62a19c92..060051dd 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -822,6 +822,149 @@ The earlier private FP16 measurement of about 170 embeddings/s (5.9 ms) matches to `.install/precision.tsv`, records the group as `degraded` in `groups.tsv` (a re-run retries FP16 instead of skipping), and `./openprocessor models status` lists it under "Degraded precision". +### Wave 0 parity and FP16 gates (set A, public COCO, 2,000 images) + +Every v0.6.0 change is gated by `scripts/bench/parity.py` (WP-0.1, #208): it captures the per-stage +outputs of one ingest of the pinned set and compares two captures against the accuracy gates of +`docs/design/triton_pipeline_optimization_plan.md` section 7 and the cluster-agreement metric of +`docs/design/v060_wave_plan.md` section 2.1. This section describes the tool and records the gate run +that accepts the FP16 PE engine of the section above. + +#### The tool + +```bash +# Reference and candidate: same stack, same pinned set, one capture per configuration. +.venv/bin/python scripts/bench/parity.py capture --tag fp32 --images DIR_OF_PINNED_IMAGES \ + --api-url http://127.0.0.1:API_PORT --triton-url http://127.0.0.1:TRITON_HTTP_PORT \ + --opensearch-url http://127.0.0.1:OPENSEARCH_PORT --compose-project NAME \ + --engine-label fp32 --engine-file PATH/TO/model.plan \ + --annotations COCO_ANNOTATIONS_DIR --probe-images 200 +.venv/bin/python scripts/bench/parity.py compare artifacts_local/bench/parity/fp32 \ + artifacts_local/bench/parity/fp16 --out docs/benchmarks/parity/fp32_vs_fp16 +``` + +`capture` creates a fresh project, ingests the 2,000 images through `POST .../ingest/upload` +(embedding policy `all`), trains clusters once, and stores under `artifacts_local/bench/parity//`: + +| File | Content | +|---|---| +| `capture.json` | Schema number, hardware (GPU, driver, CPU, RAM, load), git sha and dirty flag, the image reference and image id of every container of the compose project, Triton model configuration, the engine record (label, free `KEY=VALUE` pairs, plan size and sha256), the dataset manifest hash, the sections captured, the SHA-256 of `arrays.npz` | +| `arrays.npz` | A fixed set of arrays (float16 vectors, float32 boxes, unicode strings): per item the image, box, detector class and score, integer crop rectangle, cluster id, distance, similarity and core flag, stored class, ground-truth class, VLM label and the PE crop vector; per image the PE whole-frame vector; MobileCLIP, face (box, score, embedding) and OCR (box, text) rows for a sample of images | + +Items are keyed by image file name, box overlap and class name, never by stored id, so two captures of +different projects line up. The curation API never serves embedding vectors, so the PE vectors are read +by document id from the project's own OpenSearch indexes; everything else goes through the project-scoped +routes. The COCO ground-truth class of an item is the class of the best-IoU annotation (IoU at least 0.5, +else `none`). + +`compare` pairs the two captures and measures box matching (greedy, best IoU first, same class name +only), cosine similarity of matched vectors, neighbour overlap at k = 10 on 1,000 seeded query items, +near-duplicate decisions at cosine 0.98, cluster agreement (adjusted Rand index, NMI, nearest-reference- +centroid agreement, k-means objective, purity against the COCO classes), class-name agreement, the VLM +label agreement and the OCR exact-match share. Thresholds live in `scripts/bench/parity_gates.json` +(values from the plan); the exit code is 0 on PASS, 1 on any FAIL (a gate with no data is SKIP and fails +only under `--strict`), 2 when the captures cannot be compared. `--out` writes a small +`parity_summary.json` and `.md`; the large capture files stay out of git. + +| Gate | Rule | +|---|---| +| P-box | at least 98 % of images with every box matched at IoU 0.95 and confidence delta at most 0.02 | +| P-box-confident | zero unmatched boxes with confidence at least 0.5 | +| P-item | item count and class names identical on at least 99 % of images | +| P-pe, P-pew | cosine of crop vectors and of whole-frame vectors: 1st percentile at least 0.995, median at least 0.999 | +| P-knn, P-dup | neighbour overlap at least 0.95; near-duplicate decisions identical on at least 99.5 % of pairs | +| P-cluster-assign | at least 98 % of items keep their nearest reference centroid | +| P-cluster-purity | purity against the COCO classes not lower by more than 0.5 points | +| P-label-vote, P-knn-precision | majority neighbour class identical on at least 97 % of queries; kNN class precision not lower by more than 0.5 points | +| P-label, P-vlm | stored class name at least 99 % identical; VLM label at least 97 % identical | +| P-clip, P-face, P-ocr | MobileCLIP and ArcFace cosine 1st percentile at least 0.995; OCR exact match at least 98 % | + +Crop-rectangle identity, the adjusted Rand index, NMI, the k-means objective difference and the cluster +similarity deltas are recorded but not gated (see the noise floor below). + +#### FP16 against FP32 PE engine + +Same isolated stack (the published v0.5.0 images, GPU 0, own ports, no VLM), only the file behind +`pe_image_encoder` changes. The FP32 plan is the one of the v0.5.0 baseline (1,272,018,292 B). The FP16 plan +is built through the fixed installer path (`export/trt_utils.py` bake, then `trtexec` with the shapes of +`scripts/lib/model_setup.sh`): 641,185,140 B, built in 95 s. Each configuration was captured twice, +once with 4 and once with 1 upload request in flight (run 1 and run 2), and the FP32 plan was captured +a second time in each run as the noise floor (a repeat of the same configuration). All 12,789 to 12,792 +items per capture are the same 6.4 items per image as the baseline. + +**Result: the FP16 engine passes every gate in both runs** (P-vlm is skipped: no VLM is configured). + +| Gate | Metric | Rule | FP16 run 1 | FP16 run 2 | FP32 repeat run 1 | FP32 repeat run 2 | FP16 verdict | +|---|---|---|---:|---:|---:|---:|---| +| P-box | `boxes.images_identical_share` | >= 0.98 | 0.9934 | 0.9934 | 0.9924 | 0.9939 | PASS | +| P-box-confident | `boxes.unmatched_confident` | <= 0 | 0 | 0 | 1 | 0 | PASS | +| P-item | `boxes.item_count_class_identical_share` | >= 0.99 | 0.9995 | 0.9995 | 0.9990 | 0.9995 | PASS | +| P-pe-p01 | `pe_crop.p01` | >= 0.995 | 0.9984 | 0.9990 | 1.0000 | 1.0000 | PASS | +| P-pe-median | `pe_crop.median` | >= 0.999 | 0.9999 | 0.9999 | 1.0000 | 1.0000 | PASS | +| P-pew-p01 | `pe_frame.p01` | >= 0.995 | 0.9975 | 0.9975 | 1.0000 | 1.0000 | PASS | +| P-pew-median | `pe_frame.median` | >= 0.999 | 0.9998 | 0.9998 | 1.0000 | 1.0000 | PASS | +| P-knn | `knn.overlap_at_k` | >= 0.95 | 0.9681 | 0.9706 | 0.9978 | 0.9974 | PASS | +| P-dup | `knn.near_dup_agreement` | >= 0.995 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | PASS | +| P-cluster-assign | `cluster.assignment_agreement` | >= 0.98 | 0.9977 | 0.9977 | 0.9996 | 0.9996 | PASS | +| P-cluster-purity | `cluster.purity_drop_points` | <= 0.5 | -0.7115 | -0.4535 | -0.3988 | -0.1564 | PASS | +| P-label-vote | `knn.label_vote_agreement` | >= 0.97 | 0.9790 | 0.9820 | 0.9990 | 1.0000 | PASS | +| P-knn-precision | `knn.precision_drop_points` | <= 0.5 | 0.0382 | -0.0654 | 0.0127 | 0.0131 | PASS | +| P-label | `labels.class_agreement` | >= 0.99 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | PASS | +| P-vlm | `labels.vlm_agreement` | >= 0.97 | n/a | n/a | n/a | n/a | SKIP | +| P-clip | `clip.p01` | >= 0.995 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | PASS | +| P-face | `face.p01` | >= 0.995 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | PASS | +| P-ocr | `ocr.exact_share` | >= 0.98 | 1.0000 | 1.0000 | 1.0000 | 1.0000 | PASS | + +| Recorded metric | FP16 run 1 | FP16 run 2 | FP32 repeat run 1 | FP32 repeat run 2 | +|---|---:|---:|---:|---:| +| `boxes.crop_rect_identical_share` | 0.9899 | 0.9923 | 0.9917 | 0.9924 | +| `pe_crop.min` / `pe_crop.mean` | 0.9220 / 0.9998 | 0.9257 / 0.9998 | 0.9516 / 0.9999 | 0.9234 / 0.9999 | +| `pe_frame.min` / `pe_frame.mean` | 0.9915 / 0.9997 | 0.9915 / 0.9997 | 1.0000 / 1.0000 | 1.0000 / 1.0000 | +| `cluster.ari` / `cluster.nmi` | 0.3409 / 0.7764 | 0.3317 / 0.7701 | 0.3507 / 0.7791 | 0.3344 / 0.7698 | +| `cluster.objective_rel_diff` | -0.0016 | -0.0059 | 0.0024 | -0.0039 | +| `cluster.purity_ref` / `cluster.purity_cand` | 0.6162 / 0.6233 | 0.6202 / 0.6247 | 0.6162 / 0.6202 | 0.6202 / 0.6218 | +| `knn.precision_ref` / `knn.precision_cand` | 0.5743 / 0.5739 | 0.5849 / 0.5856 | 0.5743 / 0.5741 | 0.5857 / 0.5856 | +| `cluster.core_agreement`, `similarity_abs_delta_mean` | 0.962, 0.0215 | 0.960, 0.0222 | 0.962, 0.0213 | 0.961, 0.0219 | + +Reading the numbers: + +- **The embedding space is the same.** Crop vectors stay at median cosine 0.9999 (1st percentile 0.998 to + 0.999), whole-frame vectors at median 0.9998 (1st percentile 0.9975, worst single frame 0.9915). The + neighbour overlap at 10 is 0.97, the reference nearest neighbour is always inside the FP16 top 10, and + no near-duplicate decision at cosine 0.98 changes. +- **Clustering is unchanged within its own noise.** 99.8 % of items keep their nearest reference + centroid. The adjusted Rand index is 0.33 to 0.35 for FP16 *and* for the repeat of the FP32 plan: + k-means on 12.8k residual items is chaotic, so ARI is recorded and not gated, as the plan says. + Purity against the COCO classes is 0.616 to 0.625 in every capture; FP16 is not lower than the + reference. +- **Detection and labels do not change.** Matched detector scores differ by at most 0.0 in both FP16 runs + (0.08 in one FP32 repeat). Boxes and crop rectangles differ for about 1 % of items in FP16 against FP32 + *and* in an FP32 repeat: the detector is not bit-reproducible between runs (0.8 to 1.0 % of crop + rectangles move, worst matched IoU 0.77 to 0.78). The worst crop cosine (0.92) belongs to such a moved + crop, not to precision. A class change would show as an unmatched box and fail P-item; none does. The + kNN class precision against the COCO classes is the same to 0.1 point. +- **The noise floor decides how to read a single FAIL.** One unmatched box with confidence at least 0.5 + appeared between two FP32 captures of the same plan (run 1 repeat), which the strict P-box-confident + rule counts as a failure. Capture the reference twice and compare the two before blaming a candidate. +- Ingest through `POST .../ingest/upload` took 119 s with the FP16 plan against 224 s with FP32 in + run 1 (the rows of the Wave 1 section above); the captures are not a benchmark and the host carried + other load. + +#### Caveats + +- One host, one GPU shared with other projects, the published v0.5.0 images; the container image ids are + in each `capture.json`. The summaries of the four comparisons are in `docs/benchmarks/parity/`. +- The VLM gate and the label agreement of a registry-labelled project are not exercised: no VLM is + configured and the project has no class registry, so stored class names are empty. The label gates here + are the stored class name (trivially equal) and the majority detector class among the top 10 neighbours + (P-label-vote), the label a propagation step would assign. +- Faces and OCR are probed on the first 200 pinned images through the core routes (200 faces, 34 text + lines); MobileCLIP, ArcFace and OCR engines are unchanged by this run, so their agreement is exact. +- Cluster centroids are not served by the API; the reference centroids of the assignment-agreement + metric are the normalised mean of each reference cluster's vectors. +- Rebuild of the FP16 plan: `export/trt_utils.py` bake inside the API image, then `trtexec` in the Triton + image with the 1/8/32 x 3 x 336 x 336 profile and an 8 GB workspace; the bake log reports `fp16`. + ## Benchmarking ### Using the Go Benchmark Tool diff --git a/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.json b/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.json new file mode 100644 index 00000000..91989ef4 --- /dev/null +++ b/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.json @@ -0,0 +1,386 @@ +{ + "candidate": { + "created_at": "2026-10-10T17:02:38.022964+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "role": "repeat-control" + }, + "git_dirty": false, + "git_sha": "3b371469c820dcf28fc223af24f40a0430557ab9", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32-repeat" + }, + "evaluation": { + "checks": [ + { + "id": "P-box", + "metric": "boxes.images_identical_share", + "note": "images with every box matched at IoU >= 0.95 and confidence delta <= 0.02", + "observed": 0.9924165824064712, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-box-confident", + "metric": "boxes.unmatched_confident", + "note": "unmatched boxes with confidence >= 0.5", + "observed": 1, + "op": "<=", + "status": "FAIL", + "value": 0 + }, + { + "id": "P-item", + "metric": "boxes.item_count_class_identical_share", + "note": "images with the same item count and class names", + "observed": 0.9989888776541962, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-pe-p01", + "metric": "pe_crop.p01", + "note": "cosine, crop vectors of matched items", + "observed": 0.9999993074982279, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pe-median", + "metric": "pe_crop.median", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-pew-p01", + "metric": "pe_frame.p01", + "note": "cosine, whole-frame vectors", + "observed": 0.9999997263547513, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pew-median", + "metric": "pe_frame.median", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-knn", + "metric": "knn.overlap_at_k", + "note": "neighbour overlap at k over the query items", + "observed": 0.9977999999999999, + "op": ">=", + "status": "PASS", + "value": 0.95 + }, + { + "id": "P-dup", + "metric": "knn.near_dup_agreement", + "note": "near-duplicate decisions at the configured cosine, identical on this share of pairs", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-cluster-assign", + "metric": "cluster.assignment_agreement", + "note": "nearest reference centroid unchanged when candidate vectors are assigned to it", + "observed": 0.999609039017906, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-cluster-purity", + "metric": "cluster.purity_drop_points", + "note": "purity against COCO ground-truth classes, percentage points lost", + "observed": -0.39878020173587103, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label-vote", + "metric": "knn.label_vote_agreement", + "note": "majority detector class among the top-k neighbours, identical in both spaces (label propagation); the VLM agreement bar", + "observed": 0.999, + "op": ">=", + "status": "PASS", + "value": 0.97 + }, + { + "id": "P-knn-precision", + "metric": "knn.precision_drop_points", + "note": "kNN precision against COCO ground-truth classes, percentage points lost", + "observed": 0.01273885350318027, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label", + "metric": "labels.class_agreement", + "note": "class name per matched item (the auto-promote outcome)", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-vlm", + "metric": "labels.vlm_agreement", + "note": "VLM label on the sampled crops", + "observed": null, + "op": ">=", + "status": "SKIP", + "value": 0.97 + }, + { + "id": "P-clip", + "metric": "clip.p01", + "note": "core-route image embedding, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-face", + "metric": "face.p01", + "note": "face embedding of matched faces, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-ocr", + "metric": "ocr.exact_share", + "note": "OCR lines matched at IoU >= 0.5, exact string", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.98 + } + ], + "description": "A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent.", + "failed": 1, + "passed": false, + "profile": "precision", + "recorded": { + "boxes.crop_rect_identical_share": 0.9917116271796075, + "boxes.iou_min": 0.7736294280456725, + "boxes.score_delta_max": 0.08349609375, + "cluster.ari": 0.350731312826781, + "cluster.core_agreement": 0.9616076315583705, + "cluster.nmi": 0.7791117461884519, + "cluster.objective_rel_diff": 0.0023646115939802027, + "cluster.purity_cand": 0.620220501993901, + "cluster.purity_ref": 0.6162326999765423, + "cluster.similarity_abs_delta_mean": 0.02128332294523716, + "cluster.similarity_abs_delta_p99": 0.1488930583000183, + "knn.precision_cand": 0.574140127388535, + "knn.precision_ref": 0.5742675159235668, + "knn.top1_recall_at_k": 1.0, + "ocr.char_error_rate": 0.0, + "pe_crop.identical_share": 0.982250371412933, + "pe_crop.mean": 0.9999138128453287, + "pe_crop.min": 0.9516300523090947, + "pe_frame.identical_share": 0.989, + "pe_frame.mean": 0.9999999922424632, + "pe_frame.min": 0.9999973978011156 + }, + "skipped": 1 + }, + "metrics": { + "boxes": { + "crop_rect_identical_share": 0.9917116271796075, + "images": 1978, + "images_identical_share": 0.9924165824064712, + "iou_mean": 0.9997520844110256, + "iou_min": 0.7736294280456725, + "item_count_class_identical_share": 0.9989888776541962, + "items_cand": 12791, + "items_ref": 12789, + "matched": 12789, + "score_delta_max": 0.08349609375, + "unmatched_confident": 1 + }, + "clip": { + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998 + }, + "cluster": { + "ari": 0.350731312826781, + "assignment_agreement": 0.999609039017906, + "clusters_cand": 512, + "clusters_ref": 512, + "core_agreement": 0.9616076315583705, + "items": 12789, + "nmi": 0.7791117461884519, + "objective_cand": 0.13958507776260376, + "objective_ref": 0.13925579190254211, + "objective_rel_diff": 0.0023646115939802027, + "purity_cand": 0.620220501993901, + "purity_drop_points": -0.39878020173587103, + "purity_ref": 0.6162326999765423, + "similarity_abs_delta_mean": 0.02128332294523716, + "similarity_abs_delta_p99": 0.1488930583000183 + }, + "face": { + "cand": 200, + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998, + "ref": 200 + }, + "knn": { + "k": 10, + "label_vote_agreement": 0.999, + "near_dup_agreement": 1.0, + "near_dup_pairs": 10022, + "overlap_at_k": 0.9977999999999999, + "precision_cand": 0.574140127388535, + "precision_drop_points": 0.01273885350318027, + "precision_ref": 0.5742675159235668, + "queries": 1000, + "top1_recall_at_k": 1.0 + }, + "labels": { + "class_agreement": 1.0, + "pairs": 12789 + }, + "ocr": { + "char_error_rate": 0.0, + "exact_share": 1.0, + "lines_cand": 34, + "lines_ref": 34, + "matched": 34 + }, + "pe_crop": { + "identical_share": 0.982250371412933, + "matched": 12789, + "mean": 0.9999138128453287, + "median": 1.0, + "min": 0.9516300523090947, + "n": 12789, + "p01": 0.9999993074982279 + }, + "pe_frame": { + "identical_share": 0.989, + "matched": 2000, + "mean": 0.9999999922424632, + "median": 1.0, + "min": 0.9999973978011156, + "n": 2000, + "p01": 0.9999997263547513 + } + }, + "reference": { + "created_at": "2026-10-10T16:42:53.224136+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "source": "v0.5.0-installer-plan" + }, + "git_dirty": true, + "git_sha": "0567748f7578c21e75cfa7f3c17072eb3ef45d05", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-api-run-3a3005511072": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32" + }, + "schema": 1 +} diff --git a/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.md b/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.md new file mode 100644 index 00000000..1be54ba8 --- /dev/null +++ b/docs/benchmarks/parity/fp32_repeat_run1/parity_summary.md @@ -0,0 +1,59 @@ +# Parity: fp32-repeat against fp32 (FAIL) + +Profile `precision`: A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent. + +| | reference | candidate | +|---|---|---| +| tag | fp32 | fp32-repeat | +| engine | fp32 | fp32 | +| git sha | 0567748f7578 | 3b371469c820 | +| manifest sha256 | edde8ae3d8cecfb2 | edde8ae3d8cecfb2 | + +| Gate | Metric | Observed | Rule | Status | +|---|---|---:|---|---| +| P-box | `boxes.images_identical_share` | 0.99242 | >= 0.98 | PASS | +| P-box-confident | `boxes.unmatched_confident` | 1 | <= 0 | FAIL | +| P-item | `boxes.item_count_class_identical_share` | 0.99899 | >= 0.99 | PASS | +| P-pe-p01 | `pe_crop.p01` | 1.00000 | >= 0.995 | PASS | +| P-pe-median | `pe_crop.median` | 1.00000 | >= 0.999 | PASS | +| P-pew-p01 | `pe_frame.p01` | 1.00000 | >= 0.995 | PASS | +| P-pew-median | `pe_frame.median` | 1.00000 | >= 0.999 | PASS | +| P-knn | `knn.overlap_at_k` | 0.99780 | >= 0.95 | PASS | +| P-dup | `knn.near_dup_agreement` | 1.00000 | >= 0.995 | PASS | +| P-cluster-assign | `cluster.assignment_agreement` | 0.99961 | >= 0.98 | PASS | +| P-cluster-purity | `cluster.purity_drop_points` | -0.39878 | <= 0.5 | PASS | +| P-label-vote | `knn.label_vote_agreement` | 0.99900 | >= 0.97 | PASS | +| P-knn-precision | `knn.precision_drop_points` | 0.01274 | <= 0.5 | PASS | +| P-label | `labels.class_agreement` | 1.00000 | >= 0.99 | PASS | +| P-vlm | `labels.vlm_agreement` | n/a | >= 0.97 | SKIP | +| P-clip | `clip.p01` | 1.00000 | >= 0.995 | PASS | +| P-face | `face.p01` | 1.00000 | >= 0.995 | PASS | +| P-ocr | `ocr.exact_share` | 1.00000 | >= 0.98 | PASS | + +Recorded, not gated: + +| Metric | Value | +|---|---:| +| `boxes.crop_rect_identical_share` | 0.99171 | +| `pe_crop.identical_share` | 0.98225 | +| `pe_frame.identical_share` | 0.98900 | +| `cluster.ari` | 0.35073 | +| `cluster.nmi` | 0.77911 | +| `cluster.objective_rel_diff` | 0.00236 | +| `cluster.purity_ref` | 0.61623 | +| `cluster.similarity_abs_delta_mean` | 0.02128 | +| `cluster.similarity_abs_delta_p99` | 0.14889 | +| `cluster.core_agreement` | 0.96161 | +| `knn.precision_ref` | 0.57427 | +| `knn.precision_cand` | 0.57414 | +| `cluster.purity_cand` | 0.62022 | +| `knn.top1_recall_at_k` | 1.00000 | +| `pe_crop.min` | 0.95163 | +| `pe_crop.mean` | 0.99991 | +| `pe_frame.min` | 1.00000 | +| `pe_frame.mean` | 1.00000 | +| `boxes.iou_min` | 0.77363 | +| `boxes.score_delta_max` | 0.08350 | +| `ocr.char_error_rate` | 0.00000 | + +Items: 12789 reference, 12791 candidate, 12789 matched. diff --git a/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.json b/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.json new file mode 100644 index 00000000..5d27deb2 --- /dev/null +++ b/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.json @@ -0,0 +1,382 @@ +{ + "candidate": { + "created_at": "2026-10-10T17:30:31.039215+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "role": "repeat-control" + }, + "git_dirty": true, + "git_sha": "3b371469c820dcf28fc223af24f40a0430557ab9", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32-c1-repeat" + }, + "evaluation": { + "checks": [ + { + "id": "P-box", + "metric": "boxes.images_identical_share", + "note": "images with every box matched at IoU >= 0.95 and confidence delta <= 0.02", + "observed": 0.993933265925177, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-box-confident", + "metric": "boxes.unmatched_confident", + "note": "unmatched boxes with confidence >= 0.5", + "observed": 0, + "op": "<=", + "status": "PASS", + "value": 0 + }, + { + "id": "P-item", + "metric": "boxes.item_count_class_identical_share", + "note": "images with the same item count and class names", + "observed": 0.9994944388270981, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-pe-p01", + "metric": "pe_crop.p01", + "note": "cosine, crop vectors of matched items", + "observed": 0.9999990855940676, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pe-median", + "metric": "pe_crop.median", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-pew-p01", + "metric": "pe_frame.p01", + "note": "cosine, whole-frame vectors", + "observed": 0.9999990289747694, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pew-median", + "metric": "pe_frame.median", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-knn", + "metric": "knn.overlap_at_k", + "note": "neighbour overlap at k over the query items", + "observed": 0.9974, + "op": ">=", + "status": "PASS", + "value": 0.95 + }, + { + "id": "P-dup", + "metric": "knn.near_dup_agreement", + "note": "near-duplicate decisions at the configured cosine, identical on this share of pairs", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-cluster-assign", + "metric": "cluster.assignment_agreement", + "note": "nearest reference centroid unchanged when candidate vectors are assigned to it", + "observed": 0.999609100148542, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-cluster-purity", + "metric": "cluster.purity_drop_points", + "note": "purity against COCO ground-truth classes, percentage points lost", + "observed": -0.15635994058322833, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label-vote", + "metric": "knn.label_vote_agreement", + "note": "majority detector class among the top-k neighbours, identical in both spaces (label propagation); the VLM agreement bar", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.97 + }, + { + "id": "P-knn-precision", + "metric": "knn.precision_drop_points", + "note": "kNN precision against COCO ground-truth classes, percentage points lost", + "observed": 0.013106159895148739, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label", + "metric": "labels.class_agreement", + "note": "class name per matched item (the auto-promote outcome)", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-vlm", + "metric": "labels.vlm_agreement", + "note": "VLM label on the sampled crops", + "observed": null, + "op": ">=", + "status": "SKIP", + "value": 0.97 + }, + { + "id": "P-clip", + "metric": "clip.p01", + "note": "core-route image embedding, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-face", + "metric": "face.p01", + "note": "face embedding of matched faces, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-ocr", + "metric": "ocr.exact_share", + "note": "OCR lines matched at IoU >= 0.5, exact string", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.98 + } + ], + "description": "A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent.", + "failed": 0, + "passed": true, + "profile": "precision", + "recorded": { + "boxes.crop_rect_identical_share": 0.9924165428817137, + "boxes.iou_min": 0.7832911376249406, + "boxes.score_delta_max": 0.0, + "cluster.ari": 0.33444636318277265, + "cluster.core_agreement": 0.9613790946759441, + "cluster.nmi": 0.7698077835137355, + "cluster.objective_rel_diff": -0.003946994756784524, + "cluster.purity_cand": 0.6217653037291846, + "cluster.purity_ref": 0.6202017043233523, + "cluster.similarity_abs_delta_mean": 0.021939270198345184, + "cluster.similarity_abs_delta_p99": 0.142063170671463, + "knn.precision_cand": 0.5855832241153341, + "knn.precision_ref": 0.5857142857142856, + "knn.top1_recall_at_k": 1.0, + "ocr.char_error_rate": 0.0, + "pe_crop.identical_share": 0.9484793995778281, + "pe_crop.mean": 0.9999191256036929, + "pe_crop.min": 0.923437209647761, + "pe_frame.identical_share": 0.9485, + "pe_frame.mean": 0.9999999590444942, + "pe_frame.min": 0.9999899353817762 + }, + "skipped": 1 + }, + "metrics": { + "boxes": { + "crop_rect_identical_share": 0.9924165428817137, + "images": 1978, + "images_identical_share": 0.993933265925177, + "iou_mean": 0.9997624704132402, + "iou_min": 0.7832911376249406, + "item_count_class_identical_share": 0.9994944388270981, + "items_cand": 12792, + "items_ref": 12791, + "matched": 12791, + "score_delta_max": 0.0, + "unmatched_confident": 0 + }, + "clip": { + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998 + }, + "cluster": { + "ari": 0.33444636318277265, + "assignment_agreement": 0.999609100148542, + "clusters_cand": 512, + "clusters_ref": 512, + "core_agreement": 0.9613790946759441, + "items": 12791, + "nmi": 0.7698077835137355, + "objective_cand": 0.13918055593967438, + "objective_ref": 0.13973207771778107, + "objective_rel_diff": -0.003946994756784524, + "purity_cand": 0.6217653037291846, + "purity_drop_points": -0.15635994058322833, + "purity_ref": 0.6202017043233523, + "similarity_abs_delta_mean": 0.021939270198345184, + "similarity_abs_delta_p99": 0.142063170671463 + }, + "face": { + "cand": 200, + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998, + "ref": 200 + }, + "knn": { + "k": 10, + "label_vote_agreement": 1.0, + "near_dup_agreement": 1.0, + "near_dup_pairs": 10026, + "overlap_at_k": 0.9974, + "precision_cand": 0.5855832241153341, + "precision_drop_points": 0.013106159895148739, + "precision_ref": 0.5857142857142856, + "queries": 1000, + "top1_recall_at_k": 1.0 + }, + "labels": { + "class_agreement": 1.0, + "pairs": 12791 + }, + "ocr": { + "char_error_rate": 0.0, + "exact_share": 1.0, + "lines_cand": 34, + "lines_ref": 34, + "matched": 34 + }, + "pe_crop": { + "identical_share": 0.9484793995778281, + "matched": 12791, + "mean": 0.9999191256036929, + "median": 1.0, + "min": 0.923437209647761, + "n": 12791, + "p01": 0.9999990855940676 + }, + "pe_frame": { + "identical_share": 0.9485, + "matched": 2000, + "mean": 0.9999999590444942, + "median": 1.0, + "min": 0.9999899353817762, + "n": 2000, + "p01": 0.9999990289747694 + } + }, + "reference": { + "created_at": "2026-10-10T17:17:57.618396+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "role": "reference" + }, + "git_dirty": true, + "git_sha": "3b371469c820dcf28fc223af24f40a0430557ab9", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32-c1" + }, + "schema": 1 +} diff --git a/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.md b/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.md new file mode 100644 index 00000000..67795de6 --- /dev/null +++ b/docs/benchmarks/parity/fp32_repeat_run2/parity_summary.md @@ -0,0 +1,59 @@ +# Parity: fp32-c1-repeat against fp32-c1 (PASS) + +Profile `precision`: A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent. + +| | reference | candidate | +|---|---|---| +| tag | fp32-c1 | fp32-c1-repeat | +| engine | fp32 | fp32 | +| git sha | 3b371469c820 | 3b371469c820 | +| manifest sha256 | edde8ae3d8cecfb2 | edde8ae3d8cecfb2 | + +| Gate | Metric | Observed | Rule | Status | +|---|---|---:|---|---| +| P-box | `boxes.images_identical_share` | 0.99393 | >= 0.98 | PASS | +| P-box-confident | `boxes.unmatched_confident` | 0 | <= 0 | PASS | +| P-item | `boxes.item_count_class_identical_share` | 0.99949 | >= 0.99 | PASS | +| P-pe-p01 | `pe_crop.p01` | 1.00000 | >= 0.995 | PASS | +| P-pe-median | `pe_crop.median` | 1.00000 | >= 0.999 | PASS | +| P-pew-p01 | `pe_frame.p01` | 1.00000 | >= 0.995 | PASS | +| P-pew-median | `pe_frame.median` | 1.00000 | >= 0.999 | PASS | +| P-knn | `knn.overlap_at_k` | 0.99740 | >= 0.95 | PASS | +| P-dup | `knn.near_dup_agreement` | 1.00000 | >= 0.995 | PASS | +| P-cluster-assign | `cluster.assignment_agreement` | 0.99961 | >= 0.98 | PASS | +| P-cluster-purity | `cluster.purity_drop_points` | -0.15636 | <= 0.5 | PASS | +| P-label-vote | `knn.label_vote_agreement` | 1.00000 | >= 0.97 | PASS | +| P-knn-precision | `knn.precision_drop_points` | 0.01311 | <= 0.5 | PASS | +| P-label | `labels.class_agreement` | 1.00000 | >= 0.99 | PASS | +| P-vlm | `labels.vlm_agreement` | n/a | >= 0.97 | SKIP | +| P-clip | `clip.p01` | 1.00000 | >= 0.995 | PASS | +| P-face | `face.p01` | 1.00000 | >= 0.995 | PASS | +| P-ocr | `ocr.exact_share` | 1.00000 | >= 0.98 | PASS | + +Recorded, not gated: + +| Metric | Value | +|---|---:| +| `boxes.crop_rect_identical_share` | 0.99242 | +| `pe_crop.identical_share` | 0.94848 | +| `pe_frame.identical_share` | 0.94850 | +| `cluster.ari` | 0.33445 | +| `cluster.nmi` | 0.76981 | +| `cluster.objective_rel_diff` | -0.00395 | +| `cluster.purity_ref` | 0.62020 | +| `cluster.similarity_abs_delta_mean` | 0.02194 | +| `cluster.similarity_abs_delta_p99` | 0.14206 | +| `cluster.core_agreement` | 0.96138 | +| `knn.precision_ref` | 0.58571 | +| `knn.precision_cand` | 0.58558 | +| `cluster.purity_cand` | 0.62177 | +| `knn.top1_recall_at_k` | 1.00000 | +| `pe_crop.min` | 0.92344 | +| `pe_crop.mean` | 0.99992 | +| `pe_frame.min` | 0.99999 | +| `pe_frame.mean` | 1.00000 | +| `boxes.iou_min` | 0.78329 | +| `boxes.score_delta_max` | 0.00000 | +| `ocr.char_error_rate` | 0.00000 | + +Items: 12791 reference, 12792 candidate, 12791 matched. diff --git a/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.json b/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.json new file mode 100644 index 00000000..55f63de7 --- /dev/null +++ b/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.json @@ -0,0 +1,386 @@ +{ + "candidate": { + "created_at": "2026-10-10T16:51:16.337701+00:00", + "engine": { + "label": "fp16", + "plan_bytes": 641185140, + "plan_file": "model.plan", + "plan_sha256": "b9a6f39770e240e0cb1931931ca70fa69fdc4a35918be7fced09310eebc84feb", + "precision": "fp16", + "source": "bake_fp16_onnx+trtexec-main-0567748f" + }, + "git_dirty": true, + "git_sha": "0567748f7578c21e75cfa7f3c17072eb3ef45d05", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp16" + }, + "evaluation": { + "checks": [ + { + "id": "P-box", + "metric": "boxes.images_identical_share", + "note": "images with every box matched at IoU >= 0.95 and confidence delta <= 0.02", + "observed": 0.993427704752275, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-box-confident", + "metric": "boxes.unmatched_confident", + "note": "unmatched boxes with confidence >= 0.5", + "observed": 0, + "op": "<=", + "status": "PASS", + "value": 0 + }, + { + "id": "P-item", + "metric": "boxes.item_count_class_identical_share", + "note": "images with the same item count and class names", + "observed": 0.9994944388270981, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-pe-p01", + "metric": "pe_crop.p01", + "note": "cosine, crop vectors of matched items", + "observed": 0.9984020244926421, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pe-median", + "metric": "pe_crop.median", + "observed": 0.9999000723247468, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-pew-p01", + "metric": "pe_frame.p01", + "note": "cosine, whole-frame vectors", + "observed": 0.9975067306744878, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pew-median", + "metric": "pe_frame.median", + "observed": 0.9998077467941076, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-knn", + "metric": "knn.overlap_at_k", + "note": "neighbour overlap at k over the query items", + "observed": 0.9681000000000002, + "op": ">=", + "status": "PASS", + "value": 0.95 + }, + { + "id": "P-dup", + "metric": "knn.near_dup_agreement", + "note": "near-duplicate decisions at the configured cosine, identical on this share of pairs", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-cluster-assign", + "metric": "cluster.assignment_agreement", + "note": "nearest reference centroid unchanged when candidate vectors are assigned to it", + "observed": 0.9977324263038548, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-cluster-purity", + "metric": "cluster.purity_drop_points", + "note": "purity against COCO ground-truth classes, percentage points lost", + "observed": -0.7115489874110592, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label-vote", + "metric": "knn.label_vote_agreement", + "note": "majority detector class among the top-k neighbours, identical in both spaces (label propagation); the VLM agreement bar", + "observed": 0.979, + "op": ">=", + "status": "PASS", + "value": 0.97 + }, + { + "id": "P-knn-precision", + "metric": "knn.precision_drop_points", + "note": "kNN precision against COCO ground-truth classes, percentage points lost", + "observed": 0.03821656050955191, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label", + "metric": "labels.class_agreement", + "note": "class name per matched item (the auto-promote outcome)", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-vlm", + "metric": "labels.vlm_agreement", + "note": "VLM label on the sampled crops", + "observed": null, + "op": ">=", + "status": "SKIP", + "value": 0.97 + }, + { + "id": "P-clip", + "metric": "clip.p01", + "note": "core-route image embedding, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-face", + "metric": "face.p01", + "note": "face embedding of matched faces, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-ocr", + "metric": "ocr.exact_share", + "note": "OCR lines matched at IoU >= 0.5, exact string", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.98 + } + ], + "description": "A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent.", + "failed": 0, + "passed": true, + "profile": "precision", + "recorded": { + "boxes.crop_rect_identical_share": 0.9899132066619751, + "boxes.iou_min": 0.7832911376249406, + "boxes.score_delta_max": 0.0, + "cluster.ari": 0.3408831108655588, + "cluster.core_agreement": 0.9619204003440457, + "cluster.nmi": 0.77644366461738, + "cluster.objective_rel_diff": -0.0015903184681751188, + "cluster.purity_cand": 0.6233481898506529, + "cluster.purity_ref": 0.6162326999765423, + "cluster.similarity_abs_delta_mean": 0.02154228650033474, + "cluster.similarity_abs_delta_p99": 0.14103618264198303, + "knn.precision_cand": 0.5738853503184713, + "knn.precision_ref": 0.5742675159235668, + "knn.top1_recall_at_k": 1.0, + "ocr.char_error_rate": 0.0, + "pe_crop.identical_share": 0.0, + "pe_crop.mean": 0.9997527038425924, + "pe_crop.min": 0.9219801129058353, + "pe_frame.identical_share": 0.0, + "pe_frame.mean": 0.9996598464399681, + "pe_frame.min": 0.9914525901624422 + }, + "skipped": 1 + }, + "metrics": { + "boxes": { + "crop_rect_identical_share": 0.9899132066619751, + "images": 1978, + "images_identical_share": 0.993427704752275, + "iou_mean": 0.9997121841148262, + "iou_min": 0.7832911376249406, + "item_count_class_identical_share": 0.9994944388270981, + "items_cand": 12790, + "items_ref": 12789, + "matched": 12789, + "score_delta_max": 0.0, + "unmatched_confident": 0 + }, + "clip": { + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998 + }, + "cluster": { + "ari": 0.3408831108655588, + "assignment_agreement": 0.9977324263038548, + "clusters_cand": 512, + "clusters_ref": 512, + "core_agreement": 0.9619204003440457, + "items": 12789, + "nmi": 0.77644366461738, + "objective_cand": 0.13903433084487915, + "objective_ref": 0.13925579190254211, + "objective_rel_diff": -0.0015903184681751188, + "purity_cand": 0.6233481898506529, + "purity_drop_points": -0.7115489874110592, + "purity_ref": 0.6162326999765423, + "similarity_abs_delta_mean": 0.02154228650033474, + "similarity_abs_delta_p99": 0.14103618264198303 + }, + "face": { + "cand": 200, + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998, + "ref": 200 + }, + "knn": { + "k": 10, + "label_vote_agreement": 0.979, + "near_dup_agreement": 1.0, + "near_dup_pairs": 10319, + "overlap_at_k": 0.9681000000000002, + "precision_cand": 0.5738853503184713, + "precision_drop_points": 0.03821656050955191, + "precision_ref": 0.5742675159235668, + "queries": 1000, + "top1_recall_at_k": 1.0 + }, + "labels": { + "class_agreement": 1.0, + "pairs": 12789 + }, + "ocr": { + "char_error_rate": 0.0, + "exact_share": 1.0, + "lines_cand": 34, + "lines_ref": 34, + "matched": 34 + }, + "pe_crop": { + "identical_share": 0.0, + "matched": 12789, + "mean": 0.9997527038425924, + "median": 0.9999000723247468, + "min": 0.9219801129058353, + "n": 12789, + "p01": 0.9984020244926421 + }, + "pe_frame": { + "identical_share": 0.0, + "matched": 2000, + "mean": 0.9996598464399681, + "median": 0.9998077467941076, + "min": 0.9914525901624422, + "n": 2000, + "p01": 0.9975067306744878 + } + }, + "reference": { + "created_at": "2026-10-10T16:42:53.224136+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "source": "v0.5.0-installer-plan" + }, + "git_dirty": true, + "git_sha": "0567748f7578c21e75cfa7f3c17072eb3ef45d05", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-api-run-3a3005511072": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32" + }, + "schema": 1 +} diff --git a/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.md b/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.md new file mode 100644 index 00000000..dfa6ab1f --- /dev/null +++ b/docs/benchmarks/parity/fp32_vs_fp16_run1/parity_summary.md @@ -0,0 +1,59 @@ +# Parity: fp16 against fp32 (PASS) + +Profile `precision`: A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent. + +| | reference | candidate | +|---|---|---| +| tag | fp32 | fp16 | +| engine | fp32 | fp16 | +| git sha | 0567748f7578 | 0567748f7578 | +| manifest sha256 | edde8ae3d8cecfb2 | edde8ae3d8cecfb2 | + +| Gate | Metric | Observed | Rule | Status | +|---|---|---:|---|---| +| P-box | `boxes.images_identical_share` | 0.99343 | >= 0.98 | PASS | +| P-box-confident | `boxes.unmatched_confident` | 0 | <= 0 | PASS | +| P-item | `boxes.item_count_class_identical_share` | 0.99949 | >= 0.99 | PASS | +| P-pe-p01 | `pe_crop.p01` | 0.99840 | >= 0.995 | PASS | +| P-pe-median | `pe_crop.median` | 0.99990 | >= 0.999 | PASS | +| P-pew-p01 | `pe_frame.p01` | 0.99751 | >= 0.995 | PASS | +| P-pew-median | `pe_frame.median` | 0.99981 | >= 0.999 | PASS | +| P-knn | `knn.overlap_at_k` | 0.96810 | >= 0.95 | PASS | +| P-dup | `knn.near_dup_agreement` | 1.00000 | >= 0.995 | PASS | +| P-cluster-assign | `cluster.assignment_agreement` | 0.99773 | >= 0.98 | PASS | +| P-cluster-purity | `cluster.purity_drop_points` | -0.71155 | <= 0.5 | PASS | +| P-label-vote | `knn.label_vote_agreement` | 0.97900 | >= 0.97 | PASS | +| P-knn-precision | `knn.precision_drop_points` | 0.03822 | <= 0.5 | PASS | +| P-label | `labels.class_agreement` | 1.00000 | >= 0.99 | PASS | +| P-vlm | `labels.vlm_agreement` | n/a | >= 0.97 | SKIP | +| P-clip | `clip.p01` | 1.00000 | >= 0.995 | PASS | +| P-face | `face.p01` | 1.00000 | >= 0.995 | PASS | +| P-ocr | `ocr.exact_share` | 1.00000 | >= 0.98 | PASS | + +Recorded, not gated: + +| Metric | Value | +|---|---:| +| `boxes.crop_rect_identical_share` | 0.98991 | +| `pe_crop.identical_share` | 0.00000 | +| `pe_frame.identical_share` | 0.00000 | +| `cluster.ari` | 0.34088 | +| `cluster.nmi` | 0.77644 | +| `cluster.objective_rel_diff` | -0.00159 | +| `cluster.purity_ref` | 0.61623 | +| `cluster.similarity_abs_delta_mean` | 0.02154 | +| `cluster.similarity_abs_delta_p99` | 0.14104 | +| `cluster.core_agreement` | 0.96192 | +| `knn.precision_ref` | 0.57427 | +| `knn.precision_cand` | 0.57389 | +| `cluster.purity_cand` | 0.62335 | +| `knn.top1_recall_at_k` | 1.00000 | +| `pe_crop.min` | 0.92198 | +| `pe_crop.mean` | 0.99975 | +| `pe_frame.min` | 0.99145 | +| `pe_frame.mean` | 0.99966 | +| `boxes.iou_min` | 0.78329 | +| `boxes.score_delta_max` | 0.00000 | +| `ocr.char_error_rate` | 0.00000 | + +Items: 12789 reference, 12790 candidate, 12789 matched. diff --git a/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.json b/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.json new file mode 100644 index 00000000..717cf1b2 --- /dev/null +++ b/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.json @@ -0,0 +1,382 @@ +{ + "candidate": { + "created_at": "2026-10-10T17:41:01.261918+00:00", + "engine": { + "label": "fp16", + "plan_bytes": 641185140, + "plan_file": "model.plan", + "plan_sha256": "b9a6f39770e240e0cb1931931ca70fa69fdc4a35918be7fced09310eebc84feb", + "precision": "fp16", + "source": "bake_fp16_onnx+trtexec-main-0567748f" + }, + "git_dirty": true, + "git_sha": "3b371469c820dcf28fc223af24f40a0430557ab9", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp16-c1" + }, + "evaluation": { + "checks": [ + { + "id": "P-box", + "metric": "boxes.images_identical_share", + "note": "images with every box matched at IoU >= 0.95 and confidence delta <= 0.02", + "observed": 0.993427704752275, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-box-confident", + "metric": "boxes.unmatched_confident", + "note": "unmatched boxes with confidence >= 0.5", + "observed": 0, + "op": "<=", + "status": "PASS", + "value": 0 + }, + { + "id": "P-item", + "metric": "boxes.item_count_class_identical_share", + "note": "images with the same item count and class names", + "observed": 0.9994944388270981, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-pe-p01", + "metric": "pe_crop.p01", + "note": "cosine, crop vectors of matched items", + "observed": 0.9989973322355873, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pe-median", + "metric": "pe_crop.median", + "observed": 0.9999001369552301, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-pew-p01", + "metric": "pe_frame.p01", + "note": "cosine, whole-frame vectors", + "observed": 0.9974928117605427, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-pew-median", + "metric": "pe_frame.median", + "observed": 0.9998070705888232, + "op": ">=", + "status": "PASS", + "value": 0.999 + }, + { + "id": "P-knn", + "metric": "knn.overlap_at_k", + "note": "neighbour overlap at k over the query items", + "observed": 0.9706000000000001, + "op": ">=", + "status": "PASS", + "value": 0.95 + }, + { + "id": "P-dup", + "metric": "knn.near_dup_agreement", + "note": "near-duplicate decisions at the configured cosine, identical on this share of pairs", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-cluster-assign", + "metric": "cluster.assignment_agreement", + "note": "nearest reference centroid unchanged when candidate vectors are assigned to it", + "observed": 0.9977326035965598, + "op": ">=", + "status": "PASS", + "value": 0.98 + }, + { + "id": "P-cluster-purity", + "metric": "cluster.purity_drop_points", + "note": "purity against COCO ground-truth classes, percentage points lost", + "observed": -0.45347928068804055, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label-vote", + "metric": "knn.label_vote_agreement", + "note": "majority detector class among the top-k neighbours, identical in both spaces (label propagation); the VLM agreement bar", + "observed": 0.982, + "op": ">=", + "status": "PASS", + "value": 0.97 + }, + { + "id": "P-knn-precision", + "metric": "knn.precision_drop_points", + "note": "kNN precision against COCO ground-truth classes, percentage points lost", + "observed": -0.06544502617801262, + "op": "<=", + "status": "PASS", + "value": 0.5 + }, + { + "id": "P-label", + "metric": "labels.class_agreement", + "note": "class name per matched item (the auto-promote outcome)", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.99 + }, + { + "id": "P-vlm", + "metric": "labels.vlm_agreement", + "note": "VLM label on the sampled crops", + "observed": null, + "op": ">=", + "status": "SKIP", + "value": 0.97 + }, + { + "id": "P-clip", + "metric": "clip.p01", + "note": "core-route image embedding, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-face", + "metric": "face.p01", + "note": "face embedding of matched faces, cosine", + "observed": 0.9999999999999998, + "op": ">=", + "status": "PASS", + "value": 0.995 + }, + { + "id": "P-ocr", + "metric": "ocr.exact_share", + "note": "OCR lines matched at IoU >= 0.5, exact string", + "observed": 1.0, + "op": ">=", + "status": "PASS", + "value": 0.98 + } + ], + "description": "A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent.", + "failed": 0, + "passed": true, + "profile": "precision", + "recorded": { + "boxes.crop_rect_identical_share": 0.9922595777951525, + "boxes.iou_min": 0.7832911376249406, + "boxes.score_delta_max": 0.0, + "cluster.ari": 0.331673397574622, + "cluster.core_agreement": 0.9600469116497263, + "cluster.nmi": 0.7701179131988896, + "cluster.objective_rel_diff": -0.0059019176139696455, + "cluster.purity_cand": 0.6247068021892104, + "cluster.purity_ref": 0.62017200938233, + "cluster.similarity_abs_delta_mean": 0.022227009758353233, + "cluster.similarity_abs_delta_p99": 0.14669525623321533, + "knn.precision_cand": 0.5856020942408376, + "knn.precision_ref": 0.5849476439790575, + "knn.top1_recall_at_k": 1.0, + "ocr.char_error_rate": 0.0, + "pe_crop.identical_share": 0.0, + "pe_crop.mean": 0.9997766410285926, + "pe_crop.min": 0.9257257183446188, + "pe_frame.identical_share": 0.0, + "pe_frame.mean": 0.9996588444497198, + "pe_frame.min": 0.9914525901624422 + }, + "skipped": 1 + }, + "metrics": { + "boxes": { + "crop_rect_identical_share": 0.9922595777951525, + "images": 1978, + "images_identical_share": 0.993427704752275, + "iou_mean": 0.9997696889647099, + "iou_min": 0.7832911376249406, + "item_count_class_identical_share": 0.9994944388270981, + "items_cand": 12790, + "items_ref": 12791, + "matched": 12790, + "score_delta_max": 0.0, + "unmatched_confident": 0 + }, + "clip": { + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998 + }, + "cluster": { + "ari": 0.331673397574622, + "assignment_agreement": 0.9977326035965598, + "clusters_cand": 512, + "clusters_ref": 512, + "core_agreement": 0.9600469116497263, + "items": 12790, + "nmi": 0.7701179131988896, + "objective_cand": 0.13889528810977936, + "objective_ref": 0.1397199034690857, + "objective_rel_diff": -0.0059019176139696455, + "purity_cand": 0.6247068021892104, + "purity_drop_points": -0.45347928068804055, + "purity_ref": 0.62017200938233, + "similarity_abs_delta_mean": 0.022227009758353233, + "similarity_abs_delta_p99": 0.14669525623321533 + }, + "face": { + "cand": 200, + "identical_share": 1.0, + "mean": 1.0, + "median": 1.0, + "min": 0.9999999999999998, + "n": 200, + "p01": 0.9999999999999998, + "ref": 200 + }, + "knn": { + "k": 10, + "label_vote_agreement": 0.982, + "near_dup_agreement": 1.0, + "near_dup_pairs": 10294, + "overlap_at_k": 0.9706000000000001, + "precision_cand": 0.5856020942408376, + "precision_drop_points": -0.06544502617801262, + "precision_ref": 0.5849476439790575, + "queries": 1000, + "top1_recall_at_k": 1.0 + }, + "labels": { + "class_agreement": 1.0, + "pairs": 12790 + }, + "ocr": { + "char_error_rate": 0.0, + "exact_share": 1.0, + "lines_cand": 34, + "lines_ref": 34, + "matched": 34 + }, + "pe_crop": { + "identical_share": 0.0, + "matched": 12790, + "mean": 0.9997766410285926, + "median": 0.9999001369552301, + "min": 0.9257257183446188, + "n": 12790, + "p01": 0.9989973322355873 + }, + "pe_frame": { + "identical_share": 0.0, + "matched": 2000, + "mean": 0.9996588444497198, + "median": 0.9998070705888232, + "min": 0.9914525901624422, + "n": 2000, + "p01": 0.9974928117605427 + } + }, + "reference": { + "created_at": "2026-10-10T17:17:57.618396+00:00", + "engine": { + "label": "fp32", + "plan_bytes": 1272018292, + "plan_file": "model.plan", + "plan_sha256": "4215055e04381c18632a5a64a3a9039da46278ccef4f844761da25d6ff411f33", + "precision": "fp32", + "role": "reference" + }, + "git_dirty": true, + "git_sha": "3b371469c820dcf28fc223af24f40a0430557ab9", + "gpu": "NVIDIA RTX A6000", + "image_digests": { + "w0par-api": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-auto-label-worker": { + "image": "davidamacey/openprocessor:0.5.0", + "image_id": "sha256:89975719cc407f5842f3588773421d735973f49caa066d3b3cfe1f82a7bdb524" + }, + "w0par-opensearch": { + "image": "opensearchproject/opensearch:3.6.0", + "image_id": "sha256:b1b447d0d021b051fdb1ae6be100e106667bbe302aa8d6855a4f6d726863d766" + }, + "w0par-triton": { + "image": "davidamacey/openprocessor-triton:0.5.0", + "image_id": "sha256:c26adfad81ad2971f859c77a7899886a368fe45580af9e818c265a60842ee535" + } + }, + "manifest": { + "annotations_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268", + "count": 2000, + "name": "coco_bench_2000.json", + "seed": 20261009, + "sha256": "edde8ae3d8cecfb225040852ed7c46844009ad4f8e7e153e9c44f97c2558c691" + }, + "sections": [ + "items", + "frames", + "clusters", + "clip", + "faces", + "ocr" + ], + "tag": "fp32-c1" + }, + "schema": 1 +} diff --git a/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.md b/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.md new file mode 100644 index 00000000..a9870ea8 --- /dev/null +++ b/docs/benchmarks/parity/fp32_vs_fp16_run2/parity_summary.md @@ -0,0 +1,59 @@ +# Parity: fp16-c1 against fp32-c1 (PASS) + +Profile `precision`: A change that moves numerics of one stage by rounding only (engine precision, input dtype): matched detections and vectors must stay equivalent. + +| | reference | candidate | +|---|---|---| +| tag | fp32-c1 | fp16-c1 | +| engine | fp32 | fp16 | +| git sha | 3b371469c820 | 3b371469c820 | +| manifest sha256 | edde8ae3d8cecfb2 | edde8ae3d8cecfb2 | + +| Gate | Metric | Observed | Rule | Status | +|---|---|---:|---|---| +| P-box | `boxes.images_identical_share` | 0.99343 | >= 0.98 | PASS | +| P-box-confident | `boxes.unmatched_confident` | 0 | <= 0 | PASS | +| P-item | `boxes.item_count_class_identical_share` | 0.99949 | >= 0.99 | PASS | +| P-pe-p01 | `pe_crop.p01` | 0.99900 | >= 0.995 | PASS | +| P-pe-median | `pe_crop.median` | 0.99990 | >= 0.999 | PASS | +| P-pew-p01 | `pe_frame.p01` | 0.99749 | >= 0.995 | PASS | +| P-pew-median | `pe_frame.median` | 0.99981 | >= 0.999 | PASS | +| P-knn | `knn.overlap_at_k` | 0.97060 | >= 0.95 | PASS | +| P-dup | `knn.near_dup_agreement` | 1.00000 | >= 0.995 | PASS | +| P-cluster-assign | `cluster.assignment_agreement` | 0.99773 | >= 0.98 | PASS | +| P-cluster-purity | `cluster.purity_drop_points` | -0.45348 | <= 0.5 | PASS | +| P-label-vote | `knn.label_vote_agreement` | 0.98200 | >= 0.97 | PASS | +| P-knn-precision | `knn.precision_drop_points` | -0.06545 | <= 0.5 | PASS | +| P-label | `labels.class_agreement` | 1.00000 | >= 0.99 | PASS | +| P-vlm | `labels.vlm_agreement` | n/a | >= 0.97 | SKIP | +| P-clip | `clip.p01` | 1.00000 | >= 0.995 | PASS | +| P-face | `face.p01` | 1.00000 | >= 0.995 | PASS | +| P-ocr | `ocr.exact_share` | 1.00000 | >= 0.98 | PASS | + +Recorded, not gated: + +| Metric | Value | +|---|---:| +| `boxes.crop_rect_identical_share` | 0.99226 | +| `pe_crop.identical_share` | 0.00000 | +| `pe_frame.identical_share` | 0.00000 | +| `cluster.ari` | 0.33167 | +| `cluster.nmi` | 0.77012 | +| `cluster.objective_rel_diff` | -0.00590 | +| `cluster.purity_ref` | 0.62017 | +| `cluster.similarity_abs_delta_mean` | 0.02223 | +| `cluster.similarity_abs_delta_p99` | 0.14670 | +| `cluster.core_agreement` | 0.96005 | +| `knn.precision_ref` | 0.58495 | +| `knn.precision_cand` | 0.58560 | +| `cluster.purity_cand` | 0.62471 | +| `knn.top1_recall_at_k` | 1.00000 | +| `pe_crop.min` | 0.92573 | +| `pe_crop.mean` | 0.99978 | +| `pe_frame.min` | 0.99145 | +| `pe_frame.mean` | 0.99966 | +| `boxes.iou_min` | 0.78329 | +| `boxes.score_delta_max` | 0.00000 | +| `ocr.char_error_rate` | 0.00000 | + +Items: 12791 reference, 12790 candidate, 12790 matched. diff --git a/docs/design/v060_wave_plan.md b/docs/design/v060_wave_plan.md index 51cee5d1..1f187cde 100644 --- a/docs/design/v060_wave_plan.md +++ b/docs/design/v060_wave_plan.md @@ -1,6 +1,7 @@ # v0.6.0 wave plan: precision, host CPU, zero-copy data path, clustering, vector store -Status: plan (nothing implemented). Milestone v0.6.0. Umbrella issues: #40 (pipeline), #56 +Status: plan. WP-0.1 (parity tooling, #208) is implemented; its FP16 gate run is recorded in +`docs/PERFORMANCE.md` ("Wave 0 parity and FP16 gates"). Milestone v0.6.0. Umbrella issues: #40 (pipeline), #56 (vector quantization), #67 (request-driven plans), #42 (12 GB engines), #41 (image slimming). Work-package issues: #208 to #221, plus #207 and #153 (table in section 10), and #226 to #232 from the serving backend trade study (section 13, @@ -150,6 +151,29 @@ section 1 ("Corrections") in the same PR or in WP-1.2. and keep it as `ref-v050`; after WP-1.1 capture `ref-fp16`. Both are the references for later waves (most later WPs compare against `ref-fp16`). +**Status: implemented (#208).** The tool is `scripts/bench/parity.py` with `capture` and `compare` +subcommands (not the two scripts named above); the pure logic is split into `parity_metrics.py`, +`parity_store.py` (file format), `parity_assemble.py`, `parity_gates.py` and `parity_env.py`, the +thresholds are in `scripts/bench/parity_gates.json`, and the tests are `tests/bench/test_parity_*.py` +(synthetic fixtures; 18 injected mutants, among them a flipped IoU comparison and a dropped +class-name check, are all caught). What the first runs established: + +- `ref-v050` is the capture of the FP32 plan and `ref-fp16` the capture of the FP16 plan, both under + `artifacts_local/bench/parity/` (local, 28 MB each, not in git); `docs/benchmarks/parity/` holds the + committed summaries. FP16 passes every gate against FP32 (WP-1.1 parity gates P-pe/P-pew: + 1st percentile 0.998/0.9975, median 0.9999/0.9998). +- The detector is **not bit-reproducible between runs** on this stack: two captures of the same plan + differ in about 1 % of crop rectangles. Consequences for later WPs: capture the reference twice and + compare the two first (the noise floor); the crop-rectangle identity of rule D2 is recorded, not + gated, so every downstream gate runs every time; whole-capture bit-identity (gate P-json) cannot be + proven from two live ingests, it needs the route-JSON comparison of that WP. +- The curation API serves no embedding vectors, so PE vectors are read by id from the project's + OpenSearch indexes; the cluster centroids are likewise derived (mean of each cluster's vectors). +- The auto-label worker (compose profile `curation`) must run for the cluster step. +- The registry is empty in a fresh project, so label agreement is measured as the majority detector class + among the top-10 neighbours and, when a VLM is active, as the VLM label on a seeded crop sample + (`--vlm-crops`, not run in the FP16 acceptance: no VLM was configured). + ## 4. Wave 1: precision, host CPU, idle CPU, cluster scheduling (low risk) Goal: double the GPU-bound rate by fixing engine precision, then make sure the host does not From 36bfb7af07a50d6cca4901417850c1d2f6422143 Mon Sep 17 00:00:00 2001 From: davidamacey Date: Sat, 10 Oct 2026 18:17:32 -0400 Subject: [PATCH 3/3] docs(perf): name the release as 0.6.0 in the parity section --- docs/PERFORMANCE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 060051dd..a953f710 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -824,7 +824,7 @@ The earlier private FP16 measurement of about 170 embeddings/s (5.9 ms) matches ### Wave 0 parity and FP16 gates (set A, public COCO, 2,000 images) -Every v0.6.0 change is gated by `scripts/bench/parity.py` (WP-0.1, #208): it captures the per-stage +Every 0.6.0 change is gated by `scripts/bench/parity.py` (WP-0.1, #208): it captures the per-stage outputs of one ingest of the pinned set and compares two captures against the accuracy gates of `docs/design/triton_pipeline_optimization_plan.md` section 7 and the cluster-agreement metric of `docs/design/v060_wave_plan.md` section 2.1. This section describes the tool and records the gate run