From 74e9ef508604c52a6464709b87cf2c87d095ea31 Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Wed, 14 Oct 2026 14:00:25 +0200 Subject: [PATCH 1/2] =?UTF-8?q?measurement:=20schema=20version=202=20?= =?UTF-8?q?=E2=80=94=20task=20and=20family,=20detection=20parity,=20vendor?= =?UTF-8?q?=20figures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - task (classification | detection, required) and family on both kinds; one column after join. - quality: accuracy_source (measured | vendor_published); argmax_parity becomes parity + parity_metric (argmax | box_iou90); intervals and parity rule required when measured, absent when a vendor published the figure. - latency: latency_source may be vendor_published; power_w; the known recipe keys (RECIPE_KEYS) checked for type. - Every row is read against the fields of its own schema_version (version-1 files stay valid as written); upgrade(row, task, family) gives the version-2 row; join refuses mixed versions; to_parquet writes one version per file. - A vendor figure is well formed and never VALID as a measurement. --- CHANGELOG.md | 16 ++++ fastermodels/_modidx.py | 1 + fastermodels/measurement.py | 101 ++++++++++++++++++++--- nbs/04_measurement.ipynb | 133 +++++++++++++++++++++++++++---- nbs/tests/test_measurement.ipynb | 90 ++++++++++++++++++--- 5 files changed, 303 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab2337..df8f270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- `measurement` schema version 2 (`SCHEMA_VERSION = 2`, `SCHEMA_VERSIONS = (1, 2)`, `FIELDS_BY_VERSION`): + `task` and `family` on both kinds; detection parity (`parity` + `parity_metric`: `argmax` or `box_iou90`, + replacing `argmax_parity`); figures a vendor published (`latency_source` / `accuracy_source` = + `vendor_published`), well formed and never `VALID` as a measurement; `power_w`; the known `recipe` keys + checked for type (`RECIPE_KEYS`). +- `upgrade(row, task, family=None)`: the version-2 row a version-1 row is. + +### Changed + +- Every row is read against the fields of its own `schema_version`; version-1 files stay valid as written. +- `join` refuses to pair rows of two versions, and `to_parquet` writes one version per file. + ## 0.1.2 — 2026-09-21 The card reports the top-1 an artifact reaches, not a gap on it. diff --git a/fastermodels/_modidx.py b/fastermodels/_modidx.py index 700cdc5..6420142 100644 --- a/fastermodels/_modidx.py +++ b/fastermodels/_modidx.py @@ -60,6 +60,7 @@ 'fastermodels/measurement.py'), 'fastermodels.measurement.to_parquet': ( 'measurement.html#to_parquet', 'fastermodels/measurement.py'), + 'fastermodels.measurement.upgrade': ('measurement.html#upgrade', 'fastermodels/measurement.py'), 'fastermodels.measurement.validate': ('measurement.html#validate', 'fastermodels/measurement.py'), 'fastermodels.measurement.write_row': ( 'measurement.html#write_row', 'fastermodels/measurement.py')}, diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py index cbc3d30..59d727a 100644 --- a/fastermodels/measurement.py +++ b/fastermodels/measurement.py @@ -12,11 +12,12 @@ from pathlib import Path # %% auto #0 -__all__ = ['SCHEMA_VERSION', 'FIELDS', 'validate', 'status', 'key', 'throughput', 'join', 'graph_hash', 'read_rows', 'write_row', - 'to_parquet'] +__all__ = ['SCHEMA_VERSION', 'SCHEMA_VERSIONS', 'RECIPE_KEYS', 'FIELDS_BY_VERSION', 'FIELDS', 'validate', 'upgrade', 'status', + 'key', 'throughput', 'join', 'graph_hash', 'read_rows', 'write_row', 'to_parquet'] # %% ../nbs/04_measurement.ipynb #schema -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 # what new rows are written at +SCHEMA_VERSIONS = (1, 2) # what `validate` reads, each against its own fields _UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|\+00:00)' _HEX = r'[0-9a-f]{64}' _PCT = dict(ge=0, le=100) @@ -24,7 +25,7 @@ # name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None _SHARED = { - 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)), + 'schema_version': dict(type=int, required=True, choices=SCHEMA_VERSIONS), 'kind': dict(type=str, required=True, choices=('latency', 'quality')), 'run_id': dict(type=str, required=True), 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True), @@ -104,7 +105,32 @@ 'baseline_run_id': dict(type=str, required=False), } -FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}} +# ---- version 2 (2026-10-14): the task and the family, detection parity, figures a vendor published, power, +# and the recipe keys a variant is described by. Version 1 rows keep their own fields, unchanged. ---- +_TASK = dict(type=str, required=True, choices=('classification', 'detection')) +_FAMILY = dict(type=str, required=False) # e.g. 'ultralytics-yolo', 'yolox', 'resnet' +_SOURCE = ('measured', 'estimated', 'vendor_published') +_LATENCY_V2 = {'task': _TASK, 'family': _FAMILY, + **{k: v for k, v in _LATENCY.items() if k != 'latency_source'}, + 'latency_source': dict(type=str, required=True, choices=_SOURCE), + 'power_w': dict(type=float, required=False, gt=0)} +_QUALITY_V2 = {'task': _TASK, 'family': _FAMILY, + **{k: v for k, v in _QUALITY.items() if k not in ('argmax_parity', 'parity_floor', 'ci_low', 'ci_high', 'ci_method')}, + 'accuracy_source': dict(type=str, required=True, choices=('measured', 'vendor_published')), + 'ci_low': dict(type=float, required=False, **_PCT), # required when measured (`_cross_problems`) + 'ci_high': dict(type=float, required=False, **_PCT), + 'ci_method': dict(type=str, required=False, choices=('wilson', 'bootstrap')), + 'parity': dict(type=float, required=False, **_PCT), + 'parity_metric': dict(type=str, required=False, choices=('argmax', 'box_iou90')), + 'parity_floor': dict(type=float, required=False, **_PCT)} + +# the recipe keys a variant is described by; each is checked for its type when present, other keys are free +RECIPE_KEYS = {'resolution': int, 'num_classes': int, 'roi_crop': list, 'pruned': bool, 'prune_method': str, + 'prune_ratio': float, 'distilled': bool, 'teacher': str, 'qat': bool, 'quant': str, 'calib_n': int} + +FIELDS_BY_VERSION = {1: {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}}, + 2: {'latency': {**_SHARED, **_LATENCY_V2}, 'quality': {**_SHARED, **_QUALITY_V2}}} +FIELDS = FIELDS_BY_VERSION[SCHEMA_VERSION] def _item(o): @@ -168,6 +194,17 @@ def _cross_problems(row): out.append(f"model_id: {row['model_id']!r} given for random weights") elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None: out.append('engine_hash: missing, required when measured through the deployed engine') + if row['schema_version'] >= 2: + if kind == 'latency': + for k, v in row['recipe'].items(): + t = RECIPE_KEYS.get(k) + if t is not None and v is not None and not _typed(v, t): + out.append(f'recipe.{k}: expected {t.__name__}, got {v!r}') + elif row['accuracy_source'] == 'measured': + for f in ('ci_low', 'ci_high', 'ci_method', 'parity_metric', 'parity_floor'): + if row.get(f) is None: out.append(f'{f}: missing, required when the accuracy was measured') + if kind == 'quality' and row.get('parity') is not None and row.get('parity_metric') is None: + out.append('parity_metric: missing, required to read parity') for fields in _ORDERED[kind]: vals = [(f, row[f]) for f in fields if row.get(f) is not None] if [v for _, v in vals] != sorted(v for _, v in vals): @@ -178,16 +215,39 @@ def _cross_problems(row): def validate( row: dict, # one measurement row, of either kind ) -> list[str]: - "Every structural problem of `row`, each prefixed by its field name; an empty list means well formed" + "Every structural problem of `row`, each prefixed by its field name; an empty list means well formed. A row is read against the fields of ITS `schema_version`" if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}'] kind = row.get('kind') if kind is None: return ['kind: missing'] if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}'] - fields = FIELDS[kind] + v = row.get('schema_version') + if v is not None and (isinstance(v, bool) or v not in FIELDS_BY_VERSION): # an unknown version: nothing else can be read + return _field_problems('schema_version', _SHARED['schema_version'], v) + fields = FIELDS_BY_VERSION.get(v, FIELDS)[kind] # each version against its own fields out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields] for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name)) return out or _cross_problems(row) + +def upgrade( + row: dict, # a well-formed version-1 row + task: str, # what version 1 did not record: 'classification' or 'detection' + family: str | None = None, # and, optionally, the model family +) -> dict: + "The version-2 row a version-1 row is: `task` (and `family`) added; on a quality row `argmax_parity` becomes `parity` with `parity_metric='argmax'` and `accuracy_source='measured'`. The result is validated; a version-2 row comes back unchanged." + if row.get('schema_version') == 2: return dict(row) + problems = validate(row) + if problems: raise ValueError('row not upgraded: ' + '; '.join(problems)) + out = {**row, 'schema_version': 2, 'task': task, 'family': family} + if row['kind'] == 'quality': + out['accuracy_source'] = 'measured' + par = out.pop('argmax_parity', None) + out['parity'], out['parity_metric'] = par, 'argmax' + problems = validate(out) + if problems: raise ValueError('upgraded row refused: ' + '; '.join(problems)) + return out + + # %% ../nbs/04_measurement.ipynb #status # per kind: field, when its value makes the measurement invalid, and why; a field that is None is not recorded _VALIDITY = { @@ -201,6 +261,15 @@ def validate( ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'), ), } +# version 2: a figure a vendor published is recorded, labelled, and never a measurement of ours +_VALIDITY_V2 = { + 'latency': _VALIDITY['latency'], + 'quality': ( + ('accuracy_source', lambda v, r, m: v != 'measured', 'accuracy not measured by us ({v})'), + _VALIDITY['quality'][0], + ('parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]} ({r[parity_metric]})'), + ), +} def status( @@ -210,7 +279,7 @@ def status( "('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget" reasons = validate(row) if reasons: return 'INVALID', reasons - for f, bad, why in _VALIDITY[row['kind']]: + for f, bad, why in (_VALIDITY_V2 if row['schema_version'] >= 2 else _VALIDITY)[row['kind']]: v = row.get(f) if v is None: reasons.append(f'{f}: not recorded') elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct)) @@ -240,16 +309,20 @@ def throughput( # %% ../nbs/04_measurement.ipynb #join _OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed -_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree +_BOTH = tuple(f for f in _LATENCY_V2 if f in _QUALITY_V2) # one column, both sides must agree def _pair(lat, qual): "One joined dict from a latency row and a quality row of the same engine" - out = {'schema_version': lat['schema_version']} + if lat['schema_version'] != qual['schema_version']: + raise ValueError(f"latency row {lat['run_id']} is version {lat['schema_version']}, quality row {qual['run_id']} " + f"version {qual['schema_version']}: `upgrade` the version-1 row before joining") + out, ver = {'schema_version': lat['schema_version']}, FIELDS_BY_VERSION[lat['schema_version']] + both = [f for f in _BOTH if f in ver['latency'] and f in ver['quality']] # the shared columns of THIS version for side, r in (('latency', lat), ('quality', qual)): out |= {f'{side}_{f}': r.get(f) for f in _OWN} - out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH} - for f in _BOTH: + out |= {f: r.get(f) for f in ver[side] if f not in _SHARED and f not in both} + for f in both: a, b = lat.get(f), qual.get(f) if None not in (a, b) and a != b: raise ValueError(f"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine") @@ -338,7 +411,9 @@ def to_parquet( for i, r in enumerate(rows): problems = validate(r) if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems)) - fields = FIELDS[kinds[0]] + versions = sorted({r.get('schema_version') for r in rows}) + if len(versions) != 1: raise ValueError(f'to_parquet writes one schema version per file, got {versions}: `upgrade` first') + fields = FIELDS_BY_VERSION[versions[0]][kinds[0]] schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()]) plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields} diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb index 5342d32..591ee36 100644 --- a/nbs/04_measurement.ipynb +++ b/nbs/04_measurement.ipynb @@ -70,8 +70,11 @@ "- a **quality row** measures one artifact's accuracy on one evaluation set, through the engine that runs it.\n", "\n", "Rows are stored as JSON Lines, one object per line, appended and never rewritten; a file may hold both kinds.\n", - "`FIELDS[kind]` is the schema of each kind: every field's type, whether it is required, and the rules its\n", - "value obeys. A field that is not required may be absent or `None`, which means *not recorded*.\n", + "`FIELDS[kind]` is the schema of each kind at the current `SCHEMA_VERSION` (2): every field's type, whether it is\n", + "required, and the rules its value obeys. A field that is not required may be absent or `None`, which means\n", + "*not recorded*. A row is always read against the fields of **its own** `schema_version`\n", + "(`FIELDS_BY_VERSION[1]` or `[2]`), so a version-1 file stays valid as written; `upgrade(row, task=...)` turns a\n", + "version-1 row into the version-2 row it is.\n", "\n", "| Kind | Group | Fields |\n", "|---|---|---|\n", @@ -84,6 +87,14 @@ "| quality | model and artifact | `model_id`, `model_revision`, `model_hash`, `weights_license`, `engine_hash`, `precision`, `device` |\n", "| quality | accuracy | `metric`, `eval_set`, `eval_set_hash`, `dataset_license`, `eval_n`, `accuracy`, `ci_low`, `ci_high`, `ci_method`, `measured_through`, `argmax_parity`, `parity_floor`, `baseline_run_id` |\n", "\n", + "Version 2 (current) changes, against version 1:\n", + "\n", + "| Kind | Change |\n", + "|---|---|\n", + "| both | `task` (`classification` or `detection`, required) and `family` (e.g. `yolox`); one column after `join`, both sides must agree |\n", + "| latency | `latency_source` may be `vendor_published`; `power_w`; the known `recipe` keys are checked for type (`RECIPE_KEYS`: `resolution`, `num_classes`, `roi_crop`, `pruned`, `prune_method`, `prune_ratio`, `distilled`, `teacher`, `qat`, `quant`, `calib_n`), any other key stays free |\n", + "| quality | `accuracy_source` (`measured` or `vendor_published`, required); `argmax_parity` becomes `parity` with `parity_metric` (`argmax` for classification, `box_iou90` for detection: same class, IoU >= 0.9); `ci_low`, `ci_high`, `ci_method`, `parity_metric` and `parity_floor` are required when the accuracy was measured, absent when a vendor published it |\n", + "\n", "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of any key.\n", "- `graph_hash` is the sha256 of the ONNX graph with the values of every initializer removed, so two models of the same shape share it, whatever their weights. `graph_hash(onnx_path)` computes it.\n", "- `latency_scope` says what the latency covers, e.g. engine execution only, or host transfers included; two latency rows are only comparable when it matches.\n", @@ -99,7 +110,11 @@ "\n", "- latency: `latency_source` is not `measured`; `throttled` is true or not recorded; `latency_spread_pct` is\n", " above `max_spread_pct` (5 by default) or not recorded;\n", - "- quality: `measured_through` is not `deployed_engine`; `argmax_parity` is below `parity_floor` or not recorded.\n", + "- quality: `measured_through` is not `deployed_engine`; `argmax_parity` (version 2: `parity`) is below\n", + " `parity_floor` or not recorded; in version 2, `accuracy_source` is not `measured`.\n", + "\n", + "A figure a vendor published is therefore **recorded and labelled, never a measurement**: it is well formed, so it\n", + "can be shown next to ours with its provenance, and `INVALID` for anything that needs a measurement.\n", "\n", "`join(latency_rows, quality_rows)` pairs the VALID rows of both kinds that share an `engine_hash`: one dict per\n", "pair, with the fields of both. The identity and provenance fields are kept per side, as `latency_run_id`,\n", @@ -116,7 +131,8 @@ "outputs": [], "source": [ "#| export\n", - "SCHEMA_VERSION = 1\n", + "SCHEMA_VERSION = 2 # what new rows are written at\n", + "SCHEMA_VERSIONS = (1, 2) # what `validate` reads, each against its own fields\n", "_UTC = r'\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|\\+00:00)'\n", "_HEX = r'[0-9a-f]{64}'\n", "_PCT = dict(ge=0, le=100)\n", @@ -124,7 +140,7 @@ "\n", "# name -> type, required, and the declarative rules its value obeys; a field that is not required may be absent or None\n", "_SHARED = {\n", - " 'schema_version': dict(type=int, required=True, choices=(SCHEMA_VERSION,)),\n", + " 'schema_version': dict(type=int, required=True, choices=SCHEMA_VERSIONS),\n", " 'kind': dict(type=str, required=True, choices=('latency', 'quality')),\n", " 'run_id': dict(type=str, required=True),\n", " 'measured_at': dict(type=str, required=True, pattern=_UTC, date=True),\n", @@ -204,7 +220,32 @@ " 'baseline_run_id': dict(type=str, required=False),\n", "}\n", "\n", - "FIELDS = {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}}\n", + "# ---- version 2 (2026-10-14): the task and the family, detection parity, figures a vendor published, power,\n", + "# and the recipe keys a variant is described by. Version 1 rows keep their own fields, unchanged. ----\n", + "_TASK = dict(type=str, required=True, choices=('classification', 'detection'))\n", + "_FAMILY = dict(type=str, required=False) # e.g. 'ultralytics-yolo', 'yolox', 'resnet'\n", + "_SOURCE = ('measured', 'estimated', 'vendor_published')\n", + "_LATENCY_V2 = {'task': _TASK, 'family': _FAMILY,\n", + " **{k: v for k, v in _LATENCY.items() if k != 'latency_source'},\n", + " 'latency_source': dict(type=str, required=True, choices=_SOURCE),\n", + " 'power_w': dict(type=float, required=False, gt=0)}\n", + "_QUALITY_V2 = {'task': _TASK, 'family': _FAMILY,\n", + " **{k: v for k, v in _QUALITY.items() if k not in ('argmax_parity', 'parity_floor', 'ci_low', 'ci_high', 'ci_method')},\n", + " 'accuracy_source': dict(type=str, required=True, choices=('measured', 'vendor_published')),\n", + " 'ci_low': dict(type=float, required=False, **_PCT), # required when measured (`_cross_problems`)\n", + " 'ci_high': dict(type=float, required=False, **_PCT),\n", + " 'ci_method': dict(type=str, required=False, choices=('wilson', 'bootstrap')),\n", + " 'parity': dict(type=float, required=False, **_PCT),\n", + " 'parity_metric': dict(type=str, required=False, choices=('argmax', 'box_iou90')),\n", + " 'parity_floor': dict(type=float, required=False, **_PCT)}\n", + "\n", + "# the recipe keys a variant is described by; each is checked for its type when present, other keys are free\n", + "RECIPE_KEYS = {'resolution': int, 'num_classes': int, 'roi_crop': list, 'pruned': bool, 'prune_method': str,\n", + " 'prune_ratio': float, 'distilled': bool, 'teacher': str, 'qat': bool, 'quant': str, 'calib_n': int}\n", + "\n", + "FIELDS_BY_VERSION = {1: {'latency': {**_SHARED, **_LATENCY}, 'quality': {**_SHARED, **_QUALITY}},\n", + " 2: {'latency': {**_SHARED, **_LATENCY_V2}, 'quality': {**_SHARED, **_QUALITY_V2}}}\n", + "FIELDS = FIELDS_BY_VERSION[SCHEMA_VERSION]\n", "\n", "\n", "def _item(o):\n", @@ -268,6 +309,17 @@ " out.append(f\"model_id: {row['model_id']!r} given for random weights\")\n", " elif row['measured_through'] == 'deployed_engine' and row.get('engine_hash') is None:\n", " out.append('engine_hash: missing, required when measured through the deployed engine')\n", + " if row['schema_version'] >= 2:\n", + " if kind == 'latency':\n", + " for k, v in row['recipe'].items():\n", + " t = RECIPE_KEYS.get(k)\n", + " if t is not None and v is not None and not _typed(v, t):\n", + " out.append(f'recipe.{k}: expected {t.__name__}, got {v!r}')\n", + " elif row['accuracy_source'] == 'measured':\n", + " for f in ('ci_low', 'ci_high', 'ci_method', 'parity_metric', 'parity_floor'):\n", + " if row.get(f) is None: out.append(f'{f}: missing, required when the accuracy was measured')\n", + " if kind == 'quality' and row.get('parity') is not None and row.get('parity_metric') is None:\n", + " out.append('parity_metric: missing, required to read parity')\n", " for fields in _ORDERED[kind]:\n", " vals = [(f, row[f]) for f in fields if row.get(f) is not None]\n", " if [v for _, v in vals] != sorted(v for _, v in vals):\n", @@ -278,15 +330,37 @@ "def validate(\n", " row: dict, # one measurement row, of either kind\n", ") -> list[str]:\n", - " \"Every structural problem of `row`, each prefixed by its field name; an empty list means well formed\"\n", + " \"Every structural problem of `row`, each prefixed by its field name; an empty list means well formed. A row is read against the fields of ITS `schema_version`\"\n", " if not isinstance(row, dict): return [f'row: expected a dict, got {type(row).__name__}']\n", " kind = row.get('kind')\n", " if kind is None: return ['kind: missing']\n", " if not isinstance(kind, str) or kind not in FIELDS: return [f'kind: {kind!r} not one of {tuple(FIELDS)}']\n", - " fields = FIELDS[kind]\n", + " v = row.get('schema_version')\n", + " if v is not None and (isinstance(v, bool) or v not in FIELDS_BY_VERSION): # an unknown version: nothing else can be read\n", + " return _field_problems('schema_version', _SHARED['schema_version'], v)\n", + " fields = FIELDS_BY_VERSION.get(v, FIELDS)[kind] # each version against its own fields\n", " out = [f'{k}: unknown field for a {kind} row' for k in row if k not in fields]\n", " for name, spec in fields.items(): out += _field_problems(name, spec, row.get(name))\n", - " return out or _cross_problems(row)" + " return out or _cross_problems(row)\n", + "\n", + "\n", + "def upgrade(\n", + " row: dict, # a well-formed version-1 row\n", + " task: str, # what version 1 did not record: 'classification' or 'detection'\n", + " family: str | None = None, # and, optionally, the model family\n", + ") -> dict:\n", + " \"The version-2 row a version-1 row is: `task` (and `family`) added; on a quality row `argmax_parity` becomes `parity` with `parity_metric='argmax'` and `accuracy_source='measured'`. The result is validated; a version-2 row comes back unchanged.\"\n", + " if row.get('schema_version') == 2: return dict(row)\n", + " problems = validate(row)\n", + " if problems: raise ValueError('row not upgraded: ' + '; '.join(problems))\n", + " out = {**row, 'schema_version': 2, 'task': task, 'family': family}\n", + " if row['kind'] == 'quality':\n", + " out['accuracy_source'] = 'measured'\n", + " par = out.pop('argmax_parity', None)\n", + " out['parity'], out['parity_metric'] = par, 'argmax'\n", + " problems = validate(out)\n", + " if problems: raise ValueError('upgraded row refused: ' + '; '.join(problems))\n", + " return out\n" ] }, { @@ -319,6 +393,15 @@ " ('argmax_parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]}'),\n", " ),\n", "}\n", + "# version 2: a figure a vendor published is recorded, labelled, and never a measurement of ours\n", + "_VALIDITY_V2 = {\n", + " 'latency': _VALIDITY['latency'],\n", + " 'quality': (\n", + " ('accuracy_source', lambda v, r, m: v != 'measured', 'accuracy not measured by us ({v})'),\n", + " _VALIDITY['quality'][0],\n", + " ('parity', lambda v, r, m: v < r['parity_floor'], '{v} below parity_floor {r[parity_floor]} ({r[parity_metric]})'),\n", + " ),\n", + "}\n", "\n", "\n", "def status(\n", @@ -328,7 +411,7 @@ " \"('VALID', []) if `row` is a usable measurement, else ('INVALID', reasons); not a verdict against any budget\"\n", " reasons = validate(row)\n", " if reasons: return 'INVALID', reasons\n", - " for f, bad, why in _VALIDITY[row['kind']]:\n", + " for f, bad, why in (_VALIDITY_V2 if row['schema_version'] >= 2 else _VALIDITY)[row['kind']]:\n", " v = row.get(f)\n", " if v is None: reasons.append(f'{f}: not recorded')\n", " elif bad(v, row, max_spread_pct): reasons.append(f'{f}: ' + why.format(v=v, r=row, m=max_spread_pct))\n", @@ -414,16 +497,20 @@ "source": [ "#| export\n", "_OWN = tuple(f for f in _SHARED if f not in ('schema_version', 'kind')) # kept per side, prefixed\n", - "_BOTH = tuple(f for f in _LATENCY if f in _QUALITY) # one column, both sides must agree\n", + "_BOTH = tuple(f for f in _LATENCY_V2 if f in _QUALITY_V2) # one column, both sides must agree\n", "\n", "\n", "def _pair(lat, qual):\n", " \"One joined dict from a latency row and a quality row of the same engine\"\n", - " out = {'schema_version': lat['schema_version']}\n", + " if lat['schema_version'] != qual['schema_version']:\n", + " raise ValueError(f\"latency row {lat['run_id']} is version {lat['schema_version']}, quality row {qual['run_id']} \"\n", + " f\"version {qual['schema_version']}: `upgrade` the version-1 row before joining\")\n", + " out, ver = {'schema_version': lat['schema_version']}, FIELDS_BY_VERSION[lat['schema_version']]\n", + " both = [f for f in _BOTH if f in ver['latency'] and f in ver['quality']] # the shared columns of THIS version\n", " for side, r in (('latency', lat), ('quality', qual)):\n", " out |= {f'{side}_{f}': r.get(f) for f in _OWN}\n", - " out |= {f: r.get(f) for f in FIELDS[side] if f not in _SHARED and f not in _BOTH}\n", - " for f in _BOTH:\n", + " out |= {f: r.get(f) for f in ver[side] if f not in _SHARED and f not in both}\n", + " for f in both:\n", " a, b = lat.get(f), qual.get(f)\n", " if None not in (a, b) and a != b:\n", " raise ValueError(f\"{f}: latency row {lat['run_id']} says {a!r}, quality row {qual['run_id']} {b!r}, same engine\")\n", @@ -586,7 +673,9 @@ " for i, r in enumerate(rows):\n", " problems = validate(r)\n", " if problems: raise ValueError(f'row {i} refused: ' + '; '.join(problems))\n", - " fields = FIELDS[kinds[0]]\n", + " versions = sorted({r.get('schema_version') for r in rows})\n", + " if len(versions) != 1: raise ValueError(f'to_parquet writes one schema version per file, got {versions}: `upgrade` first')\n", + " fields = FIELDS_BY_VERSION[versions[0]][kinds[0]]\n", " schema = pa.schema([pa.field(k, getattr(pa, _ARROW[s['type']])(), nullable=not s['required']) for k, s in fields.items()])\n", " plain = [json.loads(_json(r)) for r in rows] # numpy scalars become Python numbers\n", " table = [{k: _canonical(r[k]) if fields[k]['type'] is dict and r.get(k) is not None else r.get(k) for k in fields}\n", @@ -657,6 +746,20 @@ "rows = measurement.read_rows('rows.jsonl')\n", "measurement.join(rows, rows) # each kind is picked from the list it belongs to\n", "measurement.to_parquet([r for r in rows if r['kind'] == 'latency'], 'latency.parquet')\n", + "```\n", + "\n", + "Version 2: a detection quality row scored through the board's engine, and the version-1 rows above upgraded:\n", + "\n", + "```python\n", + "det = dict(schema_version=2, kind='quality', task='detection', family='yolox', run_id='2026-10-14-d',\n", + " measured_at='2026-10-14T12:30:00Z', harness_commit='deadbee', model_id='megvii/yolox-nano',\n", + " model_revision='cd28f55', weights_license='apache-2.0', engine_hash='3' * 64, precision='fp16',\n", + " device='jetson-orin-nx-16gb', metric='map50_95', eval_set='coco-val2017-1000@seed0',\n", + " eval_set_hash='55e09567...', dataset_license='cc-by-4.0', eval_n=1000, accuracy=26.95, ci_low=25.90,\n", + " ci_high=29.24, ci_method='bootstrap', accuracy_source='measured', measured_through='deployed_engine',\n", + " parity=99.65, parity_metric='box_iou90', parity_floor=99.0)\n", + "measurement.status(det) # ('VALID', [])\n", + "measurement.upgrade(qual, task='classification')['parity'] # 99.7, with parity_metric='argmax'\n", "```" ] }, diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb index 13851d0..36ccb62 100644 --- a/nbs/tests/test_measurement.ipynb +++ b/nbs/tests/test_measurement.ipynb @@ -76,14 +76,16 @@ "metadata": {}, "outputs": [], "source": [ - "test_eq(set(_LAT), set(measurement.FIELDS['latency']))\n", - "test_eq(set(_QUAL), set(measurement.FIELDS['quality']))\n", - "test_eq(measurement.SCHEMA_VERSION, 1)\n", + "# these fixtures are version-1 rows, read against the version-1 fields (version 2 is tested below)\n", + "test_eq(set(_LAT), set(measurement.FIELDS_BY_VERSION[1]['latency']))\n", + "test_eq(set(_QUAL), set(measurement.FIELDS_BY_VERSION[1]['quality']))\n", + "test_eq(measurement.SCHEMA_VERSION, 2)\n", "for _r in (_LAT, _RAND, _QUAL): test_eq(measurement.validate(_r), [])\n", "\n", "# the shared fields are defined once, and the schema is data\n", "_shared = ('schema_version', 'kind', 'run_id', 'measured_at', 'series', 'harness_commit', 'fasterai_version', 'report_url')\n", - "for _f in _shared: test_is(measurement.FIELDS['latency'][_f], measurement.FIELDS['quality'][_f])\n", + "for _v in (1, 2):\n", + " for _f in _shared: test_is(measurement.FIELDS_BY_VERSION[_v]['latency'][_f], measurement.FIELDS_BY_VERSION[_v]['quality'][_f])\n", "\n", "# a latency row needs no accuracy, a quality row no latency\n", "test_eq(problems(_LAT, accuracy=78.9), ['accuracy: unknown field for a latency row'])\n", @@ -183,7 +185,7 @@ " test_eq(names(problems(_r, **{_f: _v, **({'int8_mode': None} if _f == 'precision' and _r is _LAT else {})})), {_f})\n", "\n", "# schema version mismatch\n", - "test_eq(problems(_LAT, schema_version=2), ['schema_version: 2 not one of (1,)'])\n", + "test_eq(problems(_LAT, schema_version=3), ['schema_version: 3 not one of (1, 2)'])\n", "\n", "# unknown fields: a derived throughput, a verdict, and a name from before the split\n", "test_eq(problems(_LAT, throughput_img_s=500.0), ['throughput_img_s: unknown field for a latency row'])\n", @@ -330,10 +332,10 @@ "test_eq(_row['schema_version'], 1)\n", "test_eq('kind' in _row or 'run_id' in _row, False)\n", "# every field of both sides, nothing overwritten\n", - "_both = [f for f in measurement.FIELDS['latency'] if f in measurement.FIELDS['quality']]\n", + "_both = [f for f in measurement.FIELDS_BY_VERSION[1]['latency'] if f in measurement.FIELDS_BY_VERSION[1]['quality']]\n", "_own = [f for f in _both if f not in ('schema_version', 'kind', 'engine_hash', 'precision', 'device', 'model_id',\n", " 'model_revision')]\n", - "test_eq(len(_row), len(measurement.FIELDS['latency']) + len(measurement.FIELDS['quality']) - len(_both) + len(_own) - 1)\n", + "test_eq(len(_row), len(measurement.FIELDS_BY_VERSION[1]['latency']) + len(measurement.FIELDS_BY_VERSION[1]['quality']) - len(_both) + len(_own) - 1)\n", "test_eq((_row['latency_ms_p50'], _row['accuracy'], _row['engine_hash']), (8.0, 78.9, _ENGINE))\n", "test_eq(measurement.throughput({**_row, 'kind': 'latency'}), 500.0)\n", "\n", @@ -487,7 +489,7 @@ "\n", "def stored(rows, kind):\n", " \"What a Parquet file of `rows` must hold: every field of the kind, None where absent\"\n", - " return [{k: r.get(k) for k in measurement.FIELDS[kind]} for r in rows]\n", + " return [{k: r.get(k) for k in measurement.FIELDS_BY_VERSION[1][kind]} for r in rows]\n", "\n", "\n", "def read_back(path):\n", @@ -514,7 +516,7 @@ "\n", " # the schema is explicit, derived from FIELDS: the file does not depend on the reader\n", " _schema = pq.read_schema(_pq)\n", - " test_eq(_schema.names, list(measurement.FIELDS['latency']))\n", + " test_eq(_schema.names, list(measurement.FIELDS_BY_VERSION[1]['latency']))\n", " for _f, _t in [('latency_ms_p50', pa.float64()), ('batch_size', pa.int64()), ('throttled', pa.bool_()),\n", " ('model_id', pa.string()), ('recipe', pa.string()), ('layer_precisions', pa.string())]:\n", " test_eq(_schema.field(_f).type, _t)\n", @@ -528,7 +530,7 @@ " _q = [_QUAL, {**_QUAL, 'run_id': 'q2', 'engine_hash': None, 'measured_through': 'dev_model', 'device': None}]\n", " measurement.to_parquet(_q, Path(_d, 'quality.parquet'))\n", " test_eq(read_back(Path(_d, 'quality.parquet')), stored(_q, 'quality'))\n", - " test_eq(pq.read_schema(Path(_d, 'quality.parquet')).names, list(measurement.FIELDS['quality']))\n", + " test_eq(pq.read_schema(Path(_d, 'quality.parquet')).names, list(measurement.FIELDS_BY_VERSION[1]['quality']))\n", "\n", " # one kind per file; no rows, no file; a malformed row is refused with its index; nothing is written\n", " with ExceptionExpected(ValueError, 'one kind per file'): measurement.to_parquet([_LAT, _QUAL], Path(_d, 'no.parquet'))\n", @@ -537,6 +539,74 @@ " test_eq(Path(_d, 'no.parquet').exists(), False)\n", "else: print('pyarrow not installed: to_parquet round trip skipped')" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "v2", + "metadata": {}, + "outputs": [], + "source": [ + "# ── version 2: task and family, detection parity, figures a vendor published, power, recipe keys ──\n", + "_LAT2 = measurement.upgrade(_LAT, task='classification', family='resnet')\n", + "_QUAL2 = measurement.upgrade(_QUAL, task='classification', family='resnet')\n", + "test_eq(_LAT2['schema_version'], 2); test_eq(_QUAL2['parity'], 99.7); test_eq(_QUAL2['parity_metric'], 'argmax')\n", + "assert 'argmax_parity' not in _QUAL2 and _QUAL2['accuracy_source'] == 'measured'\n", + "for _r in (_LAT2, _QUAL2): test_eq(measurement.validate(_r), []); test_eq(measurement.status(_r), ('VALID', []))\n", + "test_eq(measurement.upgrade(_LAT2, task='detection'), _LAT2) # a version-2 row is left as it is\n", + "with ExceptionExpected(ValueError, regex='not upgraded'): measurement.upgrade(edit(_LAT, run_id=...), task='classification')\n", + "# version-1 fields are not version-2 fields, and the other way round\n", + "test_eq(problems(_QUAL2, argmax_parity=99.7), ['argmax_parity: unknown field for a quality row'])\n", + "test_eq(problems(_QUAL, parity=99.7), ['parity: unknown field for a quality row'])\n", + "test_eq(problems(_LAT2, task=...), ['task: missing'])\n", + "test_eq(names(problems(_LAT2, task='segmentation')), {'task'})\n", + "\n", + "# a detection quality row, scored through the board's engine, with box parity\n", + "_DET = {**_QUAL2, 'run_id': 'q-det', 'task': 'detection', 'family': 'yolox', 'metric': 'map50_95',\n", + " 'eval_set': 'coco-val2017-1000@seed0', 'ci_method': 'bootstrap', 'accuracy': 26.95, 'ci_low': 25.9,\n", + " 'ci_high': 29.24, 'eval_n': 1000, 'parity': 99.65, 'parity_metric': 'box_iou90'}\n", + "test_eq(measurement.status(_DET), ('VALID', []))\n", + "test_eq(measurement.status(edit(_DET, parity=98.0))[1], ['parity: 98.0 below parity_floor 99.0 (box_iou90)'])\n", + "test_eq(measurement.status(edit(_DET, parity=None))[1], ['parity: not recorded'])\n", + "test_eq(problems(_DET, parity_metric='iou'), [\"parity_metric: 'iou' not one of ('argmax', 'box_iou90')\"])\n", + "\n", + "# a figure a vendor published: recorded and labelled, never one of our measurements\n", + "_VEND = {**{k: v for k, v in _DET.items() if k not in ('ci_low', 'ci_high', 'ci_method', 'parity', 'parity_metric',\n", + " 'parity_floor', 'engine_hash')},\n", + " 'run_id': 'q-vendor', 'accuracy_source': 'vendor_published', 'measured_through': 'dev_model', 'accuracy': 25.8}\n", + "test_eq(measurement.validate(_VEND), [])\n", + "test_eq(measurement.status(_VEND)[0], 'INVALID')\n", + "assert 'accuracy_source: accuracy not measured by us (vendor_published)' in measurement.status(_VEND)[1]\n", + "# a measured accuracy states its interval and its parity rule\n", + "test_eq(names(problems(_DET, ci_low=None, parity_metric=None)), {'ci_low', 'parity_metric'})\n", + "test_eq(problems(_LAT2, latency_source='vendor_published'), [])\n", + "test_eq(measurement.status(edit(_LAT2, latency_source='vendor_published'))[1], ['latency_source: latency not measured on the device'])\n", + "\n", + "# power, and the recipe keys a variant is described by (checked for type; other keys stay free)\n", + "test_eq(problems(_LAT2, power_w=9.5), []); test_eq(names(problems(_LAT2, power_w=0.0)), {'power_w'})\n", + "_rec = dict(resolution=416, num_classes=1, pruned=True, prune_method='taylor', prune_ratio=0.25, distilled=True,\n", + " teacher='yolox-s', qat=True, roi_crop=[0, 120, 640, 480], anything_else={'free': 1})\n", + "test_eq(problems(_LAT2, recipe=_rec), [])\n", + "test_eq(sorted(problems(_LAT2, recipe={**_rec, 'pruned': 'yes', 'num_classes': 1.5})),\n", + " ['recipe.num_classes: expected int, got 1.5', \"recipe.pruned: expected bool, got 'yes'\"])\n", + "test_eq(problems(_LAT, recipe={'pruned': 'yes'}), []) # version 1 never checked its recipe\n", + "\n", + "# the join: one version per pair; task and family are one column both sides agree on\n", + "_L2 = {**_LAT2, 'engine_hash': _ENGINE}; _Q2 = {**_QUAL2, 'engine_hash': _ENGINE}\n", + "_j = measurement.join([_L2], [_Q2])\n", + "test_eq(len(_j), 1); test_eq((_j[0]['task'], _j[0]['family']), ('classification', 'resnet'))\n", + "with ExceptionExpected(ValueError, regex='family'): measurement.join([_L2], [{**_Q2, 'family': 'vgg'}])\n", + "with ExceptionExpected(ValueError, regex='upgrade'): measurement.join([_LAT], [_Q2])\n", + "# Parquet: one version per file, with that version's columns\n", + "with tempfile.TemporaryDirectory() as _d:\n", + " try:\n", + " import pyarrow.parquet as pq\n", + " measurement.to_parquet([_DET], Path(_d, 'q2.parquet'))\n", + " test_eq(pq.read_schema(Path(_d, 'q2.parquet')).names, list(measurement.FIELDS_BY_VERSION[2]['quality']))\n", + " with ExceptionExpected(ValueError, regex='one schema version'): measurement.to_parquet([_QUAL, _QUAL2], Path(_d, 'x.parquet'))\n", + " except ImportError: pass\n", + "print('schema v2: ok')" + ] } ], "metadata": { From 581ca228d5397f9f5f07374ec8972824e7dc9cff Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Wed, 14 Oct 2026 14:34:36 +0200 Subject: [PATCH 2/2] measurement v2: runtime may be stedgeai (ST Edge AI, STM32N6) --- CHANGELOG.md | 2 +- fastermodels/measurement.py | 2 ++ nbs/04_measurement.ipynb | 4 +++- nbs/tests/test_measurement.ipynb | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df8f270..a93e68b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `measurement` schema version 2 (`SCHEMA_VERSION = 2`, `SCHEMA_VERSIONS = (1, 2)`, `FIELDS_BY_VERSION`): `task` and `family` on both kinds; detection parity (`parity` + `parity_metric`: `argmax` or `box_iou90`, replacing `argmax_parity`); figures a vendor published (`latency_source` / `accuracy_source` = - `vendor_published`), well formed and never `VALID` as a measurement; `power_w`; the known `recipe` keys + `vendor_published`), well formed and never `VALID` as a measurement; `runtime` may be `stedgeai`; `power_w`; the known `recipe` keys checked for type (`RECIPE_KEYS`). - `upgrade(row, task, family=None)`: the version-2 row a version-1 row is. diff --git a/fastermodels/measurement.py b/fastermodels/measurement.py index 59d727a..424e2d6 100644 --- a/fastermodels/measurement.py +++ b/fastermodels/measurement.py @@ -113,6 +113,8 @@ _LATENCY_V2 = {'task': _TASK, 'family': _FAMILY, **{k: v for k, v in _LATENCY.items() if k != 'latency_source'}, 'latency_source': dict(type=str, required=True, choices=_SOURCE), + 'runtime': dict(type=str, required=True, # + ST Edge AI (STM32N6 NPU) + choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera', 'stedgeai')), 'power_w': dict(type=float, required=False, gt=0)} _QUALITY_V2 = {'task': _TASK, 'family': _FAMILY, **{k: v for k, v in _QUALITY.items() if k not in ('argmax_parity', 'parity_floor', 'ci_low', 'ci_high', 'ci_method')}, diff --git a/nbs/04_measurement.ipynb b/nbs/04_measurement.ipynb index 591ee36..84f3818 100644 --- a/nbs/04_measurement.ipynb +++ b/nbs/04_measurement.ipynb @@ -92,7 +92,7 @@ "| Kind | Change |\n", "|---|---|\n", "| both | `task` (`classification` or `detection`, required) and `family` (e.g. `yolox`); one column after `join`, both sides must agree |\n", - "| latency | `latency_source` may be `vendor_published`; `power_w`; the known `recipe` keys are checked for type (`RECIPE_KEYS`: `resolution`, `num_classes`, `roi_crop`, `pruned`, `prune_method`, `prune_ratio`, `distilled`, `teacher`, `qat`, `quant`, `calib_n`), any other key stays free |\n", + "| latency | `latency_source` may be `vendor_published`; `runtime` may be `stedgeai` (ST Edge AI, STM32N6); `power_w`; the known `recipe` keys are checked for type (`RECIPE_KEYS`: `resolution`, `num_classes`, `roi_crop`, `pruned`, `prune_method`, `prune_ratio`, `distilled`, `teacher`, `qat`, `quant`, `calib_n`), any other key stays free |\n", "| quality | `accuracy_source` (`measured` or `vendor_published`, required); `argmax_parity` becomes `parity` with `parity_metric` (`argmax` for classification, `box_iou90` for detection: same class, IoU >= 0.9); `ci_low`, `ci_high`, `ci_method`, `parity_metric` and `parity_floor` are required when the accuracy was measured, absent when a vendor published it |\n", "\n", "- `series` is a free-text label grouping the rows measured in one session, typically one night; it is not part of any key.\n", @@ -228,6 +228,8 @@ "_LATENCY_V2 = {'task': _TASK, 'family': _FAMILY,\n", " **{k: v for k, v in _LATENCY.items() if k != 'latency_source'},\n", " 'latency_source': dict(type=str, required=True, choices=_SOURCE),\n", + " 'runtime': dict(type=str, required=True, # + ST Edge AI (STM32N6 NPU)\n", + " choices=('tensorrt', 'openvino', 'onnxruntime', 'hailo', 'axelera', 'stedgeai')),\n", " 'power_w': dict(type=float, required=False, gt=0)}\n", "_QUALITY_V2 = {'task': _TASK, 'family': _FAMILY,\n", " **{k: v for k, v in _QUALITY.items() if k not in ('argmax_parity', 'parity_floor', 'ci_low', 'ci_high', 'ci_method')},\n", diff --git a/nbs/tests/test_measurement.ipynb b/nbs/tests/test_measurement.ipynb index 36ccb62..182508d 100644 --- a/nbs/tests/test_measurement.ipynb +++ b/nbs/tests/test_measurement.ipynb @@ -582,6 +582,9 @@ "test_eq(problems(_LAT2, latency_source='vendor_published'), [])\n", "test_eq(measurement.status(edit(_LAT2, latency_source='vendor_published'))[1], ['latency_source: latency not measured on the device'])\n", "\n", + "# ST Edge AI is a runtime from version 2 on; version 1 never knew it\n", + "test_eq(problems(_LAT2, runtime='stedgeai'), [])\n", + "test_eq(names(problems(_LAT, runtime='stedgeai')), {'runtime'})\n", "# power, and the recipe keys a variant is described by (checked for type; other keys stay free)\n", "test_eq(problems(_LAT2, power_w=9.5), []); test_eq(names(problems(_LAT2, power_w=0.0)), {'power_w'})\n", "_rec = dict(resolution=416, num_classes=1, pruned=True, prune_method='taylor', prune_ratio=0.25, distilled=True,\n",