From a9ff9af2c7736efce3848f284c17475474cbdada Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Mon, 12 Oct 2026 08:09:01 -0400 Subject: [PATCH 1/4] feat(explorer): distinguish execution variants in identity, ranking and labels Pipeline records variant_id, variant_label, requested execution engine and gateway routing from bundle platform blocks into the read model, cohort rows and leaderboard aggregates. Engine variants rank as separate rows, routed gateway runs are excluded from native ranking, and legacy bundles keep platform-id identity. Read model advances to version 16. --- CHANGELOG.md | 9 + .../scripts/explorer_pipeline/contract.py | 2 +- .../explorer_pipeline/duckdb_builder.py | 36 ++- _project/scripts/explorer_pipeline/models.py | 64 +++++ .../scripts/explorer_pipeline/pipeline.py | 18 +- .../scripts/explorer_pipeline/transformer.py | 44 ++++ docs/development/browser-duckdb-schema.sql | 18 +- results-data/CORPUS_NOTES.md | 7 + results-data/SEED_CORPUS_SPEC.md | 6 +- results-data/validate_corpus.py | 3 + .../src/__tests__/dbRecovery.test.ts | 2 +- .../src/components/ComparabilityReceipt.tsx | 10 + .../src/components/MetaLeaderboard.tsx | 28 ++- .../__tests__/ComparabilityReceipt.test.tsx | 20 ++ results-explorer/src/db.ts | 2 +- .../lib/__tests__/db-remediation-pin.test.ts | 2 +- results-explorer/src/pages/Leaderboard.tsx | 18 +- results-explorer/src/types.ts | 9 + scripts/publication/check_explorer_compat.py | 61 ++++- .../test_duckdb_browser_contract.py | 18 +- .../test_variant_identity.py | 220 ++++++++++++++++++ .../publication/test_check_explorer_compat.py | 50 ++-- .../unit/scripts/test_corpus_cohort_depth.py | 53 +++++ 23 files changed, 640 insertions(+), 60 deletions(-) create mode 100644 tests/unit/scripts/explorer_pipeline/test_variant_identity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index da27de51ff..e86d451289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Documentation +- The Explorer read model is at version 16 with execution-variant identity: + the pipeline records `variant_id`, `variant_label`, the requested execution + engine and gateway routing on every result, cohort and leaderboard row. + Two bundles that differ only in requested engine rank as separate variants + ("Polars (streaming)" beside "Polars"), and gateway-routed runs are + excluded from native ranking with `ranking_exclusion_reason=gateway_routed`. + Existing corpus values are unchanged: bundles without variant data keep + `variant_id` equal to the platform id. + - The Polars version-matrix reproduction instructions now use `--execution-engine` for default, in-memory and streaming setups, and identify the earlier `engine` option name used by the recorded bundles. diff --git a/_project/scripts/explorer_pipeline/contract.py b/_project/scripts/explorer_pipeline/contract.py index 12c983703c..435bb49cc4 100644 --- a/_project/scripts/explorer_pipeline/contract.py +++ b/_project/scripts/explorer_pipeline/contract.py @@ -1,7 +1,7 @@ from __future__ import annotations EXPLORER_BUILD_CONTRACT_VERSION = "6" -EXPLORER_READ_MODEL_VERSION = 15 +EXPLORER_READ_MODEL_VERSION = 16 EXPLORER_READ_MODEL_COMPATIBILITY = { "minimum_supported": EXPLORER_READ_MODEL_VERSION, "newer_policy": "warn-and-continue", diff --git a/_project/scripts/explorer_pipeline/duckdb_builder.py b/_project/scripts/explorer_pipeline/duckdb_builder.py index 00a20a910d..7cd233b7da 100644 --- a/_project/scripts/explorer_pipeline/duckdb_builder.py +++ b/_project/scripts/explorer_pipeline/duckdb_builder.py @@ -330,6 +330,11 @@ def _create_schema(self, con: Any) -> None: scale_factor DOUBLE NOT NULL, platform VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, + requested_execution_engine VARCHAR NOT NULL, + gateway_name VARCHAR, + gateway_routed BOOLEAN NOT NULL, driver_version VARCHAR, run_date VARCHAR NOT NULL, power_score DOUBLE, @@ -403,6 +408,11 @@ def _create_schema(self, con: Any) -> None: r.scale_factor, r.platform, r.platform_id, + r.variant_id, + r.variant_label, + r.requested_execution_engine, + r.gateway_name, + r.gateway_routed, r.driver_version, r.run_date, r.power_score, @@ -527,6 +537,8 @@ def _create_schema(self, con: Any) -> None: phase VARCHAR NOT NULL, result_id VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, short_id VARCHAR NOT NULL, trust_label VARCHAR NOT NULL, @@ -575,6 +587,11 @@ def _create_schema(self, con: Any) -> None: scale_factor, platform, platform_id, + variant_id, + variant_label, + requested_execution_engine, + gateway_name, + gateway_routed, driver_version, run_date, power_score, @@ -629,6 +646,8 @@ def _create_schema(self, con: Any) -> None: primary_metric VARCHAR NOT NULL, primary_order VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, result_id VARCHAR NOT NULL, short_id VARCHAR NOT NULL, @@ -653,7 +672,9 @@ def _create_schema(self, con: Any) -> None: con.execute("CREATE INDEX idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id)") con.execute(""" CREATE TABLE meta_leaderboard ( - platform_id VARCHAR PRIMARY KEY, + variant_id VARCHAR PRIMARY KEY, + variant_label VARCHAR NOT NULL, + platform_id VARCHAR NOT NULL, platform VARCHAR NOT NULL, avg_rank DOUBLE, n_cohorts INTEGER NOT NULL @@ -774,6 +795,11 @@ def _populate_results( entry.scale_factor, entry.platform, entry.platform_id, + entry.variant_id, + entry.variant_label, + entry.requested_execution_engine, + entry.gateway_name, + entry.gateway_routed, entry.driver_version, entry.run_date, entry.power_score, @@ -971,6 +997,8 @@ def _populate_benchmark_rankings( phase, platform_row.result_id, platform_row.platform_id, + platform_row.variant_id, + platform_row.variant_label, platform_row.platform, full_to_short.get(platform_row.result_id, ""), platform_row.trust_label, @@ -1094,6 +1122,8 @@ def _populate_cohort_metadata( primary_metric, primary_order, p["platform_id"], + p.get("variant_id") or p["platform_id"], + p.get("variant_label") or p["platform"], p["platform"], p["result_id"], p["short_id"], @@ -1123,6 +1153,8 @@ def _populate_meta_leaderboard(self, con: Any, meta: dict[str, Any]) -> None: for p in meta.get("platforms", []): rows.append( ( + p.get("variant_id") or p["platform_id"], + p.get("variant_label") or p["platform"], p["platform_id"], p["platform"], p.get("avg_rank"), @@ -1130,7 +1162,7 @@ def _populate_meta_leaderboard(self, con: Any, meta: dict[str, Any]) -> None: ) ) if rows: - con.executemany("INSERT INTO meta_leaderboard VALUES (?, ?, ?, ?)", rows) + con.executemany("INSERT INTO meta_leaderboard VALUES (?, ?, ?, ?, ?, ?)", rows) def _populate_short_ids(self, con: Any, short_id_map: dict[str, str]) -> None: rows = list(short_id_map.items()) diff --git a/_project/scripts/explorer_pipeline/models.py b/_project/scripts/explorer_pipeline/models.py index 87638fda63..1cb43470e5 100644 --- a/_project/scripts/explorer_pipeline/models.py +++ b/_project/scripts/explorer_pipeline/models.py @@ -62,6 +62,23 @@ def _platform_id(raw: str) -> str: return stripped.lower().replace(" ", "-") +def variant_label( + platform: str, + *, + variant_id: str, + requested_execution_engine: str = "default", + gateway_name: str | None = None, + gateway_routed: bool = False, +) -> str: + if gateway_routed and gateway_name: + return f"{platform} via {str(gateway_name).capitalize()}" + if requested_execution_engine and requested_execution_engine != "default": + return f"{platform} ({requested_execution_engine})" + if variant_id and variant_id != _platform_id(platform): + return f"{platform} ({variant_id})" + return platform + + RANKING_ELIGIBLE_VISIBILITIES: frozenset[str] = frozenset( { "public-curated", @@ -95,6 +112,11 @@ class ManifestEntry(BaseModel): scale_factor: float platform: str platform_id: str = "" + variant_id: str = "" + variant_label: str = "" + requested_execution_engine: str = "default" + gateway_name: str | None = None + gateway_routed: bool = False driver_version: str | None run_date: str power_score: float | None @@ -147,6 +169,22 @@ class ManifestEntry(BaseModel): def _serialize_normalized_cost(self, cost: NormalizedCost) -> dict[str, Any]: return cost.to_dict() + @model_validator(mode="after") + def _default_variant_identity(self) -> ManifestEntry: + if not self.variant_id: + self.variant_id = self.platform_id + if not self.variant_label: + self.variant_label = variant_label( + self.platform, + variant_id=self.variant_id, + requested_execution_engine=self.requested_execution_engine, + gateway_name=self.gateway_name, + gateway_routed=self.gateway_routed, + ) + if not self.requested_execution_engine: + self.requested_execution_engine = "default" + return self + @model_validator(mode="after") def _default_logical_query_count(self) -> ManifestEntry: if self.logical_query_count == 0 and self.query_count > 0: @@ -315,6 +353,11 @@ class DetailResult(BaseModel): scale_factor: float platform: str platform_id: str = "" + variant_id: str = "" + variant_label: str = "" + requested_execution_engine: str = "default" + gateway_name: str | None = None + gateway_routed: bool = False driver_version: str | None run_date: str total_duration_s: float @@ -365,6 +408,22 @@ class DetailResult(BaseModel): physical_rendering_id: str | None = None basis_availability: BasisAvailability | None = None + @model_validator(mode="after") + def _default_variant_identity(self) -> DetailResult: + if not self.variant_id: + self.variant_id = self.platform_id + if not self.variant_label: + self.variant_label = variant_label( + self.platform, + variant_id=self.variant_id, + requested_execution_engine=self.requested_execution_engine, + gateway_name=self.gateway_name, + gateway_routed=self.gateway_routed, + ) + if not self.requested_execution_engine: + self.requested_execution_engine = "default" + return self + @field_serializer("normalized_cost") def _serialize_normalized_cost(self, cost: NormalizedCost) -> dict[str, Any]: return cost.to_dict() @@ -408,6 +467,8 @@ def ranking_exclusion_reason(entry: ManifestEntry, primary_metric: str | None = return "tuning_not_applied" if entry.comparison_exclusion_reason is not None: return entry.comparison_exclusion_reason + if entry.gateway_routed: + return "gateway_routed" metric = primary_metric or get_ranking_config(entry.benchmark, canonical_phase(entry.test_type)).primary_metric value = primary_metric_value(entry, metric) @@ -529,6 +590,8 @@ class PlatformRow(BaseModel): result_id: str short_id: str = "" platform_id: str + variant_id: str = "" + variant_label: str = "" platform: str platform_version: str | None tuning_mode: str | None @@ -907,4 +970,5 @@ def _coerce_cost_block(cls, value: Any) -> Any: "timing_eligibility", "timing_exclusion_reason", "unavailable_normalized_cost_payload", + "variant_label", ] diff --git a/_project/scripts/explorer_pipeline/pipeline.py b/_project/scripts/explorer_pipeline/pipeline.py index b85af9f292..950b56a6cb 100644 --- a/_project/scripts/explorer_pipeline/pipeline.py +++ b/_project/scripts/explorer_pipeline/pipeline.py @@ -313,6 +313,8 @@ def _build_benchmark_summaries( result_id=entry.result_id, short_id=full_to_short.get(entry.result_id, ""), platform_id=entry.platform_id, + variant_id=entry.variant_id, + variant_label=entry.variant_label, platform=entry.platform, platform_version=entry.platform_version, tuning_mode=entry.tuning_mode, @@ -412,6 +414,8 @@ def _rank_platforms_in_cohort( entries.append( { "platform_id": row.platform_id, + "variant_id": row.variant_id, + "variant_label": row.variant_label, "platform": row.platform, "result_id": row.result_id, "short_id": full_to_short.get(row.result_id, ""), @@ -434,13 +438,15 @@ def _compute_average_ranks(platform_agg: dict[str, dict[str, Any]]) -> list[dict records.append( { "platform_id": pdata["platform_id"], + "variant_id": pdata.get("variant_id") or pdata["platform_id"], + "variant_label": pdata.get("variant_label") or pdata["platform"], "platform": pdata["platform"], "ranks": ranks, "avg_rank": avg_rank, "n_cohorts": len(ranks), } ) - records.sort(key=lambda p: (p["avg_rank"] is None, p["avg_rank"] or float("inf"), p["platform_id"])) + records.sort(key=lambda p: (p["avg_rank"] is None, p["avg_rank"] or float("inf"), p["variant_id"])) return records @@ -473,9 +479,15 @@ def _build_meta_leaderboard( rank = entry["rank"] if rank is None: continue - pid = entry["platform_id"] + pid = entry["variant_id"] or entry["platform_id"] if pid not in platform_agg: - platform_agg[pid] = {"platform_id": pid, "platform": entry["platform"], "ranks": {}} + platform_agg[pid] = { + "platform_id": entry["platform_id"], + "variant_id": entry["variant_id"], + "variant_label": entry["variant_label"], + "platform": entry["platform"], + "ranks": {}, + } existing = platform_agg[pid]["ranks"].get(cohort_key) if existing is None or rank < existing["rank"]: platform_agg[pid]["ranks"][cohort_key] = { diff --git a/_project/scripts/explorer_pipeline/transformer.py b/_project/scripts/explorer_pipeline/transformer.py index 32a6accbf3..14cb971df6 100644 --- a/_project/scripts/explorer_pipeline/transformer.py +++ b/_project/scripts/explorer_pipeline/transformer.py @@ -8,6 +8,7 @@ import math import re from collections import Counter +from collections.abc import Mapping from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, cast @@ -32,6 +33,7 @@ canonical_phase, ranking_exclusion_reason, timing_eligibility, + variant_label, ) from benchbox.core.benchmark_registry import get_benchmark_support_status from benchbox.core.cost.models import CostScope, CostStatus, DeploymentMetadata, NormalizedCost @@ -394,6 +396,36 @@ def _platform_version(bundle: BundleDocument) -> str | None: return str(val) if val and val != "unknown" else None +def _variant_info(bundle: BundleDocument) -> tuple[str, str, str, str | None, bool]: + platform = bundle.platform + platform_name = str(platform.name) + pid = _platform_id(platform_name) + raw_variant = getattr(platform, "variant", None) + engine_raw = getattr(platform, "execution_engine", None) + engine = engine_raw if isinstance(engine_raw, Mapping) else {} + requested = engine.get("requested") or "default" + requested = str(requested) + gateway_raw = getattr(platform, "gateway", None) + gateway = gateway_raw if isinstance(gateway_raw, Mapping) else {} + routed = bool(gateway.get("routed", False)) + gateway_name = gateway.get("name") + gateway_name = str(gateway_name) if gateway_name else None + if isinstance(raw_variant, str) and raw_variant.strip(): + variant_id = raw_variant.strip() + elif requested != "default": + variant_id = f"{pid}+{requested}" + else: + variant_id = pid + label = variant_label( + platform_name, + variant_id=variant_id, + requested_execution_engine=requested, + gateway_name=gateway_name, + gateway_routed=routed, + ) + return variant_id, label, requested, gateway_name, routed + + _EXECUTION_MODES = frozenset({"sql", "dataframe"}) @@ -1181,12 +1213,18 @@ def to_manifest_entry( ) override = _override_display(bundle_path) test_type = _test_type(bundle) + variant = _variant_info(bundle) entry = ManifestEntry( result_id=rid, benchmark=benchmark, scale_factor=scale_factor, platform=platform, platform_id=_platform_id(platform), + variant_id=variant[0], + variant_label=variant[1], + requested_execution_engine=variant[2], + gateway_name=variant[3], + gateway_routed=variant[4], driver_version=_driver_version(bundle), run_date=run_date, power_score=_power_score(bundle), @@ -1273,12 +1311,18 @@ def to_detail_result( timing_contract = timing_eligibility(display_timings, logical_query_count) normalized_cost = _normalized_cost(bundle) test_type = _test_type(bundle) + variant = _variant_info(bundle) detail = DetailResult( result_id=result_id, benchmark=benchmark, scale_factor=scale_factor, platform=platform, platform_id=_platform_id(platform), + variant_id=variant[0], + variant_label=variant[1], + requested_execution_engine=variant[2], + gateway_name=variant[3], + gateway_routed=variant[4], driver_version=_driver_version(bundle), run_date=run_date, total_duration_s=total_duration_s, diff --git a/docs/development/browser-duckdb-schema.sql b/docs/development/browser-duckdb-schema.sql index e7fb6fe0d2..acd4312ef7 100644 --- a/docs/development/browser-duckdb-schema.sql +++ b/docs/development/browser-duckdb-schema.sql @@ -41,6 +41,11 @@ CREATE TABLE IF NOT EXISTS results ( scale_factor DOUBLE NOT NULL, platform VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, + requested_execution_engine VARCHAR NOT NULL, + gateway_name VARCHAR, + gateway_routed BOOLEAN NOT NULL, driver_version VARCHAR, run_date VARCHAR NOT NULL, power_score DOUBLE, @@ -136,6 +141,11 @@ SELECT r.scale_factor, r.platform, r.platform_id, + r.variant_id, + r.variant_label, + r.requested_execution_engine, + r.gateway_name, + r.gateway_routed, r.driver_version, r.run_date, r.power_score, @@ -237,6 +247,8 @@ CREATE TABLE IF NOT EXISTS benchmark_rankings ( phase VARCHAR NOT NULL, result_id VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, short_id VARCHAR NOT NULL, trust_label VARCHAR NOT NULL, @@ -339,6 +351,8 @@ CREATE TABLE IF NOT EXISTS cohort_metadata ( primary_metric VARCHAR NOT NULL, primary_order VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, result_id VARCHAR NOT NULL, short_id VARCHAR NOT NULL, @@ -366,7 +380,9 @@ CREATE INDEX IF NOT EXISTS idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id); CREATE TABLE IF NOT EXISTS meta_leaderboard ( - platform_id VARCHAR PRIMARY KEY, + variant_id VARCHAR PRIMARY KEY, + variant_label VARCHAR NOT NULL, + platform_id VARCHAR NOT NULL, platform VARCHAR NOT NULL, avg_rank DOUBLE, n_cohorts INTEGER NOT NULL diff --git a/results-data/CORPUS_NOTES.md b/results-data/CORPUS_NOTES.md index 23be13d40d..8b55c52bdf 100644 --- a/results-data/CORPUS_NOTES.md +++ b/results-data/CORPUS_NOTES.md @@ -64,6 +64,13 @@ Both cohorts meet the >=3-platform depth criterion required for the Compare view - `tpch SF=0.01`: DuckDB, DataFusion, Polars - `star_schema SF=0.01`: DuckDB, DataFusion, SQLite +## Execution-variant identity (schema 2.3 and later) +Bundles that record `platform.variant` count toward cohort depth under the variant +string; every bundle committed here predates that field, so all existing identities +are unchanged platform names and no cohort gains or loses depth from the rename. +Gateway-routed bundles are excluded from native ranking with +`ranking_exclusion_reason=gateway_routed` and contribute no identity. + ## Zero-query DataFrame withdrawal (2026-08-24) Sixty legacy DataFrame bundles were withdrawn because they reported diff --git a/results-data/SEED_CORPUS_SPEC.md b/results-data/SEED_CORPUS_SPEC.md index 66fa3f857f..635aa41bd5 100644 --- a/results-data/SEED_CORPUS_SPEC.md +++ b/results-data/SEED_CORPUS_SPEC.md @@ -18,7 +18,11 @@ Phase 3 hosted service. The hard requirement today is still the validator gate in `results-data/validate_corpus.py`: every committed cohort must have at least 3 distinct comparison identities. Ordinary -cohorts count platform names, preserving the historical three-platform floor. Bundles in +cohorts count platform names, preserving the historical three-platform floor. A bundle +that records `platform.variant` counts under that variant string instead of the bare +platform name, so two execution variants of one platform (for example +`polars-df+streaming` alongside `polars-df`) are distinct identities while repeated +runs of one variant do not pad the floor. Bundles in the explicitly segregated `duckdb-version-matrix/` corpus count platform plus version; repeated runs of one platform/version do not pad that matrix. diff --git a/results-data/validate_corpus.py b/results-data/validate_corpus.py index 98c1c04793..8ed295d10b 100644 --- a/results-data/validate_corpus.py +++ b/results-data/validate_corpus.py @@ -426,6 +426,9 @@ def _comparison_identity(bundle: pathlib.Path, payload: dict) -> str: except Exception as exc: raise CorpusReadError(f"ERROR reading platform identity from {bundle}: {exc}") from exc identity = str(platform) + variant = platform_section.get("variant") + if variant and str(variant) != str(platform): + identity = str(variant) if version and str(version) != "unknown": identity = f"{identity} v{str(version)[:120]}" return identity diff --git a/results-explorer/src/__tests__/dbRecovery.test.ts b/results-explorer/src/__tests__/dbRecovery.test.ts index 27178611bd..ba8eaa7b90 100644 --- a/results-explorer/src/__tests__/dbRecovery.test.ts +++ b/results-explorer/src/__tests__/dbRecovery.test.ts @@ -23,7 +23,7 @@ const hoisted = vi.hoisted(() => { ((sql: string, params: unknown[], attempt: number) => Array>) | null = null; function healthyQueryImpl(sql: string): FakeRows { - if (/read_model_version/i.test(sql)) return rows([{ read_model_version: 15 }]); + if (/read_model_version/i.test(sql)) return rows([{ read_model_version: 16 }]); if (/^ATTACH/i.test(sql)) return rows([]); if (/^SET /i.test(sql)) return rows([]); if (/COUNT\(\*\)/i.test(sql)) return rows([{ n: 1 }]); diff --git a/results-explorer/src/components/ComparabilityReceipt.tsx b/results-explorer/src/components/ComparabilityReceipt.tsx index bd0b9dc170..ebd67760bf 100644 --- a/results-explorer/src/components/ComparabilityReceipt.tsx +++ b/results-explorer/src/components/ComparabilityReceipt.tsx @@ -98,6 +98,16 @@ export function buildComparabilityFields(results: DetailResult[]): Comparability compareValues("Platform version", results, (result) => valueOrMissing(result.platform_version)), compareValues("Driver version", results, (result) => valueOrMissing(result.driver_version)), compareValues("Execution mode", results, (result) => result.execution_mode ? formatExecutionMode(result.execution_mode) : "Not recorded"), + compareValues( + "Execution engine", + results, + (result) => result.requested_execution_engine ?? "Not recorded", + ), + compareValues( + "Gateway", + results, + (result) => (result.gateway_routed ? `Routed via ${result.gateway_name ?? "unknown"}` : "Native"), + ), buildTuningField(results), compareValues( "Validation", diff --git a/results-explorer/src/components/MetaLeaderboard.tsx b/results-explorer/src/components/MetaLeaderboard.tsx index 7922ad2e0a..bb552fb45b 100644 --- a/results-explorer/src/components/MetaLeaderboard.tsx +++ b/results-explorer/src/components/MetaLeaderboard.tsx @@ -96,7 +96,7 @@ export function MetaLeaderboard({ for (const cohort of cohorts) { const values = platforms .map((platform) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); const state = metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort); return state.kind === "ranked" ? cellShadeMetric(state.rank, mode, cohort) : null; }) @@ -121,11 +121,11 @@ export function MetaLeaderboard({ [cohorts, sortedPlatforms], ); const collapsedPlatformIds = useMemo( - () => new Set(collapsedPlatforms.map((platform) => platform.platform_id)), + () => new Set(collapsedPlatforms.map((platform) => variantKeyOf(platform))), [collapsedPlatforms], ); const defaultPlatforms = useMemo( - () => sortedPlatforms.filter((platform) => !collapsedPlatformIds.has(platform.platform_id)), + () => sortedPlatforms.filter((platform) => !collapsedPlatformIds.has(variantKeyOf(platform))), [collapsedPlatformIds, sortedPlatforms], ); const renderedPlatforms = showUnrankedPlatforms @@ -359,7 +359,7 @@ export function MetaLeaderboard({ {visiblePlatforms.map((platform, rowIdx) => ( route(platformHref(platform.platform_id))} > @@ -370,7 +370,7 @@ export function MetaLeaderboard({ class="no-underline hover:text-[var(--bb-accent-hover)]" onClick={(event) => event.stopPropagation()} > - {platform.platform} + {variantLabelOf(platform)} {cohorts.map((cohort, colIdx) => { const rank = platform.ranks[cohort.key]; - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); const cellState = metaLeaderboardCellState(rank, result, cohort); const missing = cellState.kind === "missing"; const unranked = cellState.kind === "unranked"; @@ -595,8 +595,16 @@ function exactMetricTitle(rank: MetaRank | undefined, cohort: MetaCohort): strin return null; } +export function variantKeyOf(row: { variant_id?: string | null; platform_id: string }): string { + return row.variant_id ?? row.platform_id; +} + +export function variantLabelOf(row: { variant_label?: string | null; platform: string }): string { + return row.variant_label ?? row.platform; +} + function findCohortPlatform(cohort: MetaCohort, platformId: string): MetaCohortPlatform | undefined { - const rows = cohort.platforms?.filter((row) => row.platform_id === platformId); + const rows = cohort.platforms?.filter((row) => variantKeyOf(row) === platformId); if (!rows || rows.length === 0) return undefined; return rows.find((row) => row.rank !== null) ?? rows[0]; } @@ -748,12 +756,12 @@ function runAgeText(runAge: string | null): string | null { } function hasPublishedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boolean { - return cohorts.some((cohort) => findCohortPlatform(cohort, platform.platform_id) !== undefined); + return cohorts.some((cohort) => findCohortPlatform(cohort, variantKeyOf(platform)) !== undefined); } function hasRankedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boolean { return cohorts.some((cohort) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); return metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort).kind === "ranked"; }); } @@ -761,7 +769,7 @@ function hasRankedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boole function allExcludedRankingReason(cohort: MetaCohort, platforms: MetaPlatform[]): string | null { const results = cohort.platforms ?? []; const hasRankedCell = platforms.some((platform) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); return metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort).kind === "ranked"; }); if (results.length === 0 || hasRankedCell) return null; diff --git a/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx b/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx index f998a8d8f2..1e88b4b405 100644 --- a/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx +++ b/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx @@ -76,6 +76,26 @@ describe("ComparabilityReceipt", () => { expect(receipt).not.toHaveTextContent("benchbox.core.cost.pricing"); }); + it("shows execution engine and gateway routing", () => { + const fields = buildComparabilityFields([ + makeDetail({ requested_execution_engine: "default" }), + makeDetail({ + result_id: "r2", + requested_execution_engine: "streaming", + gateway_name: "greybeam", + gateway_routed: true, + }), + ]); + expect(fields.find((field) => field.label === "Execution engine")).toMatchObject({ + status: "diff", + detail: "DuckDB: default; DuckDB: streaming", + }); + expect(fields.find((field) => field.label === "Gateway")).toMatchObject({ + status: "diff", + detail: "DuckDB: Native; DuckDB: Routed via greybeam", + }); + }); + it("flags normalized cost metadata differences", () => { const fields = buildComparabilityFields([ makeDetail({ normalized_cost_usd: 0.42, cost_status: "normalized" }), diff --git a/results-explorer/src/db.ts b/results-explorer/src/db.ts index 36780f5503..bac9510aba 100644 --- a/results-explorer/src/db.ts +++ b/results-explorer/src/db.ts @@ -45,7 +45,7 @@ async function createCspBoundWorker(workerPath: string): Promise { return worker; } -const EXPECTED_READ_MODEL_VERSION = 15; +const EXPECTED_READ_MODEL_VERSION = 16; const SNAPSHOT_READY_SCANS = [ { diff --git a/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts b/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts index ecc678ac07..c67bd0aaf9 100644 --- a/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts +++ b/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts @@ -5,7 +5,7 @@ import { _EXPECTED_READ_MODEL_VERSION_FOR_TEST, _verifyReadModelVersionForTest } const EXPECTED_EXPLORER_BUILD_COMMAND = "uv run -- python _project/scripts/explorer_publish.py build"; -const CURRENT_READ_MODEL_VERSION = 15; +const CURRENT_READ_MODEL_VERSION = 16; const NEWER_READ_MODEL_POLICY = "warn-and-continue"; const repoRoot = resolve(process.cwd(), ".."); diff --git a/results-explorer/src/pages/Leaderboard.tsx b/results-explorer/src/pages/Leaderboard.tsx index 0ea6f56af1..e9737118da 100644 --- a/results-explorer/src/pages/Leaderboard.tsx +++ b/results-explorer/src/pages/Leaderboard.tsx @@ -15,7 +15,7 @@ import { humanizeBenchmark, errMsg } from "@/utils"; import { canonicalBenchmarkSlug, formatBenchmarkLabel } from "@/lib/displayLabels"; import { MetaLeaderboardSkeleton, SkeletonBlock } from "@/components/LoadingSpinner"; import { ErrorMessage } from "@/components/ErrorMessage"; -import { MetaLeaderboard } from "@/components/MetaLeaderboard"; +import { MetaLeaderboard, variantKeyOf, variantLabelOf } from "@/components/MetaLeaderboard"; import type { MetaLeaderboardMode } from "@/components/MetaLeaderboard"; import { ALL_FACET_KEYS, @@ -233,9 +233,9 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { for (const cohort of metaLeaderboard.cohorts) { const inner = new Map(); for (const row of cohort.platforms ?? []) { - const rows = inner.get(row.platform_id) ?? []; + const rows = inner.get(variantKeyOf(row)) ?? []; rows.push(row); - inner.set(row.platform_id, rows); + inner.set(variantKeyOf(row), rows); } index.set(cohort.key, inner); } @@ -264,13 +264,13 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { }); const visibleEvidencePlatformIds = new Set( - visibleCohorts.flatMap((cohort) => (cohort.platforms ?? []).map((row) => row.platform_id)), + visibleCohorts.flatMap((cohort) => (cohort.platforms ?? []).map((row) => variantKeyOf(row))), ); const platforms = metaLeaderboard.platforms .map((platform) => { const ranks = Object.fromEntries( visibleCohorts.flatMap((cohort) => { - const rows = cohortPlatformIndex.get(cohort.key)?.get(platform.platform_id) ?? []; + const rows = cohortPlatformIndex.get(cohort.key)?.get(variantKeyOf(platform)) ?? []; const rankedRows = rows .filter((row) => { const result = resultById.get(row.result_id); @@ -300,7 +300,7 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { n_cohorts: rankValues.length, } satisfies MetaPlatform; }) - .filter((platform) => platform.n_cohorts > 0 || visibleEvidencePlatformIds.has(platform.platform_id)) + .filter((platform) => platform.n_cohorts > 0 || visibleEvidencePlatformIds.has(variantKeyOf(platform))) .sort((a, b) => { if (a.avg_rank === null && b.avg_rank === null) return 0; if (a.avg_rank === null) return 1; @@ -353,12 +353,12 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { modeRaw === "times" || modeRaw === "ranks" ? modeRaw : "speedup"; const benchmarks = [...new Set(results.map((result) => canonicalBenchmarkSlug(result.benchmark)))].sort(); const platformIdToName = new Map( - (metaLeaderboard?.platforms ?? []).map((platform) => [platform.platform_id, platform.platform]), + (metaLeaderboard?.platforms ?? []).map((platform) => [variantKeyOf(platform), variantLabelOf(platform)]), ); for (const result of results) { - platformIdToName.set(result.platform_id, result.platform); + platformIdToName.set(variantKeyOf(result), variantLabelOf(result)); } - const platformIds = [...new Set(results.map((result) => result.platform_id))].sort(); + const platformIds = [...new Set(results.map((result) => variantKeyOf(result)))].sort(); const activeFacetSummaries = summarizeActiveFacets(facets, platformIdToName); const benchmarkOptions = metaLeaderboard ? [...new Set(metaLeaderboard.cohorts.map((cohort) => cohort.benchmark))].sort() diff --git a/results-explorer/src/types.ts b/results-explorer/src/types.ts index f9af5855aa..2f2a719efd 100644 --- a/results-explorer/src/types.ts +++ b/results-explorer/src/types.ts @@ -58,6 +58,11 @@ export interface DetailResult extends CostDeploymentFields { scale_factor: number; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; + requested_execution_engine?: string | null; + gateway_name?: string | null; + gateway_routed?: boolean | null; driver_version: string | null; run_date: string; total_duration_s: number; @@ -182,6 +187,8 @@ export interface MetaRank { export interface MetaPlatform { platform_id: string; + variant_id?: string | null; + variant_label?: string | null; platform: string; ranks: Record; avg_rank: number | null; @@ -190,6 +197,8 @@ export interface MetaPlatform { export interface MetaCohortPlatform { platform_id: string; + variant_id?: string | null; + variant_label?: string | null; platform: string; result_id: string; rank: number | null; diff --git a/scripts/publication/check_explorer_compat.py b/scripts/publication/check_explorer_compat.py index 6f03a1a4ac..ff505b849b 100755 --- a/scripts/publication/check_explorer_compat.py +++ b/scripts/publication/check_explorer_compat.py @@ -354,6 +354,40 @@ def normalize_type(type_str: str) -> str: "results": _V15_RESULTS_COLUMNS, } +_VARIANT_COLUMNS: dict[str, str] = { + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + "requested_execution_engine": "VARCHAR", + "gateway_name": "VARCHAR", + "gateway_routed": "BOOLEAN", +} + +TABLE_COLUMNS_V16: dict[str, dict[str, str]] = { + **TABLE_COLUMNS_V15, + "results": { + **TABLE_COLUMNS_V15["results"], + **_VARIANT_COLUMNS, + }, + "benchmark_rankings": { + **TABLE_COLUMNS_V15["benchmark_rankings"], + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + }, + "cohort_metadata": { + **TABLE_COLUMNS_V15["cohort_metadata"], + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + }, + "meta_leaderboard": { + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + "platform_id": "VARCHAR", + "platform": "VARCHAR", + "avg_rank": "DOUBLE", + "n_cohorts": "INTEGER", + }, +} + SCHEMA_REGISTRY: dict[int, dict[str, dict[str, str]]] = { 9: TABLE_COLUMNS_V9, @@ -363,6 +397,7 @@ def normalize_type(type_str: str) -> str: 13: TABLE_COLUMNS_V13, 14: TABLE_COLUMNS_V14, 15: TABLE_COLUMNS_V15, + 16: TABLE_COLUMNS_V16, } REQUIRED_INDEXES_V9: list[tuple[str, str, list[str]]] = [ @@ -419,6 +454,16 @@ def normalize_type(type_str: str) -> str: "template_evidence", ], } +REQUIRED_VIEW_COLUMNS_V16: dict[str, list[str]] = { + "result_detail_metrics": [ + *REQUIRED_VIEW_COLUMNS_V15["result_detail_metrics"], + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", + ], +} def get_table_columns_for_version(version: int) -> dict[str, dict[str, str]]: @@ -443,6 +488,12 @@ def get_indexes_for_version(version: int) -> list[tuple[str, str, list[str]]]: return list(REQUIRED_INDEXES_V10) +def _meta_leaderboard_pk(columns_map: dict[str, dict[str, str]]) -> str: + if "variant_id" in columns_map.get("meta_leaderboard", {}): + return "variant_id" + return "platform_id" + + def generate_schema_ddl(version: int) -> list[str]: columns_map = get_table_columns_for_version(version) ddl_statements: list[str] = [] @@ -492,9 +543,9 @@ def generate_schema_ddl(version: int) -> list[str]: ddl_statements.append("CREATE INDEX idx_cohort_metadata_key ON cohort_metadata (cohort_key);") ddl_statements.append("CREATE INDEX idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id);") + ml_cols = ", ".join(f"{c} {t}" for c, t in columns_map["meta_leaderboard"].items()) ddl_statements.append( - "CREATE TABLE meta_leaderboard (platform_id VARCHAR PRIMARY KEY, platform VARCHAR NOT NULL, " - "avg_rank DOUBLE, n_cohorts INTEGER NOT NULL);" + f"CREATE TABLE meta_leaderboard ({ml_cols}, PRIMARY KEY ({_meta_leaderboard_pk(columns_map)}));" ) ddl_statements.append("CREATE TABLE short_ids (short_id VARCHAR PRIMARY KEY, result_id VARCHAR NOT NULL UNIQUE);") @@ -557,7 +608,7 @@ def generate_schema_ddl(version: int) -> list[str]: ), ( "Meta leaderboard summary", - "SELECT platform_id, platform, avg_rank, n_cohorts FROM meta_leaderboard", + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM meta_leaderboard", ), ( "Short ID resolution", @@ -639,7 +690,9 @@ def validate_database_schema(con: Any, expected_version: int | None = None) -> l if missing_views: errors.append(f"missing required views for v{version_to_check}: {', '.join(missing_views)}") - if version_to_check >= 15: + if version_to_check >= 16: + view_column_requirements = REQUIRED_VIEW_COLUMNS_V16 + elif version_to_check >= 15: view_column_requirements = REQUIRED_VIEW_COLUMNS_V15 elif version_to_check >= 14: view_column_requirements = REQUIRED_VIEW_COLUMNS_V14 diff --git a/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py b/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py index d23b504b7b..593dd807a4 100644 --- a/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py +++ b/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py @@ -41,6 +41,11 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -149,6 +154,8 @@ class TestG1SchemaContract: "phase", "result_id", "platform_id", + "variant_id", + "variant_label", "platform", "short_id", "trust_label", @@ -199,6 +206,8 @@ class TestG1SchemaContract: "primary_metric", "primary_order", "platform_id", + "variant_id", + "variant_label", "platform", "result_id", "has_display_timing", @@ -214,7 +223,7 @@ class TestG1SchemaContract: "speedup_vs_best", "stream_count", }, - "meta_leaderboard": {"platform_id", "platform", "avg_rank", "n_cohorts"}, + "meta_leaderboard": {"variant_id", "variant_label", "platform_id", "platform", "avg_rank", "n_cohorts"}, "short_ids": {"short_id", "result_id"}, "metadata": {"read_model_version"}, } @@ -226,6 +235,11 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -300,6 +314,8 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", "driver_version", "run_date", "power_score", diff --git a/tests/unit/scripts/explorer_pipeline/test_variant_identity.py b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py new file mode 100644 index 0000000000..77098baa0c --- /dev/null +++ b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from _project.scripts.explorer_pipeline.models import ( + BenchmarkSummary, + ManifestEntry, + PlatformRow, + RankingConfig, + ranking_exclusion_reason, + variant_label, +) +from _project.scripts.explorer_pipeline.ranking import rank_platforms +from _project.scripts.explorer_pipeline.transformer import BundleTransformer + +pytestmark = [pytest.mark.unit, pytest.mark.fast] + +_EXPLORER_FIXTURES = pytest.importorskip( + "tests.unit.scripts.explorer_pipeline.conftest", reason="Explorer pipeline test fixtures are not in this checkout" +) +MINIMAL_BUNDLE = _EXPLORER_FIXTURES.MINIMAL_BUNDLE + + +def _entry(**overrides) -> ManifestEntry: + base = { + "result_id": "r1", + "benchmark": "tpch", + "scale_factor": 0.01, + "platform": "polars-df", + "platform_id": "polars-df", + "driver_version": "1.0", + "run_date": "2026-01-01", + "power_score": 1.0, + "total_duration_s": 1.0, + "query_count": 22, + "display_geomean_ms": 100.0, + "trust_label": "maintainer-run", + "visibility": "public-curated", + "validation_status": "passed", + } + base.update(overrides) + return ManifestEntry(**base) + + +def test_legacy_bundle_keeps_platform_identity() -> None: + entry = _entry() + assert entry.variant_id == "polars-df" + assert entry.variant_label == "polars-df" + assert entry.requested_execution_engine == "default" + assert entry.gateway_routed is False + assert ranking_exclusion_reason(entry) is None + + +def test_engine_request_defines_variant_identity() -> None: + entry = _entry( + variant_id="polars-df+streaming", + requested_execution_engine="streaming", + variant_label="", + ) + assert entry.variant_id == "polars-df+streaming" + assert entry.variant_label == "polars-df (streaming)" + assert ranking_exclusion_reason(entry) is None + + +def test_gateway_routed_bundle_is_excluded_from_native_ranking() -> None: + entry = _entry( + platform="snowflake", + platform_id="snowflake", + variant_id="snowflake@greybeam", + gateway_name="greybeam", + gateway_routed=True, + ) + assert entry.variant_label == "snowflake via Greybeam" + assert ranking_exclusion_reason(entry) == "gateway_routed" + + +def test_variant_label_cases() -> None: + assert variant_label("duckdb", variant_id="duckdb") == "duckdb" + assert ( + variant_label("polars-df", variant_id="polars-df+streaming", requested_execution_engine="streaming") + == "polars-df (streaming)" + ) + assert ( + variant_label("snowflake", variant_id="snowflake@greybeam", gateway_name="greybeam", gateway_routed=True) + == "snowflake via Greybeam" + ) + + +def _write_bundle(tmp_path: Path, name: str, platform_block: dict) -> Path: + payload = copy.deepcopy(MINIMAL_BUNDLE) + payload["platform"] = platform_block + path = tmp_path / name + path.write_text(json.dumps(payload)) + return path + + +def test_transformer_carries_recorded_variant(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "variant.json", + { + "name": "polars-df", + "variant": "polars-df+streaming", + "execution_engine": {"requested": "streaming", "resolution": "explicit"}, + }, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "polars-df+streaming" + assert entry.variant_label == "polars-df (streaming)" + assert entry.requested_execution_engine == "streaming" + assert entry.gateway_routed is False + detail = transformer.to_detail_result(bundle, "variant-result") + assert detail.variant_id == "polars-df+streaming" + assert detail.variant_label == "polars-df (streaming)" + + +def test_transformer_carries_gateway_routing(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "gateway.json", + { + "name": "snowflake", + "variant": "snowflake@greybeam", + "gateway": {"name": "greybeam", "routed": True}, + }, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.gateway_routed is True + assert entry.gateway_name == "greybeam" + assert entry.variant_label == "snowflake via Greybeam" + assert ranking_exclusion_reason(entry) == "gateway_routed" + + +def test_transformer_legacy_bundle_keeps_identity(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle(tmp_path, "legacy.json", {"name": "duckdb", "version": "1.2.0"}) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == entry.platform_id == "duckdb" + assert entry.variant_label == "duckdb" + + +def _row(variant_id: str, result_id: str, geomean_ms: float, *, eligible: bool = True) -> PlatformRow: + return PlatformRow( + result_id=result_id, + short_id=result_id, + platform_id="polars-df", + variant_id=variant_id, + variant_label=f"polars-df ({variant_id})", + platform="polars-df", + platform_version=None, + tuning_mode=None, + tuning_hash=None, + execution_mode=None, + trust_label="maintainer-run", + run_date="2026-05-05T00:00:00Z", + is_ranking_eligible=eligible, + ranking_exclusion_reason=None if eligible else "gateway_routed", + power_score=None, + display_geomean_ms=geomean_ms, + sample_geomean_ms=None, + cost_usd=None, + compliance_class=None, + percentile_stats=None, + phase_durations=None, + timings={}, + ) + + +def test_engine_variants_rank_as_separate_rows() -> None: + summary = BenchmarkSummary( + benchmark="tpch", + scale_factor=0.01, + phase="power", + query_ids=[], + platforms=[ + _row("polars-df", "result-default", 100.0), + _row("polars-df+streaming", "result-streaming", 200.0), + ], + ranking=RankingConfig( + primary_metric="display_geomean_ms", + secondary_metric="display_geomean_ms", + primary_order="asc", + ), + ) + ranked = rank_platforms(summary) + by_result = {row.row.result_id: row for row in ranked.rows} + assert ranked.total_ranked == 2 + assert by_result["result-default"].rank == 1 + assert by_result["result-streaming"].rank == 2 + assert by_result["result-streaming"].row.variant_id == "polars-df+streaming" + + +def test_gateway_row_is_excluded_with_reason() -> None: + summary = BenchmarkSummary( + benchmark="tpch", + scale_factor=0.01, + phase="power", + query_ids=[], + platforms=[ + _row("snowflake", "result-native", 100.0), + _row("snowflake@greybeam", "result-gateway", 50.0, eligible=False), + ], + ranking=RankingConfig( + primary_metric="display_geomean_ms", + secondary_metric="display_geomean_ms", + primary_order="asc", + ), + ) + ranked = rank_platforms(summary) + by_result = {row.row.result_id: row for row in ranked.rows} + assert ranked.total_ranked == 1 + assert by_result["result-native"].rank == 1 + assert by_result["result-gateway"].rank is None + assert by_result["result-gateway"].ranking_exclusion_reason == "gateway_routed" diff --git a/tests/unit/scripts/publication/test_check_explorer_compat.py b/tests/unit/scripts/publication/test_check_explorer_compat.py index 45dbed2c57..7aa43c3381 100644 --- a/tests/unit/scripts/publication/test_check_explorer_compat.py +++ b/tests/unit/scripts/publication/test_check_explorer_compat.py @@ -46,14 +46,14 @@ def test_normalize_type() -> None: def test_schema_versions_definitions() -> None: - assert checker.SUPPORTED_SCHEMA_VERSIONS == (15,) - assert checker.CURRENT_SCHEMA_VERSION == 15 + assert checker.SUPPORTED_SCHEMA_VERSIONS == (16,) + assert checker.CURRENT_SCHEMA_VERSION == 16 from _project.scripts.explorer_pipeline.contract import EXPLORER_READ_MODEL_VERSION assert checker.CURRENT_SCHEMA_VERSION == EXPLORER_READ_MODEL_VERSION assert checker.SUPPORTED_SCHEMA_VERSIONS == (EXPLORER_READ_MODEL_VERSION,) - cols = checker.get_table_columns_for_version(15) + cols = checker.get_table_columns_for_version(16) assert "results" in cols assert "metadata" in cols assert "result_environment" in cols @@ -110,50 +110,50 @@ def test_check_schema_compatibility_all_supported() -> None: def test_validate_database_schema_detects_missing_table() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("DROP TABLE cohort_metadata") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("missing required tables for v15: cohort_metadata" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("missing required tables for v16: cohort_metadata" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_column() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("ALTER TABLE results DROP COLUMN funding") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any("table 'results' missing required columns: funding" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_template_evidence() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("ALTER TABLE results DROP COLUMN template_evidence") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any("table 'results' missing required columns: template_evidence" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_view() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("DROP VIEW result_detail_metrics") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("missing required views for v15: result_detail_metrics" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("missing required views for v16: result_detail_metrics" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_view_column() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("CREATE OR REPLACE VIEW result_detail_metrics AS SELECT result_id FROM results") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any( "view 'result_detail_metrics' missing required columns:" in err and "override_rules" in err for err in errors @@ -177,11 +177,11 @@ def test_validate_database_schema_detects_missing_view_column() -> None: def test_validate_database_schema_version_mismatch() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("UPDATE metadata SET read_model_version = 8") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("read_model_version mismatch: expected 15, got 8" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("read_model_version mismatch: expected 16, got 8" in err for err in errors) finally: con.close() @@ -345,7 +345,7 @@ def test_cli_default_schema_checks(capsys: pytest.CaptureFixture[str]) -> None: assert exit_code == 0 captured = capsys.readouterr() assert "Results Explorer Compatibility" in captured.out - assert "Schema v15" in captured.out + assert "Schema v16" in captured.out assert "Schema v12" not in captured.out assert "Schema v11" not in captured.out assert "Schema v10" not in captured.out @@ -359,18 +359,18 @@ def test_cli_json_mode(capsys: pytest.CaptureFixture[str]) -> None: captured = capsys.readouterr() data = json.loads(captured.out) assert data["status"] == "passed" - assert data["current_version"] == 15 - assert "v15" in data["schema_checks"] + assert data["current_version"] == 16 + assert "v16" in data["schema_checks"] assert "v12" not in data["schema_checks"] assert "v11" not in data["schema_checks"] assert "v10" not in data["schema_checks"] def test_cli_specific_schema_version(capsys: pytest.CaptureFixture[str]) -> None: - exit_code = checker.main(["--schema-only", "--schema-versions", "15"]) + exit_code = checker.main(["--schema-only", "--schema-versions", "16"]) assert exit_code == 0 captured = capsys.readouterr() - assert "Schema v15" in captured.out + assert "Schema v16" in captured.out assert "Schema v12" not in captured.out assert "Schema v11" not in captured.out assert "Schema v10" not in captured.out @@ -438,9 +438,9 @@ def test_cli_db_path_check(tmp_path: Path, capsys: pytest.CaptureFixture[str]) - db_file = tmp_path / "test.duckdb" with duckdb.connect(str(db_file)) as con: - for stmt in checker.generate_schema_ddl(15): + for stmt in checker.generate_schema_ddl(16): con.execute(stmt) - con.execute("INSERT INTO metadata VALUES (15)") + con.execute("INSERT INTO metadata VALUES (16)") exit_code = checker.main(["--db-path", str(db_file)]) assert exit_code == 0 diff --git a/tests/unit/scripts/test_corpus_cohort_depth.py b/tests/unit/scripts/test_corpus_cohort_depth.py index 2f721ef739..260260c148 100644 --- a/tests/unit/scripts/test_corpus_cohort_depth.py +++ b/tests/unit/scripts/test_corpus_cohort_depth.py @@ -67,6 +67,7 @@ def _write_bundle( platform: str, platform_version: str | None = None, execution_version: str | None = None, + platform_variant: str | None = None, run_timestamp: str = "2026-08-01T12:00:00", test_type: str | None = "power", ) -> None: @@ -74,6 +75,8 @@ def _write_bundle( platform_payload = {"name": platform} if platform_version is not None: platform_payload["version"] = platform_version + if platform_variant is not None: + platform_payload["variant"] = platform_variant payload = { "benchmark": {"id": benchmark, "scale_factor": scale}, "platform": platform_payload, @@ -111,6 +114,56 @@ def test_the_gate_detects_a_one_platform_cohort(tmp_path: Path) -> None: assert shallow == {("tpcds", "10.0"): {"DuckDB"}} +def test_engine_variants_count_as_distinct_cohort_identities(tmp_path: Path) -> None: + validator = _load_validator() + _write_bundle( + tmp_path, + "default.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+default", + ) + _write_bundle( + tmp_path, + "streaming.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + _write_bundle(tmp_path, "duckdb.json", benchmark="tpch", scale=1.0, platform="DuckDB") + + cohorts = validator.cohort_platforms(validator.discover_bundles(tmp_path)) + + assert cohorts == {("tpch", "1.0"): {"polars+default", "polars+streaming", "DuckDB"}} + assert validator.shallow_cohorts(cohorts) == {} + + +def test_repeated_variant_bundles_do_not_pad_cohort_depth(tmp_path: Path) -> None: + validator = _load_validator() + _write_bundle( + tmp_path, + "a.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + _write_bundle( + tmp_path, + "b.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + + assert validator.shallow_cohorts(validator.cohort_platforms(validator.discover_bundles(tmp_path))) == { + ("tpch", "1.0"): {"polars+streaming"} + } + + def _write_phase_bundle( directory: Path, name: str, From f0dfa4e772fc4c516f1d68ee61e5b4d63cb7172b Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Mon, 12 Oct 2026 09:28:10 -0400 Subject: [PATCH 2/4] fix(explorer): align read-model version pins with v16 contract --- results-explorer/scripts/explorer-build-contract.mjs | 2 +- tests/unit/scripts/test_explorer_build_contract.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/results-explorer/scripts/explorer-build-contract.mjs b/results-explorer/scripts/explorer-build-contract.mjs index 4265fcacb4..0b093c6da9 100644 --- a/results-explorer/scripts/explorer-build-contract.mjs +++ b/results-explorer/scripts/explorer-build-contract.mjs @@ -3,7 +3,7 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; const EXPECTED_CONTRACT_VERSION = "6"; -export const EXPECTED_READ_MODEL_VERSION = 15; +export const EXPECTED_READ_MODEL_VERSION = 16; const EXPECTED_NEWER_READ_MODEL_POLICY = "warn-and-continue"; const REQUIRED_FLAGS = ["--data-dir", "--output", "--trust-label", "--visibility"]; const REQUIRED_OUTPUTS = ["results.duckdb", "bundles/{result_id}.json"]; diff --git a/tests/unit/scripts/test_explorer_build_contract.py b/tests/unit/scripts/test_explorer_build_contract.py index 95a275ca29..201e55c4c1 100644 --- a/tests/unit/scripts/test_explorer_build_contract.py +++ b/tests/unit/scripts/test_explorer_build_contract.py @@ -28,7 +28,7 @@ def test_explorer_build_contract_command_emits_expected_json() -> None: def test_explorer_build_contract_matches_duckdb_only_output_contract() -> None: outputs = EXPLORER_BUILD_CONTRACT["outputs"] - current_read_model_version = 15 + current_read_model_version = 16 assert EXPLORER_BUILD_CONTRACT["version"] == "6" assert EXPLORER_BUILD_CONTRACT["read_model_version"] == current_read_model_version From 249c4b8c3dc739b4462510d879c189bfdf37d344 Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Mon, 12 Oct 2026 10:17:16 -0400 Subject: [PATCH 3/4] fix(explorer): address review findings on variant identity divergence --- .../explorer_pipeline/duckdb_builder.py | 4 +-- results-data/validate_corpus.py | 24 ++++++++++++-- .../src/components/MetaLeaderboard.tsx | 10 +++--- results-explorer/src/lib/duckdbQueries.ts | 33 ++++++++++++++++--- results-explorer/src/pages/Leaderboard.tsx | 2 +- results-explorer/src/pages/PlatformIndex.tsx | 4 +++ .../src/pages/__tests__/Leaderboard.test.tsx | 28 ++++++++-------- .../src/pages/__tests__/Overview.test.tsx | 10 +++--- .../unit/scripts/test_corpus_cohort_depth.py | 31 +++++++++++++++++ 9 files changed, 112 insertions(+), 34 deletions(-) diff --git a/_project/scripts/explorer_pipeline/duckdb_builder.py b/_project/scripts/explorer_pipeline/duckdb_builder.py index 7cd233b7da..612ca203b8 100644 --- a/_project/scripts/explorer_pipeline/duckdb_builder.py +++ b/_project/scripts/explorer_pipeline/duckdb_builder.py @@ -997,8 +997,8 @@ def _populate_benchmark_rankings( phase, platform_row.result_id, platform_row.platform_id, - platform_row.variant_id, - platform_row.variant_label, + platform_row.variant_id or platform_row.platform_id, + platform_row.variant_label or platform_row.platform, platform_row.platform, full_to_short.get(platform_row.result_id, ""), platform_row.trust_label, diff --git a/results-data/validate_corpus.py b/results-data/validate_corpus.py index 8ed295d10b..7e9a5b52cd 100644 --- a/results-data/validate_corpus.py +++ b/results-data/validate_corpus.py @@ -376,6 +376,11 @@ def exclusion_reason(payload: dict) -> str | None: return "validation_not_clean" if not _tuning_applied(payload): return "tuning_not_applied" + platform_section = payload.get("platform") + if isinstance(platform_section, dict): + gateway = platform_section.get("gateway") + if isinstance(gateway, dict) and gateway.get("routed"): + return "gateway_routed" display = _display_timings(payload) timing = _timing_exclusion(display, _logical_query_count(payload, display)) if timing is not None: @@ -404,6 +409,21 @@ def _cohort_key(payload: dict) -> CohortKey: return (benchmark_id, f"{scale_factor}{cohort_phase_suffix(payload)}") +def _variant_aware_identity(platform_section: dict, platform_name: str) -> str: + variant = platform_section.get("variant") + if variant and str(variant) != platform_name: + return str(variant) + engine = platform_section.get("execution_engine") + requested = engine.get("requested") if isinstance(engine, dict) else None + if requested and str(requested) != "default": + pid = _VARIANT_PLATFORM_SUFFIX_RE.sub("", platform_name).strip().lower().replace(" ", "-") + return f"{pid}+{requested}" + return platform_name + + +_VARIANT_PLATFORM_SUFFIX_RE = re.compile(r"[-_]trust[-_](?:ci|community|local|unknown)", re.IGNORECASE) + + def _comparison_identity(bundle: pathlib.Path, payload: dict) -> str: try: platform_section = payload["platform"] @@ -426,9 +446,7 @@ def _comparison_identity(bundle: pathlib.Path, payload: dict) -> str: except Exception as exc: raise CorpusReadError(f"ERROR reading platform identity from {bundle}: {exc}") from exc identity = str(platform) - variant = platform_section.get("variant") - if variant and str(variant) != str(platform): - identity = str(variant) + identity = _variant_aware_identity(platform_section, identity) if version and str(version) != "unknown": identity = f"{identity} v{str(version)[:120]}" return identity diff --git a/results-explorer/src/components/MetaLeaderboard.tsx b/results-explorer/src/components/MetaLeaderboard.tsx index bb552fb45b..17e5e31d7f 100644 --- a/results-explorer/src/components/MetaLeaderboard.tsx +++ b/results-explorer/src/components/MetaLeaderboard.tsx @@ -194,7 +194,7 @@ export function MetaLeaderboard({ case " ": { event.preventDefault(); const platform = visiblePlatforms[rowIdx]; - if (platform) route(platformHref(platform.platform_id)); + if (platform) route(platformHref(variantKeyOf(platform))); return; } default: @@ -361,12 +361,12 @@ export function MetaLeaderboard({ route(platformHref(platform.platform_id))} + onClick={() => route(platformHref(variantKeyOf(platform)))} >
event.stopPropagation()} > @@ -596,11 +596,11 @@ function exactMetricTitle(rank: MetaRank | undefined, cohort: MetaCohort): strin } export function variantKeyOf(row: { variant_id?: string | null; platform_id: string }): string { - return row.variant_id ?? row.platform_id; + return row.variant_id || row.platform_id; } export function variantLabelOf(row: { variant_label?: string | null; platform: string }): string { - return row.variant_label ?? row.platform; + return row.variant_label || row.platform; } function findCohortPlatform(cohort: MetaCohort, platformId: string): MetaCohortPlatform | undefined { diff --git a/results-explorer/src/lib/duckdbQueries.ts b/results-explorer/src/lib/duckdbQueries.ts index 4ebe5e6acb..86c0b7afcc 100644 --- a/results-explorer/src/lib/duckdbQueries.ts +++ b/results-explorer/src/lib/duckdbQueries.ts @@ -106,6 +106,11 @@ export interface ResultRow extends CostDeploymentFields { scale_factor: number; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; + requested_execution_engine?: string | null; + gateway_name?: string | null; + gateway_routed?: boolean | null; driver_version: string | null; run_date: string; power_score: number | null; @@ -272,6 +277,8 @@ export interface PlatformIndexRowRow extends CostDeploymentFields { phase: string; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; driver_version: string | null; platform_version?: string | null; run_date: string; @@ -338,6 +345,8 @@ export interface CohortMetadataRow { } export interface MetaLeaderboardRow { + variant_id?: string | null; + variant_label?: string | null; platform_id: string; platform: string; avg_rank: number | null; @@ -366,6 +375,11 @@ const RESULT_COLUMNS = [ "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -434,6 +448,11 @@ const RESULT_DETAIL_METRICS_COLUMNS = [ "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -524,6 +543,8 @@ const COHORT_METADATA_COLUMNS = [ "primary_metric", "primary_order", "platform_id", + "variant_id", + "variant_label", "platform", "result_id", "short_id", @@ -548,6 +569,8 @@ const BENCHMARK_RANKING_COLUMNS = [ "br.phase", "br.result_id", "br.platform_id", + "br.variant_id", + "br.variant_label", "br.platform", "br.short_id", "br.trust_label", @@ -1133,6 +1156,8 @@ function loadPlatformIndexRows(platformId?: string): Promise { "meta-leaderboard", () => queryRows( - "SELECT platform_id, platform, avg_rank, n_cohorts" + + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts" + " FROM bench.meta_leaderboard" + - " ORDER BY avg_rank NULLS LAST, platform_id", + " ORDER BY avg_rank NULLS LAST, variant_id", ), ); } @@ -1220,9 +1245,9 @@ export async function getMetaLeaderboardData(): Promise async function loadMetaLeaderboardData(): Promise { const [platformRows, cohortRows] = await Promise.all([ queryRows( - "SELECT platform_id, platform, avg_rank, n_cohorts" + + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts" + " FROM bench.meta_leaderboard" + - " ORDER BY avg_rank NULLS LAST, platform_id", + " ORDER BY avg_rank NULLS LAST, variant_id", ), queryRows( `SELECT ${COHORT_METADATA_COLUMNS} FROM bench.cohort_metadata` + diff --git a/results-explorer/src/pages/Leaderboard.tsx b/results-explorer/src/pages/Leaderboard.tsx index e9737118da..ab86342a67 100644 --- a/results-explorer/src/pages/Leaderboard.tsx +++ b/results-explorer/src/pages/Leaderboard.tsx @@ -390,7 +390,7 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { const publicBenchmarksOutsideLeaderboard = benchmarks.filter((benchmark) => !leaderboardBenchmarkSet.has(canonicalBenchmarkSlug(benchmark))); const leaderboardEvidencePlatformIds = new Set( (metaLeaderboard?.cohorts ?? []).flatMap((cohort) => - (cohort.platforms ?? []).map((platform) => platform.platform_id), + (cohort.platforms ?? []).map((platform) => variantKeyOf(platform)), ), ); const publicPlatformIdsOutsideLeaderboard = platformIds.filter((platformId) => !leaderboardEvidencePlatformIds.has(platformId)); diff --git a/results-explorer/src/pages/PlatformIndex.tsx b/results-explorer/src/pages/PlatformIndex.tsx index f2d3d69098..d628be9596 100644 --- a/results-explorer/src/pages/PlatformIndex.tsx +++ b/results-explorer/src/pages/PlatformIndex.tsx @@ -151,6 +151,10 @@ function canonicalPlatformRouteKey(value: string): string { function platformRowsForRequest(rows: PlatformIndexRowRow[], platform: string): PlatformIndexRowRow[] { const requested = canonicalPlatformRouteKey(platform); + const variantMatches = rows.filter( + (row) => normalizePlatformKey(row.variant_id || row.platform_id) === requested, + ); + if (variantMatches.length > 0) return variantMatches; const idMatches = rows.filter((row) => normalizePlatformKey(row.platform_id) === requested); if (idMatches.length > 0) return idMatches; return rows.filter((row) => normalizePlatformKey(row.platform) === requested); diff --git a/results-explorer/src/pages/__tests__/Leaderboard.test.tsx b/results-explorer/src/pages/__tests__/Leaderboard.test.tsx index 93849d946b..b8de7936ca 100644 --- a/results-explorer/src/pages/__tests__/Leaderboard.test.tsx +++ b/results-explorer/src/pages/__tests__/Leaderboard.test.tsx @@ -37,7 +37,7 @@ beforeEach(() => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -51,7 +51,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows.promise; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) return []; + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) return []; if (s.includes("FROM bench.cohort_metadata")) return []; return []; }); @@ -90,7 +90,7 @@ describe("Leaderboard", () => { resultCalls += 1; return resultCalls === 1 ? [] : retryRows.promise; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return metaRows.promise; } if (s.includes("FROM bench.cohort_metadata")) { @@ -135,7 +135,7 @@ describe("Leaderboard", () => { resultCalls += 1; return []; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) { @@ -165,7 +165,7 @@ describe("Leaderboard", () => { if (shouldFail) throw new Error("Network error loading results"); return RESULT_ROWS; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { metaCalls += 1; if (shouldFail) throw new Error("Network error loading metadata"); return META_LEADERBOARD_ROWS; @@ -229,7 +229,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return unlabelledRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -297,7 +297,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows; @@ -341,7 +341,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows.promise; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return metaRows.promise; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows.promise; @@ -635,7 +635,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return results; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohorts; @@ -694,7 +694,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return results; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohorts; @@ -719,7 +719,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows; @@ -771,7 +771,7 @@ describe("toggleFacetValue (w13)", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return versionedRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -798,7 +798,7 @@ describe("toggleFacetValue (w13)", () => { ? versionedRows.filter((row) => row.platform_version === "1.4.0") : versionedRows; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -828,7 +828,7 @@ describe("toggleFacetValue (w13)", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("JOIN bench.result_detail_metrics")) return hardwareRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; diff --git a/results-explorer/src/pages/__tests__/Overview.test.tsx b/results-explorer/src/pages/__tests__/Overview.test.tsx index bad4272c87..4b7a3f0274 100644 --- a/results-explorer/src/pages/__tests__/Overview.test.tsx +++ b/results-explorer/src/pages/__tests__/Overview.test.tsx @@ -18,7 +18,7 @@ beforeEach(() => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -65,7 +65,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return singleResultRow; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return singleMetaLeaderboard; } if (s.includes("FROM bench.cohort_metadata")) return singleCohort; @@ -88,7 +88,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return []; @@ -119,7 +119,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return withoutCost; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -140,7 +140,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return withoutPower; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; diff --git a/tests/unit/scripts/test_corpus_cohort_depth.py b/tests/unit/scripts/test_corpus_cohort_depth.py index 260260c148..b205b869de 100644 --- a/tests/unit/scripts/test_corpus_cohort_depth.py +++ b/tests/unit/scripts/test_corpus_cohort_depth.py @@ -164,6 +164,37 @@ def test_repeated_variant_bundles_do_not_pad_cohort_depth(tmp_path: Path) -> Non } +def test_gateway_routed_bundles_contribute_no_identity(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": { + "name": "Snowflake", + "variant": "snowflake@greybeam", + "gateway": {"name": "greybeam", "routed": True}, + }, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "gateway.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.exclusion_reason(payload) == "gateway_routed" + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): set()} + + +def test_requested_engine_without_variant_string_matches_pipeline(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "constructed.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"polars+streaming"}} + + def _write_phase_bundle( directory: Path, name: str, From c33aeae0b09593279b946fb25ef6692c8d754046 Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Mon, 12 Oct 2026 11:00:19 -0400 Subject: [PATCH 4/4] fix(explorer): align validator variant precedence with pipeline transformer --- results-data/validate_corpus.py | 4 +-- .../test_variant_identity.py | 22 ++++++++++++++++ .../unit/scripts/test_corpus_cohort_depth.py | 26 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/results-data/validate_corpus.py b/results-data/validate_corpus.py index 7e9a5b52cd..9a5f4127dc 100644 --- a/results-data/validate_corpus.py +++ b/results-data/validate_corpus.py @@ -411,8 +411,8 @@ def _cohort_key(payload: dict) -> CohortKey: def _variant_aware_identity(platform_section: dict, platform_name: str) -> str: variant = platform_section.get("variant") - if variant and str(variant) != platform_name: - return str(variant) + if isinstance(variant, str) and variant.strip(): + return variant.strip() engine = platform_section.get("execution_engine") requested = engine.get("requested") if isinstance(engine, dict) else None if requested and str(requested) != "default": diff --git a/tests/unit/scripts/explorer_pipeline/test_variant_identity.py b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py index 77098baa0c..cb66446243 100644 --- a/tests/unit/scripts/explorer_pipeline/test_variant_identity.py +++ b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py @@ -145,6 +145,28 @@ def test_transformer_legacy_bundle_keeps_identity(tmp_path: Path) -> None: assert entry.variant_label == "duckdb" +def test_transformer_blank_variant_falls_through_to_requested(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "blank.json", + {"name": "Polars", "variant": " ", "execution_engine": {"requested": "streaming"}}, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "polars+streaming" + + +def test_transformer_variant_equal_to_platform_name_is_kept(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "same.json", + {"name": "Polars", "variant": "Polars", "execution_engine": {"requested": "streaming"}}, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "Polars" + + def _row(variant_id: str, result_id: str, geomean_ms: float, *, eligible: bool = True) -> PlatformRow: return PlatformRow( result_id=result_id, diff --git a/tests/unit/scripts/test_corpus_cohort_depth.py b/tests/unit/scripts/test_corpus_cohort_depth.py index b205b869de..4d1322a04c 100644 --- a/tests/unit/scripts/test_corpus_cohort_depth.py +++ b/tests/unit/scripts/test_corpus_cohort_depth.py @@ -195,6 +195,32 @@ def test_requested_engine_without_variant_string_matches_pipeline(tmp_path: Path assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"polars+streaming"}} +def test_blank_variant_falls_through_to_requested_engine(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "variant": " ", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "blank.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"polars+streaming"}} + + +def test_variant_equal_to_platform_name_matches_pipeline(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "variant": "Polars", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "same.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"Polars"}} + + def _write_phase_bundle( directory: Path, name: str,