diff --git a/CHANGELOG.md b/CHANGELOG.md index da27de51ff..e86d451289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Documentation +- The Explorer read model is at version 16 with execution-variant identity: + the pipeline records `variant_id`, `variant_label`, the requested execution + engine and gateway routing on every result, cohort and leaderboard row. + Two bundles that differ only in requested engine rank as separate variants + ("Polars (streaming)" beside "Polars"), and gateway-routed runs are + excluded from native ranking with `ranking_exclusion_reason=gateway_routed`. + Existing corpus values are unchanged: bundles without variant data keep + `variant_id` equal to the platform id. + - The Polars version-matrix reproduction instructions now use `--execution-engine` for default, in-memory and streaming setups, and identify the earlier `engine` option name used by the recorded bundles. diff --git a/_project/scripts/explorer_pipeline/contract.py b/_project/scripts/explorer_pipeline/contract.py index 12c983703c..435bb49cc4 100644 --- a/_project/scripts/explorer_pipeline/contract.py +++ b/_project/scripts/explorer_pipeline/contract.py @@ -1,7 +1,7 @@ from __future__ import annotations EXPLORER_BUILD_CONTRACT_VERSION = "6" -EXPLORER_READ_MODEL_VERSION = 15 +EXPLORER_READ_MODEL_VERSION = 16 EXPLORER_READ_MODEL_COMPATIBILITY = { "minimum_supported": EXPLORER_READ_MODEL_VERSION, "newer_policy": "warn-and-continue", diff --git a/_project/scripts/explorer_pipeline/duckdb_builder.py b/_project/scripts/explorer_pipeline/duckdb_builder.py index 00a20a910d..612ca203b8 100644 --- a/_project/scripts/explorer_pipeline/duckdb_builder.py +++ b/_project/scripts/explorer_pipeline/duckdb_builder.py @@ -330,6 +330,11 @@ def _create_schema(self, con: Any) -> None: scale_factor DOUBLE NOT NULL, platform VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, + requested_execution_engine VARCHAR NOT NULL, + gateway_name VARCHAR, + gateway_routed BOOLEAN NOT NULL, driver_version VARCHAR, run_date VARCHAR NOT NULL, power_score DOUBLE, @@ -403,6 +408,11 @@ def _create_schema(self, con: Any) -> None: r.scale_factor, r.platform, r.platform_id, + r.variant_id, + r.variant_label, + r.requested_execution_engine, + r.gateway_name, + r.gateway_routed, r.driver_version, r.run_date, r.power_score, @@ -527,6 +537,8 @@ def _create_schema(self, con: Any) -> None: phase VARCHAR NOT NULL, result_id VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, short_id VARCHAR NOT NULL, trust_label VARCHAR NOT NULL, @@ -575,6 +587,11 @@ def _create_schema(self, con: Any) -> None: scale_factor, platform, platform_id, + variant_id, + variant_label, + requested_execution_engine, + gateway_name, + gateway_routed, driver_version, run_date, power_score, @@ -629,6 +646,8 @@ def _create_schema(self, con: Any) -> None: primary_metric VARCHAR NOT NULL, primary_order VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, result_id VARCHAR NOT NULL, short_id VARCHAR NOT NULL, @@ -653,7 +672,9 @@ def _create_schema(self, con: Any) -> None: con.execute("CREATE INDEX idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id)") con.execute(""" CREATE TABLE meta_leaderboard ( - platform_id VARCHAR PRIMARY KEY, + variant_id VARCHAR PRIMARY KEY, + variant_label VARCHAR NOT NULL, + platform_id VARCHAR NOT NULL, platform VARCHAR NOT NULL, avg_rank DOUBLE, n_cohorts INTEGER NOT NULL @@ -774,6 +795,11 @@ def _populate_results( entry.scale_factor, entry.platform, entry.platform_id, + entry.variant_id, + entry.variant_label, + entry.requested_execution_engine, + entry.gateway_name, + entry.gateway_routed, entry.driver_version, entry.run_date, entry.power_score, @@ -971,6 +997,8 @@ def _populate_benchmark_rankings( phase, platform_row.result_id, platform_row.platform_id, + platform_row.variant_id or platform_row.platform_id, + platform_row.variant_label or platform_row.platform, platform_row.platform, full_to_short.get(platform_row.result_id, ""), platform_row.trust_label, @@ -1094,6 +1122,8 @@ def _populate_cohort_metadata( primary_metric, primary_order, p["platform_id"], + p.get("variant_id") or p["platform_id"], + p.get("variant_label") or p["platform"], p["platform"], p["result_id"], p["short_id"], @@ -1123,6 +1153,8 @@ def _populate_meta_leaderboard(self, con: Any, meta: dict[str, Any]) -> None: for p in meta.get("platforms", []): rows.append( ( + p.get("variant_id") or p["platform_id"], + p.get("variant_label") or p["platform"], p["platform_id"], p["platform"], p.get("avg_rank"), @@ -1130,7 +1162,7 @@ def _populate_meta_leaderboard(self, con: Any, meta: dict[str, Any]) -> None: ) ) if rows: - con.executemany("INSERT INTO meta_leaderboard VALUES (?, ?, ?, ?)", rows) + con.executemany("INSERT INTO meta_leaderboard VALUES (?, ?, ?, ?, ?, ?)", rows) def _populate_short_ids(self, con: Any, short_id_map: dict[str, str]) -> None: rows = list(short_id_map.items()) diff --git a/_project/scripts/explorer_pipeline/models.py b/_project/scripts/explorer_pipeline/models.py index 87638fda63..1cb43470e5 100644 --- a/_project/scripts/explorer_pipeline/models.py +++ b/_project/scripts/explorer_pipeline/models.py @@ -62,6 +62,23 @@ def _platform_id(raw: str) -> str: return stripped.lower().replace(" ", "-") +def variant_label( + platform: str, + *, + variant_id: str, + requested_execution_engine: str = "default", + gateway_name: str | None = None, + gateway_routed: bool = False, +) -> str: + if gateway_routed and gateway_name: + return f"{platform} via {str(gateway_name).capitalize()}" + if requested_execution_engine and requested_execution_engine != "default": + return f"{platform} ({requested_execution_engine})" + if variant_id and variant_id != _platform_id(platform): + return f"{platform} ({variant_id})" + return platform + + RANKING_ELIGIBLE_VISIBILITIES: frozenset[str] = frozenset( { "public-curated", @@ -95,6 +112,11 @@ class ManifestEntry(BaseModel): scale_factor: float platform: str platform_id: str = "" + variant_id: str = "" + variant_label: str = "" + requested_execution_engine: str = "default" + gateway_name: str | None = None + gateway_routed: bool = False driver_version: str | None run_date: str power_score: float | None @@ -147,6 +169,22 @@ class ManifestEntry(BaseModel): def _serialize_normalized_cost(self, cost: NormalizedCost) -> dict[str, Any]: return cost.to_dict() + @model_validator(mode="after") + def _default_variant_identity(self) -> ManifestEntry: + if not self.variant_id: + self.variant_id = self.platform_id + if not self.variant_label: + self.variant_label = variant_label( + self.platform, + variant_id=self.variant_id, + requested_execution_engine=self.requested_execution_engine, + gateway_name=self.gateway_name, + gateway_routed=self.gateway_routed, + ) + if not self.requested_execution_engine: + self.requested_execution_engine = "default" + return self + @model_validator(mode="after") def _default_logical_query_count(self) -> ManifestEntry: if self.logical_query_count == 0 and self.query_count > 0: @@ -315,6 +353,11 @@ class DetailResult(BaseModel): scale_factor: float platform: str platform_id: str = "" + variant_id: str = "" + variant_label: str = "" + requested_execution_engine: str = "default" + gateway_name: str | None = None + gateway_routed: bool = False driver_version: str | None run_date: str total_duration_s: float @@ -365,6 +408,22 @@ class DetailResult(BaseModel): physical_rendering_id: str | None = None basis_availability: BasisAvailability | None = None + @model_validator(mode="after") + def _default_variant_identity(self) -> DetailResult: + if not self.variant_id: + self.variant_id = self.platform_id + if not self.variant_label: + self.variant_label = variant_label( + self.platform, + variant_id=self.variant_id, + requested_execution_engine=self.requested_execution_engine, + gateway_name=self.gateway_name, + gateway_routed=self.gateway_routed, + ) + if not self.requested_execution_engine: + self.requested_execution_engine = "default" + return self + @field_serializer("normalized_cost") def _serialize_normalized_cost(self, cost: NormalizedCost) -> dict[str, Any]: return cost.to_dict() @@ -408,6 +467,8 @@ def ranking_exclusion_reason(entry: ManifestEntry, primary_metric: str | None = return "tuning_not_applied" if entry.comparison_exclusion_reason is not None: return entry.comparison_exclusion_reason + if entry.gateway_routed: + return "gateway_routed" metric = primary_metric or get_ranking_config(entry.benchmark, canonical_phase(entry.test_type)).primary_metric value = primary_metric_value(entry, metric) @@ -529,6 +590,8 @@ class PlatformRow(BaseModel): result_id: str short_id: str = "" platform_id: str + variant_id: str = "" + variant_label: str = "" platform: str platform_version: str | None tuning_mode: str | None @@ -907,4 +970,5 @@ def _coerce_cost_block(cls, value: Any) -> Any: "timing_eligibility", "timing_exclusion_reason", "unavailable_normalized_cost_payload", + "variant_label", ] diff --git a/_project/scripts/explorer_pipeline/pipeline.py b/_project/scripts/explorer_pipeline/pipeline.py index b85af9f292..950b56a6cb 100644 --- a/_project/scripts/explorer_pipeline/pipeline.py +++ b/_project/scripts/explorer_pipeline/pipeline.py @@ -313,6 +313,8 @@ def _build_benchmark_summaries( result_id=entry.result_id, short_id=full_to_short.get(entry.result_id, ""), platform_id=entry.platform_id, + variant_id=entry.variant_id, + variant_label=entry.variant_label, platform=entry.platform, platform_version=entry.platform_version, tuning_mode=entry.tuning_mode, @@ -412,6 +414,8 @@ def _rank_platforms_in_cohort( entries.append( { "platform_id": row.platform_id, + "variant_id": row.variant_id, + "variant_label": row.variant_label, "platform": row.platform, "result_id": row.result_id, "short_id": full_to_short.get(row.result_id, ""), @@ -434,13 +438,15 @@ def _compute_average_ranks(platform_agg: dict[str, dict[str, Any]]) -> list[dict records.append( { "platform_id": pdata["platform_id"], + "variant_id": pdata.get("variant_id") or pdata["platform_id"], + "variant_label": pdata.get("variant_label") or pdata["platform"], "platform": pdata["platform"], "ranks": ranks, "avg_rank": avg_rank, "n_cohorts": len(ranks), } ) - records.sort(key=lambda p: (p["avg_rank"] is None, p["avg_rank"] or float("inf"), p["platform_id"])) + records.sort(key=lambda p: (p["avg_rank"] is None, p["avg_rank"] or float("inf"), p["variant_id"])) return records @@ -473,9 +479,15 @@ def _build_meta_leaderboard( rank = entry["rank"] if rank is None: continue - pid = entry["platform_id"] + pid = entry["variant_id"] or entry["platform_id"] if pid not in platform_agg: - platform_agg[pid] = {"platform_id": pid, "platform": entry["platform"], "ranks": {}} + platform_agg[pid] = { + "platform_id": entry["platform_id"], + "variant_id": entry["variant_id"], + "variant_label": entry["variant_label"], + "platform": entry["platform"], + "ranks": {}, + } existing = platform_agg[pid]["ranks"].get(cohort_key) if existing is None or rank < existing["rank"]: platform_agg[pid]["ranks"][cohort_key] = { diff --git a/_project/scripts/explorer_pipeline/transformer.py b/_project/scripts/explorer_pipeline/transformer.py index 32a6accbf3..14cb971df6 100644 --- a/_project/scripts/explorer_pipeline/transformer.py +++ b/_project/scripts/explorer_pipeline/transformer.py @@ -8,6 +8,7 @@ import math import re from collections import Counter +from collections.abc import Mapping from decimal import Decimal, InvalidOperation from pathlib import Path from typing import Any, cast @@ -32,6 +33,7 @@ canonical_phase, ranking_exclusion_reason, timing_eligibility, + variant_label, ) from benchbox.core.benchmark_registry import get_benchmark_support_status from benchbox.core.cost.models import CostScope, CostStatus, DeploymentMetadata, NormalizedCost @@ -394,6 +396,36 @@ def _platform_version(bundle: BundleDocument) -> str | None: return str(val) if val and val != "unknown" else None +def _variant_info(bundle: BundleDocument) -> tuple[str, str, str, str | None, bool]: + platform = bundle.platform + platform_name = str(platform.name) + pid = _platform_id(platform_name) + raw_variant = getattr(platform, "variant", None) + engine_raw = getattr(platform, "execution_engine", None) + engine = engine_raw if isinstance(engine_raw, Mapping) else {} + requested = engine.get("requested") or "default" + requested = str(requested) + gateway_raw = getattr(platform, "gateway", None) + gateway = gateway_raw if isinstance(gateway_raw, Mapping) else {} + routed = bool(gateway.get("routed", False)) + gateway_name = gateway.get("name") + gateway_name = str(gateway_name) if gateway_name else None + if isinstance(raw_variant, str) and raw_variant.strip(): + variant_id = raw_variant.strip() + elif requested != "default": + variant_id = f"{pid}+{requested}" + else: + variant_id = pid + label = variant_label( + platform_name, + variant_id=variant_id, + requested_execution_engine=requested, + gateway_name=gateway_name, + gateway_routed=routed, + ) + return variant_id, label, requested, gateway_name, routed + + _EXECUTION_MODES = frozenset({"sql", "dataframe"}) @@ -1181,12 +1213,18 @@ def to_manifest_entry( ) override = _override_display(bundle_path) test_type = _test_type(bundle) + variant = _variant_info(bundle) entry = ManifestEntry( result_id=rid, benchmark=benchmark, scale_factor=scale_factor, platform=platform, platform_id=_platform_id(platform), + variant_id=variant[0], + variant_label=variant[1], + requested_execution_engine=variant[2], + gateway_name=variant[3], + gateway_routed=variant[4], driver_version=_driver_version(bundle), run_date=run_date, power_score=_power_score(bundle), @@ -1273,12 +1311,18 @@ def to_detail_result( timing_contract = timing_eligibility(display_timings, logical_query_count) normalized_cost = _normalized_cost(bundle) test_type = _test_type(bundle) + variant = _variant_info(bundle) detail = DetailResult( result_id=result_id, benchmark=benchmark, scale_factor=scale_factor, platform=platform, platform_id=_platform_id(platform), + variant_id=variant[0], + variant_label=variant[1], + requested_execution_engine=variant[2], + gateway_name=variant[3], + gateway_routed=variant[4], driver_version=_driver_version(bundle), run_date=run_date, total_duration_s=total_duration_s, diff --git a/docs/development/browser-duckdb-schema.sql b/docs/development/browser-duckdb-schema.sql index e7fb6fe0d2..acd4312ef7 100644 --- a/docs/development/browser-duckdb-schema.sql +++ b/docs/development/browser-duckdb-schema.sql @@ -41,6 +41,11 @@ CREATE TABLE IF NOT EXISTS results ( scale_factor DOUBLE NOT NULL, platform VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, + requested_execution_engine VARCHAR NOT NULL, + gateway_name VARCHAR, + gateway_routed BOOLEAN NOT NULL, driver_version VARCHAR, run_date VARCHAR NOT NULL, power_score DOUBLE, @@ -136,6 +141,11 @@ SELECT r.scale_factor, r.platform, r.platform_id, + r.variant_id, + r.variant_label, + r.requested_execution_engine, + r.gateway_name, + r.gateway_routed, r.driver_version, r.run_date, r.power_score, @@ -237,6 +247,8 @@ CREATE TABLE IF NOT EXISTS benchmark_rankings ( phase VARCHAR NOT NULL, result_id VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, short_id VARCHAR NOT NULL, trust_label VARCHAR NOT NULL, @@ -339,6 +351,8 @@ CREATE TABLE IF NOT EXISTS cohort_metadata ( primary_metric VARCHAR NOT NULL, primary_order VARCHAR NOT NULL, platform_id VARCHAR NOT NULL, + variant_id VARCHAR NOT NULL, + variant_label VARCHAR NOT NULL, platform VARCHAR NOT NULL, result_id VARCHAR NOT NULL, short_id VARCHAR NOT NULL, @@ -366,7 +380,9 @@ CREATE INDEX IF NOT EXISTS idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id); CREATE TABLE IF NOT EXISTS meta_leaderboard ( - platform_id VARCHAR PRIMARY KEY, + variant_id VARCHAR PRIMARY KEY, + variant_label VARCHAR NOT NULL, + platform_id VARCHAR NOT NULL, platform VARCHAR NOT NULL, avg_rank DOUBLE, n_cohorts INTEGER NOT NULL diff --git a/results-data/CORPUS_NOTES.md b/results-data/CORPUS_NOTES.md index 23be13d40d..8b55c52bdf 100644 --- a/results-data/CORPUS_NOTES.md +++ b/results-data/CORPUS_NOTES.md @@ -64,6 +64,13 @@ Both cohorts meet the >=3-platform depth criterion required for the Compare view - `tpch SF=0.01`: DuckDB, DataFusion, Polars - `star_schema SF=0.01`: DuckDB, DataFusion, SQLite +## Execution-variant identity (schema 2.3 and later) +Bundles that record `platform.variant` count toward cohort depth under the variant +string; every bundle committed here predates that field, so all existing identities +are unchanged platform names and no cohort gains or loses depth from the rename. +Gateway-routed bundles are excluded from native ranking with +`ranking_exclusion_reason=gateway_routed` and contribute no identity. + ## Zero-query DataFrame withdrawal (2026-08-24) Sixty legacy DataFrame bundles were withdrawn because they reported diff --git a/results-data/SEED_CORPUS_SPEC.md b/results-data/SEED_CORPUS_SPEC.md index 66fa3f857f..635aa41bd5 100644 --- a/results-data/SEED_CORPUS_SPEC.md +++ b/results-data/SEED_CORPUS_SPEC.md @@ -18,7 +18,11 @@ Phase 3 hosted service. The hard requirement today is still the validator gate in `results-data/validate_corpus.py`: every committed cohort must have at least 3 distinct comparison identities. Ordinary -cohorts count platform names, preserving the historical three-platform floor. Bundles in +cohorts count platform names, preserving the historical three-platform floor. A bundle +that records `platform.variant` counts under that variant string instead of the bare +platform name, so two execution variants of one platform (for example +`polars-df+streaming` alongside `polars-df`) are distinct identities while repeated +runs of one variant do not pad the floor. Bundles in the explicitly segregated `duckdb-version-matrix/` corpus count platform plus version; repeated runs of one platform/version do not pad that matrix. diff --git a/results-data/validate_corpus.py b/results-data/validate_corpus.py index 98c1c04793..9a5f4127dc 100644 --- a/results-data/validate_corpus.py +++ b/results-data/validate_corpus.py @@ -376,6 +376,11 @@ def exclusion_reason(payload: dict) -> str | None: return "validation_not_clean" if not _tuning_applied(payload): return "tuning_not_applied" + platform_section = payload.get("platform") + if isinstance(platform_section, dict): + gateway = platform_section.get("gateway") + if isinstance(gateway, dict) and gateway.get("routed"): + return "gateway_routed" display = _display_timings(payload) timing = _timing_exclusion(display, _logical_query_count(payload, display)) if timing is not None: @@ -404,6 +409,21 @@ def _cohort_key(payload: dict) -> CohortKey: return (benchmark_id, f"{scale_factor}{cohort_phase_suffix(payload)}") +def _variant_aware_identity(platform_section: dict, platform_name: str) -> str: + variant = platform_section.get("variant") + if isinstance(variant, str) and variant.strip(): + return variant.strip() + engine = platform_section.get("execution_engine") + requested = engine.get("requested") if isinstance(engine, dict) else None + if requested and str(requested) != "default": + pid = _VARIANT_PLATFORM_SUFFIX_RE.sub("", platform_name).strip().lower().replace(" ", "-") + return f"{pid}+{requested}" + return platform_name + + +_VARIANT_PLATFORM_SUFFIX_RE = re.compile(r"[-_]trust[-_](?:ci|community|local|unknown)", re.IGNORECASE) + + def _comparison_identity(bundle: pathlib.Path, payload: dict) -> str: try: platform_section = payload["platform"] @@ -426,6 +446,7 @@ def _comparison_identity(bundle: pathlib.Path, payload: dict) -> str: except Exception as exc: raise CorpusReadError(f"ERROR reading platform identity from {bundle}: {exc}") from exc identity = str(platform) + identity = _variant_aware_identity(platform_section, identity) if version and str(version) != "unknown": identity = f"{identity} v{str(version)[:120]}" return identity diff --git a/results-explorer/scripts/explorer-build-contract.mjs b/results-explorer/scripts/explorer-build-contract.mjs index 4265fcacb4..0b093c6da9 100644 --- a/results-explorer/scripts/explorer-build-contract.mjs +++ b/results-explorer/scripts/explorer-build-contract.mjs @@ -3,7 +3,7 @@ import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; const EXPECTED_CONTRACT_VERSION = "6"; -export const EXPECTED_READ_MODEL_VERSION = 15; +export const EXPECTED_READ_MODEL_VERSION = 16; const EXPECTED_NEWER_READ_MODEL_POLICY = "warn-and-continue"; const REQUIRED_FLAGS = ["--data-dir", "--output", "--trust-label", "--visibility"]; const REQUIRED_OUTPUTS = ["results.duckdb", "bundles/{result_id}.json"]; diff --git a/results-explorer/src/__tests__/dbRecovery.test.ts b/results-explorer/src/__tests__/dbRecovery.test.ts index 27178611bd..ba8eaa7b90 100644 --- a/results-explorer/src/__tests__/dbRecovery.test.ts +++ b/results-explorer/src/__tests__/dbRecovery.test.ts @@ -23,7 +23,7 @@ const hoisted = vi.hoisted(() => { ((sql: string, params: unknown[], attempt: number) => Array>) | null = null; function healthyQueryImpl(sql: string): FakeRows { - if (/read_model_version/i.test(sql)) return rows([{ read_model_version: 15 }]); + if (/read_model_version/i.test(sql)) return rows([{ read_model_version: 16 }]); if (/^ATTACH/i.test(sql)) return rows([]); if (/^SET /i.test(sql)) return rows([]); if (/COUNT\(\*\)/i.test(sql)) return rows([{ n: 1 }]); diff --git a/results-explorer/src/components/ComparabilityReceipt.tsx b/results-explorer/src/components/ComparabilityReceipt.tsx index bd0b9dc170..ebd67760bf 100644 --- a/results-explorer/src/components/ComparabilityReceipt.tsx +++ b/results-explorer/src/components/ComparabilityReceipt.tsx @@ -98,6 +98,16 @@ export function buildComparabilityFields(results: DetailResult[]): Comparability compareValues("Platform version", results, (result) => valueOrMissing(result.platform_version)), compareValues("Driver version", results, (result) => valueOrMissing(result.driver_version)), compareValues("Execution mode", results, (result) => result.execution_mode ? formatExecutionMode(result.execution_mode) : "Not recorded"), + compareValues( + "Execution engine", + results, + (result) => result.requested_execution_engine ?? "Not recorded", + ), + compareValues( + "Gateway", + results, + (result) => (result.gateway_routed ? `Routed via ${result.gateway_name ?? "unknown"}` : "Native"), + ), buildTuningField(results), compareValues( "Validation", diff --git a/results-explorer/src/components/MetaLeaderboard.tsx b/results-explorer/src/components/MetaLeaderboard.tsx index 7922ad2e0a..17e5e31d7f 100644 --- a/results-explorer/src/components/MetaLeaderboard.tsx +++ b/results-explorer/src/components/MetaLeaderboard.tsx @@ -96,7 +96,7 @@ export function MetaLeaderboard({ for (const cohort of cohorts) { const values = platforms .map((platform) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); const state = metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort); return state.kind === "ranked" ? cellShadeMetric(state.rank, mode, cohort) : null; }) @@ -121,11 +121,11 @@ export function MetaLeaderboard({ [cohorts, sortedPlatforms], ); const collapsedPlatformIds = useMemo( - () => new Set(collapsedPlatforms.map((platform) => platform.platform_id)), + () => new Set(collapsedPlatforms.map((platform) => variantKeyOf(platform))), [collapsedPlatforms], ); const defaultPlatforms = useMemo( - () => sortedPlatforms.filter((platform) => !collapsedPlatformIds.has(platform.platform_id)), + () => sortedPlatforms.filter((platform) => !collapsedPlatformIds.has(variantKeyOf(platform))), [collapsedPlatformIds, sortedPlatforms], ); const renderedPlatforms = showUnrankedPlatforms @@ -194,7 +194,7 @@ export function MetaLeaderboard({ case " ": { event.preventDefault(); const platform = visiblePlatforms[rowIdx]; - if (platform) route(platformHref(platform.platform_id)); + if (platform) route(platformHref(variantKeyOf(platform))); return; } default: @@ -359,18 +359,18 @@ export function MetaLeaderboard({ {visiblePlatforms.map((platform, rowIdx) => ( route(platformHref(platform.platform_id))} + onClick={() => route(platformHref(variantKeyOf(platform)))} >
event.stopPropagation()} > - {platform.platform} + {variantLabelOf(platform)} {cohorts.map((cohort, colIdx) => { const rank = platform.ranks[cohort.key]; - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); const cellState = metaLeaderboardCellState(rank, result, cohort); const missing = cellState.kind === "missing"; const unranked = cellState.kind === "unranked"; @@ -595,8 +595,16 @@ function exactMetricTitle(rank: MetaRank | undefined, cohort: MetaCohort): strin return null; } +export function variantKeyOf(row: { variant_id?: string | null; platform_id: string }): string { + return row.variant_id || row.platform_id; +} + +export function variantLabelOf(row: { variant_label?: string | null; platform: string }): string { + return row.variant_label || row.platform; +} + function findCohortPlatform(cohort: MetaCohort, platformId: string): MetaCohortPlatform | undefined { - const rows = cohort.platforms?.filter((row) => row.platform_id === platformId); + const rows = cohort.platforms?.filter((row) => variantKeyOf(row) === platformId); if (!rows || rows.length === 0) return undefined; return rows.find((row) => row.rank !== null) ?? rows[0]; } @@ -748,12 +756,12 @@ function runAgeText(runAge: string | null): string | null { } function hasPublishedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boolean { - return cohorts.some((cohort) => findCohortPlatform(cohort, platform.platform_id) !== undefined); + return cohorts.some((cohort) => findCohortPlatform(cohort, variantKeyOf(platform)) !== undefined); } function hasRankedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boolean { return cohorts.some((cohort) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); return metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort).kind === "ranked"; }); } @@ -761,7 +769,7 @@ function hasRankedEvidence(platform: MetaPlatform, cohorts: MetaCohort[]): boole function allExcludedRankingReason(cohort: MetaCohort, platforms: MetaPlatform[]): string | null { const results = cohort.platforms ?? []; const hasRankedCell = platforms.some((platform) => { - const result = findCohortPlatform(cohort, platform.platform_id); + const result = findCohortPlatform(cohort, variantKeyOf(platform)); return metaLeaderboardCellState(platform.ranks[cohort.key], result, cohort).kind === "ranked"; }); if (results.length === 0 || hasRankedCell) return null; diff --git a/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx b/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx index f998a8d8f2..1e88b4b405 100644 --- a/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx +++ b/results-explorer/src/components/__tests__/ComparabilityReceipt.test.tsx @@ -76,6 +76,26 @@ describe("ComparabilityReceipt", () => { expect(receipt).not.toHaveTextContent("benchbox.core.cost.pricing"); }); + it("shows execution engine and gateway routing", () => { + const fields = buildComparabilityFields([ + makeDetail({ requested_execution_engine: "default" }), + makeDetail({ + result_id: "r2", + requested_execution_engine: "streaming", + gateway_name: "greybeam", + gateway_routed: true, + }), + ]); + expect(fields.find((field) => field.label === "Execution engine")).toMatchObject({ + status: "diff", + detail: "DuckDB: default; DuckDB: streaming", + }); + expect(fields.find((field) => field.label === "Gateway")).toMatchObject({ + status: "diff", + detail: "DuckDB: Native; DuckDB: Routed via greybeam", + }); + }); + it("flags normalized cost metadata differences", () => { const fields = buildComparabilityFields([ makeDetail({ normalized_cost_usd: 0.42, cost_status: "normalized" }), diff --git a/results-explorer/src/db.ts b/results-explorer/src/db.ts index 36780f5503..bac9510aba 100644 --- a/results-explorer/src/db.ts +++ b/results-explorer/src/db.ts @@ -45,7 +45,7 @@ async function createCspBoundWorker(workerPath: string): Promise { return worker; } -const EXPECTED_READ_MODEL_VERSION = 15; +const EXPECTED_READ_MODEL_VERSION = 16; const SNAPSHOT_READY_SCANS = [ { diff --git a/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts b/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts index ecc678ac07..c67bd0aaf9 100644 --- a/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts +++ b/results-explorer/src/lib/__tests__/db-remediation-pin.test.ts @@ -5,7 +5,7 @@ import { _EXPECTED_READ_MODEL_VERSION_FOR_TEST, _verifyReadModelVersionForTest } const EXPECTED_EXPLORER_BUILD_COMMAND = "uv run -- python _project/scripts/explorer_publish.py build"; -const CURRENT_READ_MODEL_VERSION = 15; +const CURRENT_READ_MODEL_VERSION = 16; const NEWER_READ_MODEL_POLICY = "warn-and-continue"; const repoRoot = resolve(process.cwd(), ".."); diff --git a/results-explorer/src/lib/duckdbQueries.ts b/results-explorer/src/lib/duckdbQueries.ts index 4ebe5e6acb..86c0b7afcc 100644 --- a/results-explorer/src/lib/duckdbQueries.ts +++ b/results-explorer/src/lib/duckdbQueries.ts @@ -106,6 +106,11 @@ export interface ResultRow extends CostDeploymentFields { scale_factor: number; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; + requested_execution_engine?: string | null; + gateway_name?: string | null; + gateway_routed?: boolean | null; driver_version: string | null; run_date: string; power_score: number | null; @@ -272,6 +277,8 @@ export interface PlatformIndexRowRow extends CostDeploymentFields { phase: string; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; driver_version: string | null; platform_version?: string | null; run_date: string; @@ -338,6 +345,8 @@ export interface CohortMetadataRow { } export interface MetaLeaderboardRow { + variant_id?: string | null; + variant_label?: string | null; platform_id: string; platform: string; avg_rank: number | null; @@ -366,6 +375,11 @@ const RESULT_COLUMNS = [ "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -434,6 +448,11 @@ const RESULT_DETAIL_METRICS_COLUMNS = [ "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -524,6 +543,8 @@ const COHORT_METADATA_COLUMNS = [ "primary_metric", "primary_order", "platform_id", + "variant_id", + "variant_label", "platform", "result_id", "short_id", @@ -548,6 +569,8 @@ const BENCHMARK_RANKING_COLUMNS = [ "br.phase", "br.result_id", "br.platform_id", + "br.variant_id", + "br.variant_label", "br.platform", "br.short_id", "br.trust_label", @@ -1133,6 +1156,8 @@ function loadPlatformIndexRows(platformId?: string): Promise { "meta-leaderboard", () => queryRows( - "SELECT platform_id, platform, avg_rank, n_cohorts" + + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts" + " FROM bench.meta_leaderboard" + - " ORDER BY avg_rank NULLS LAST, platform_id", + " ORDER BY avg_rank NULLS LAST, variant_id", ), ); } @@ -1220,9 +1245,9 @@ export async function getMetaLeaderboardData(): Promise async function loadMetaLeaderboardData(): Promise { const [platformRows, cohortRows] = await Promise.all([ queryRows( - "SELECT platform_id, platform, avg_rank, n_cohorts" + + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts" + " FROM bench.meta_leaderboard" + - " ORDER BY avg_rank NULLS LAST, platform_id", + " ORDER BY avg_rank NULLS LAST, variant_id", ), queryRows( `SELECT ${COHORT_METADATA_COLUMNS} FROM bench.cohort_metadata` + diff --git a/results-explorer/src/pages/Leaderboard.tsx b/results-explorer/src/pages/Leaderboard.tsx index 0ea6f56af1..ab86342a67 100644 --- a/results-explorer/src/pages/Leaderboard.tsx +++ b/results-explorer/src/pages/Leaderboard.tsx @@ -15,7 +15,7 @@ import { humanizeBenchmark, errMsg } from "@/utils"; import { canonicalBenchmarkSlug, formatBenchmarkLabel } from "@/lib/displayLabels"; import { MetaLeaderboardSkeleton, SkeletonBlock } from "@/components/LoadingSpinner"; import { ErrorMessage } from "@/components/ErrorMessage"; -import { MetaLeaderboard } from "@/components/MetaLeaderboard"; +import { MetaLeaderboard, variantKeyOf, variantLabelOf } from "@/components/MetaLeaderboard"; import type { MetaLeaderboardMode } from "@/components/MetaLeaderboard"; import { ALL_FACET_KEYS, @@ -233,9 +233,9 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { for (const cohort of metaLeaderboard.cohorts) { const inner = new Map(); for (const row of cohort.platforms ?? []) { - const rows = inner.get(row.platform_id) ?? []; + const rows = inner.get(variantKeyOf(row)) ?? []; rows.push(row); - inner.set(row.platform_id, rows); + inner.set(variantKeyOf(row), rows); } index.set(cohort.key, inner); } @@ -264,13 +264,13 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { }); const visibleEvidencePlatformIds = new Set( - visibleCohorts.flatMap((cohort) => (cohort.platforms ?? []).map((row) => row.platform_id)), + visibleCohorts.flatMap((cohort) => (cohort.platforms ?? []).map((row) => variantKeyOf(row))), ); const platforms = metaLeaderboard.platforms .map((platform) => { const ranks = Object.fromEntries( visibleCohorts.flatMap((cohort) => { - const rows = cohortPlatformIndex.get(cohort.key)?.get(platform.platform_id) ?? []; + const rows = cohortPlatformIndex.get(cohort.key)?.get(variantKeyOf(platform)) ?? []; const rankedRows = rows .filter((row) => { const result = resultById.get(row.result_id); @@ -300,7 +300,7 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { n_cohorts: rankValues.length, } satisfies MetaPlatform; }) - .filter((platform) => platform.n_cohorts > 0 || visibleEvidencePlatformIds.has(platform.platform_id)) + .filter((platform) => platform.n_cohorts > 0 || visibleEvidencePlatformIds.has(variantKeyOf(platform))) .sort((a, b) => { if (a.avg_rank === null && b.avg_rank === null) return 0; if (a.avg_rank === null) return 1; @@ -353,12 +353,12 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { modeRaw === "times" || modeRaw === "ranks" ? modeRaw : "speedup"; const benchmarks = [...new Set(results.map((result) => canonicalBenchmarkSlug(result.benchmark)))].sort(); const platformIdToName = new Map( - (metaLeaderboard?.platforms ?? []).map((platform) => [platform.platform_id, platform.platform]), + (metaLeaderboard?.platforms ?? []).map((platform) => [variantKeyOf(platform), variantLabelOf(platform)]), ); for (const result of results) { - platformIdToName.set(result.platform_id, result.platform); + platformIdToName.set(variantKeyOf(result), variantLabelOf(result)); } - const platformIds = [...new Set(results.map((result) => result.platform_id))].sort(); + const platformIds = [...new Set(results.map((result) => variantKeyOf(result)))].sort(); const activeFacetSummaries = summarizeActiveFacets(facets, platformIdToName); const benchmarkOptions = metaLeaderboard ? [...new Set(metaLeaderboard.cohorts.map((cohort) => cohort.benchmark))].sort() @@ -390,7 +390,7 @@ export function Leaderboard({ notice = null }: LeaderboardProps) { const publicBenchmarksOutsideLeaderboard = benchmarks.filter((benchmark) => !leaderboardBenchmarkSet.has(canonicalBenchmarkSlug(benchmark))); const leaderboardEvidencePlatformIds = new Set( (metaLeaderboard?.cohorts ?? []).flatMap((cohort) => - (cohort.platforms ?? []).map((platform) => platform.platform_id), + (cohort.platforms ?? []).map((platform) => variantKeyOf(platform)), ), ); const publicPlatformIdsOutsideLeaderboard = platformIds.filter((platformId) => !leaderboardEvidencePlatformIds.has(platformId)); diff --git a/results-explorer/src/pages/PlatformIndex.tsx b/results-explorer/src/pages/PlatformIndex.tsx index f2d3d69098..d628be9596 100644 --- a/results-explorer/src/pages/PlatformIndex.tsx +++ b/results-explorer/src/pages/PlatformIndex.tsx @@ -151,6 +151,10 @@ function canonicalPlatformRouteKey(value: string): string { function platformRowsForRequest(rows: PlatformIndexRowRow[], platform: string): PlatformIndexRowRow[] { const requested = canonicalPlatformRouteKey(platform); + const variantMatches = rows.filter( + (row) => normalizePlatformKey(row.variant_id || row.platform_id) === requested, + ); + if (variantMatches.length > 0) return variantMatches; const idMatches = rows.filter((row) => normalizePlatformKey(row.platform_id) === requested); if (idMatches.length > 0) return idMatches; return rows.filter((row) => normalizePlatformKey(row.platform) === requested); diff --git a/results-explorer/src/pages/__tests__/Leaderboard.test.tsx b/results-explorer/src/pages/__tests__/Leaderboard.test.tsx index 93849d946b..b8de7936ca 100644 --- a/results-explorer/src/pages/__tests__/Leaderboard.test.tsx +++ b/results-explorer/src/pages/__tests__/Leaderboard.test.tsx @@ -37,7 +37,7 @@ beforeEach(() => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -51,7 +51,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows.promise; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) return []; + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) return []; if (s.includes("FROM bench.cohort_metadata")) return []; return []; }); @@ -90,7 +90,7 @@ describe("Leaderboard", () => { resultCalls += 1; return resultCalls === 1 ? [] : retryRows.promise; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return metaRows.promise; } if (s.includes("FROM bench.cohort_metadata")) { @@ -135,7 +135,7 @@ describe("Leaderboard", () => { resultCalls += 1; return []; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) { @@ -165,7 +165,7 @@ describe("Leaderboard", () => { if (shouldFail) throw new Error("Network error loading results"); return RESULT_ROWS; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { metaCalls += 1; if (shouldFail) throw new Error("Network error loading metadata"); return META_LEADERBOARD_ROWS; @@ -229,7 +229,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return unlabelledRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -297,7 +297,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows; @@ -341,7 +341,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows.promise; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return metaRows.promise; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows.promise; @@ -635,7 +635,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return results; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohorts; @@ -694,7 +694,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return results; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohorts; @@ -719,7 +719,7 @@ describe("Leaderboard", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return resultRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return cohortRows; @@ -771,7 +771,7 @@ describe("toggleFacetValue (w13)", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return versionedRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -798,7 +798,7 @@ describe("toggleFacetValue (w13)", () => { ? versionedRows.filter((row) => row.platform_version === "1.4.0") : versionedRows; } - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -828,7 +828,7 @@ describe("toggleFacetValue (w13)", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("JOIN bench.result_detail_metrics")) return hardwareRows; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; diff --git a/results-explorer/src/pages/__tests__/Overview.test.tsx b/results-explorer/src/pages/__tests__/Overview.test.tsx index bad4272c87..4b7a3f0274 100644 --- a/results-explorer/src/pages/__tests__/Overview.test.tsx +++ b/results-explorer/src/pages/__tests__/Overview.test.tsx @@ -18,7 +18,7 @@ beforeEach(() => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -65,7 +65,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return singleResultRow; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return singleMetaLeaderboard; } if (s.includes("FROM bench.cohort_metadata")) return singleCohort; @@ -88,7 +88,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return RESULT_ROWS; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return []; @@ -119,7 +119,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return withoutCost; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; @@ -140,7 +140,7 @@ describe("Overview", () => { vi.mocked(queryRows).mockImplementation(async (sql: string) => { const s = String(sql).replace(/\s+/g, " ").trim(); if (s.includes("FROM bench.results")) return withoutPower; - if (s.startsWith("SELECT platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { + if (s.startsWith("SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM bench.meta_leaderboard")) { return META_LEADERBOARD_ROWS; } if (s.includes("FROM bench.cohort_metadata")) return COHORT_ROWS; diff --git a/results-explorer/src/types.ts b/results-explorer/src/types.ts index f9af5855aa..2f2a719efd 100644 --- a/results-explorer/src/types.ts +++ b/results-explorer/src/types.ts @@ -58,6 +58,11 @@ export interface DetailResult extends CostDeploymentFields { scale_factor: number; platform: string; platform_id: string; + variant_id?: string | null; + variant_label?: string | null; + requested_execution_engine?: string | null; + gateway_name?: string | null; + gateway_routed?: boolean | null; driver_version: string | null; run_date: string; total_duration_s: number; @@ -182,6 +187,8 @@ export interface MetaRank { export interface MetaPlatform { platform_id: string; + variant_id?: string | null; + variant_label?: string | null; platform: string; ranks: Record; avg_rank: number | null; @@ -190,6 +197,8 @@ export interface MetaPlatform { export interface MetaCohortPlatform { platform_id: string; + variant_id?: string | null; + variant_label?: string | null; platform: string; result_id: string; rank: number | null; diff --git a/scripts/publication/check_explorer_compat.py b/scripts/publication/check_explorer_compat.py index 6f03a1a4ac..ff505b849b 100755 --- a/scripts/publication/check_explorer_compat.py +++ b/scripts/publication/check_explorer_compat.py @@ -354,6 +354,40 @@ def normalize_type(type_str: str) -> str: "results": _V15_RESULTS_COLUMNS, } +_VARIANT_COLUMNS: dict[str, str] = { + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + "requested_execution_engine": "VARCHAR", + "gateway_name": "VARCHAR", + "gateway_routed": "BOOLEAN", +} + +TABLE_COLUMNS_V16: dict[str, dict[str, str]] = { + **TABLE_COLUMNS_V15, + "results": { + **TABLE_COLUMNS_V15["results"], + **_VARIANT_COLUMNS, + }, + "benchmark_rankings": { + **TABLE_COLUMNS_V15["benchmark_rankings"], + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + }, + "cohort_metadata": { + **TABLE_COLUMNS_V15["cohort_metadata"], + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + }, + "meta_leaderboard": { + "variant_id": "VARCHAR", + "variant_label": "VARCHAR", + "platform_id": "VARCHAR", + "platform": "VARCHAR", + "avg_rank": "DOUBLE", + "n_cohorts": "INTEGER", + }, +} + SCHEMA_REGISTRY: dict[int, dict[str, dict[str, str]]] = { 9: TABLE_COLUMNS_V9, @@ -363,6 +397,7 @@ def normalize_type(type_str: str) -> str: 13: TABLE_COLUMNS_V13, 14: TABLE_COLUMNS_V14, 15: TABLE_COLUMNS_V15, + 16: TABLE_COLUMNS_V16, } REQUIRED_INDEXES_V9: list[tuple[str, str, list[str]]] = [ @@ -419,6 +454,16 @@ def normalize_type(type_str: str) -> str: "template_evidence", ], } +REQUIRED_VIEW_COLUMNS_V16: dict[str, list[str]] = { + "result_detail_metrics": [ + *REQUIRED_VIEW_COLUMNS_V15["result_detail_metrics"], + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", + ], +} def get_table_columns_for_version(version: int) -> dict[str, dict[str, str]]: @@ -443,6 +488,12 @@ def get_indexes_for_version(version: int) -> list[tuple[str, str, list[str]]]: return list(REQUIRED_INDEXES_V10) +def _meta_leaderboard_pk(columns_map: dict[str, dict[str, str]]) -> str: + if "variant_id" in columns_map.get("meta_leaderboard", {}): + return "variant_id" + return "platform_id" + + def generate_schema_ddl(version: int) -> list[str]: columns_map = get_table_columns_for_version(version) ddl_statements: list[str] = [] @@ -492,9 +543,9 @@ def generate_schema_ddl(version: int) -> list[str]: ddl_statements.append("CREATE INDEX idx_cohort_metadata_key ON cohort_metadata (cohort_key);") ddl_statements.append("CREATE INDEX idx_cohort_metadata_platform ON cohort_metadata (cohort_key, platform_id);") + ml_cols = ", ".join(f"{c} {t}" for c, t in columns_map["meta_leaderboard"].items()) ddl_statements.append( - "CREATE TABLE meta_leaderboard (platform_id VARCHAR PRIMARY KEY, platform VARCHAR NOT NULL, " - "avg_rank DOUBLE, n_cohorts INTEGER NOT NULL);" + f"CREATE TABLE meta_leaderboard ({ml_cols}, PRIMARY KEY ({_meta_leaderboard_pk(columns_map)}));" ) ddl_statements.append("CREATE TABLE short_ids (short_id VARCHAR PRIMARY KEY, result_id VARCHAR NOT NULL UNIQUE);") @@ -557,7 +608,7 @@ def generate_schema_ddl(version: int) -> list[str]: ), ( "Meta leaderboard summary", - "SELECT platform_id, platform, avg_rank, n_cohorts FROM meta_leaderboard", + "SELECT variant_id, variant_label, platform_id, platform, avg_rank, n_cohorts FROM meta_leaderboard", ), ( "Short ID resolution", @@ -639,7 +690,9 @@ def validate_database_schema(con: Any, expected_version: int | None = None) -> l if missing_views: errors.append(f"missing required views for v{version_to_check}: {', '.join(missing_views)}") - if version_to_check >= 15: + if version_to_check >= 16: + view_column_requirements = REQUIRED_VIEW_COLUMNS_V16 + elif version_to_check >= 15: view_column_requirements = REQUIRED_VIEW_COLUMNS_V15 elif version_to_check >= 14: view_column_requirements = REQUIRED_VIEW_COLUMNS_V14 diff --git a/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py b/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py index d23b504b7b..593dd807a4 100644 --- a/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py +++ b/tests/unit/scripts/explorer_pipeline/test_duckdb_browser_contract.py @@ -41,6 +41,11 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -149,6 +154,8 @@ class TestG1SchemaContract: "phase", "result_id", "platform_id", + "variant_id", + "variant_label", "platform", "short_id", "trust_label", @@ -199,6 +206,8 @@ class TestG1SchemaContract: "primary_metric", "primary_order", "platform_id", + "variant_id", + "variant_label", "platform", "result_id", "has_display_timing", @@ -214,7 +223,7 @@ class TestG1SchemaContract: "speedup_vs_best", "stream_count", }, - "meta_leaderboard": {"platform_id", "platform", "avg_rank", "n_cohorts"}, + "meta_leaderboard": {"variant_id", "variant_label", "platform_id", "platform", "avg_rank", "n_cohorts"}, "short_ids": {"short_id", "result_id"}, "metadata": {"read_model_version"}, } @@ -226,6 +235,11 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", + "requested_execution_engine", + "gateway_name", + "gateway_routed", "driver_version", "run_date", "power_score", @@ -300,6 +314,8 @@ class TestG1SchemaContract: "scale_factor", "platform", "platform_id", + "variant_id", + "variant_label", "driver_version", "run_date", "power_score", diff --git a/tests/unit/scripts/explorer_pipeline/test_variant_identity.py b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py new file mode 100644 index 0000000000..cb66446243 --- /dev/null +++ b/tests/unit/scripts/explorer_pipeline/test_variant_identity.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from _project.scripts.explorer_pipeline.models import ( + BenchmarkSummary, + ManifestEntry, + PlatformRow, + RankingConfig, + ranking_exclusion_reason, + variant_label, +) +from _project.scripts.explorer_pipeline.ranking import rank_platforms +from _project.scripts.explorer_pipeline.transformer import BundleTransformer + +pytestmark = [pytest.mark.unit, pytest.mark.fast] + +_EXPLORER_FIXTURES = pytest.importorskip( + "tests.unit.scripts.explorer_pipeline.conftest", reason="Explorer pipeline test fixtures are not in this checkout" +) +MINIMAL_BUNDLE = _EXPLORER_FIXTURES.MINIMAL_BUNDLE + + +def _entry(**overrides) -> ManifestEntry: + base = { + "result_id": "r1", + "benchmark": "tpch", + "scale_factor": 0.01, + "platform": "polars-df", + "platform_id": "polars-df", + "driver_version": "1.0", + "run_date": "2026-01-01", + "power_score": 1.0, + "total_duration_s": 1.0, + "query_count": 22, + "display_geomean_ms": 100.0, + "trust_label": "maintainer-run", + "visibility": "public-curated", + "validation_status": "passed", + } + base.update(overrides) + return ManifestEntry(**base) + + +def test_legacy_bundle_keeps_platform_identity() -> None: + entry = _entry() + assert entry.variant_id == "polars-df" + assert entry.variant_label == "polars-df" + assert entry.requested_execution_engine == "default" + assert entry.gateway_routed is False + assert ranking_exclusion_reason(entry) is None + + +def test_engine_request_defines_variant_identity() -> None: + entry = _entry( + variant_id="polars-df+streaming", + requested_execution_engine="streaming", + variant_label="", + ) + assert entry.variant_id == "polars-df+streaming" + assert entry.variant_label == "polars-df (streaming)" + assert ranking_exclusion_reason(entry) is None + + +def test_gateway_routed_bundle_is_excluded_from_native_ranking() -> None: + entry = _entry( + platform="snowflake", + platform_id="snowflake", + variant_id="snowflake@greybeam", + gateway_name="greybeam", + gateway_routed=True, + ) + assert entry.variant_label == "snowflake via Greybeam" + assert ranking_exclusion_reason(entry) == "gateway_routed" + + +def test_variant_label_cases() -> None: + assert variant_label("duckdb", variant_id="duckdb") == "duckdb" + assert ( + variant_label("polars-df", variant_id="polars-df+streaming", requested_execution_engine="streaming") + == "polars-df (streaming)" + ) + assert ( + variant_label("snowflake", variant_id="snowflake@greybeam", gateway_name="greybeam", gateway_routed=True) + == "snowflake via Greybeam" + ) + + +def _write_bundle(tmp_path: Path, name: str, platform_block: dict) -> Path: + payload = copy.deepcopy(MINIMAL_BUNDLE) + payload["platform"] = platform_block + path = tmp_path / name + path.write_text(json.dumps(payload)) + return path + + +def test_transformer_carries_recorded_variant(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "variant.json", + { + "name": "polars-df", + "variant": "polars-df+streaming", + "execution_engine": {"requested": "streaming", "resolution": "explicit"}, + }, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "polars-df+streaming" + assert entry.variant_label == "polars-df (streaming)" + assert entry.requested_execution_engine == "streaming" + assert entry.gateway_routed is False + detail = transformer.to_detail_result(bundle, "variant-result") + assert detail.variant_id == "polars-df+streaming" + assert detail.variant_label == "polars-df (streaming)" + + +def test_transformer_carries_gateway_routing(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "gateway.json", + { + "name": "snowflake", + "variant": "snowflake@greybeam", + "gateway": {"name": "greybeam", "routed": True}, + }, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.gateway_routed is True + assert entry.gateway_name == "greybeam" + assert entry.variant_label == "snowflake via Greybeam" + assert ranking_exclusion_reason(entry) == "gateway_routed" + + +def test_transformer_legacy_bundle_keeps_identity(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle(tmp_path, "legacy.json", {"name": "duckdb", "version": "1.2.0"}) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == entry.platform_id == "duckdb" + assert entry.variant_label == "duckdb" + + +def test_transformer_blank_variant_falls_through_to_requested(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "blank.json", + {"name": "Polars", "variant": " ", "execution_engine": {"requested": "streaming"}}, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "polars+streaming" + + +def test_transformer_variant_equal_to_platform_name_is_kept(tmp_path: Path) -> None: + transformer = BundleTransformer() + bundle = _write_bundle( + tmp_path, + "same.json", + {"name": "Polars", "variant": "Polars", "execution_engine": {"requested": "streaming"}}, + ) + entry = transformer.to_manifest_entry(bundle) + assert entry.variant_id == "Polars" + + +def _row(variant_id: str, result_id: str, geomean_ms: float, *, eligible: bool = True) -> PlatformRow: + return PlatformRow( + result_id=result_id, + short_id=result_id, + platform_id="polars-df", + variant_id=variant_id, + variant_label=f"polars-df ({variant_id})", + platform="polars-df", + platform_version=None, + tuning_mode=None, + tuning_hash=None, + execution_mode=None, + trust_label="maintainer-run", + run_date="2026-05-05T00:00:00Z", + is_ranking_eligible=eligible, + ranking_exclusion_reason=None if eligible else "gateway_routed", + power_score=None, + display_geomean_ms=geomean_ms, + sample_geomean_ms=None, + cost_usd=None, + compliance_class=None, + percentile_stats=None, + phase_durations=None, + timings={}, + ) + + +def test_engine_variants_rank_as_separate_rows() -> None: + summary = BenchmarkSummary( + benchmark="tpch", + scale_factor=0.01, + phase="power", + query_ids=[], + platforms=[ + _row("polars-df", "result-default", 100.0), + _row("polars-df+streaming", "result-streaming", 200.0), + ], + ranking=RankingConfig( + primary_metric="display_geomean_ms", + secondary_metric="display_geomean_ms", + primary_order="asc", + ), + ) + ranked = rank_platforms(summary) + by_result = {row.row.result_id: row for row in ranked.rows} + assert ranked.total_ranked == 2 + assert by_result["result-default"].rank == 1 + assert by_result["result-streaming"].rank == 2 + assert by_result["result-streaming"].row.variant_id == "polars-df+streaming" + + +def test_gateway_row_is_excluded_with_reason() -> None: + summary = BenchmarkSummary( + benchmark="tpch", + scale_factor=0.01, + phase="power", + query_ids=[], + platforms=[ + _row("snowflake", "result-native", 100.0), + _row("snowflake@greybeam", "result-gateway", 50.0, eligible=False), + ], + ranking=RankingConfig( + primary_metric="display_geomean_ms", + secondary_metric="display_geomean_ms", + primary_order="asc", + ), + ) + ranked = rank_platforms(summary) + by_result = {row.row.result_id: row for row in ranked.rows} + assert ranked.total_ranked == 1 + assert by_result["result-native"].rank == 1 + assert by_result["result-gateway"].rank is None + assert by_result["result-gateway"].ranking_exclusion_reason == "gateway_routed" diff --git a/tests/unit/scripts/publication/test_check_explorer_compat.py b/tests/unit/scripts/publication/test_check_explorer_compat.py index 45dbed2c57..7aa43c3381 100644 --- a/tests/unit/scripts/publication/test_check_explorer_compat.py +++ b/tests/unit/scripts/publication/test_check_explorer_compat.py @@ -46,14 +46,14 @@ def test_normalize_type() -> None: def test_schema_versions_definitions() -> None: - assert checker.SUPPORTED_SCHEMA_VERSIONS == (15,) - assert checker.CURRENT_SCHEMA_VERSION == 15 + assert checker.SUPPORTED_SCHEMA_VERSIONS == (16,) + assert checker.CURRENT_SCHEMA_VERSION == 16 from _project.scripts.explorer_pipeline.contract import EXPLORER_READ_MODEL_VERSION assert checker.CURRENT_SCHEMA_VERSION == EXPLORER_READ_MODEL_VERSION assert checker.SUPPORTED_SCHEMA_VERSIONS == (EXPLORER_READ_MODEL_VERSION,) - cols = checker.get_table_columns_for_version(15) + cols = checker.get_table_columns_for_version(16) assert "results" in cols assert "metadata" in cols assert "result_environment" in cols @@ -110,50 +110,50 @@ def test_check_schema_compatibility_all_supported() -> None: def test_validate_database_schema_detects_missing_table() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("DROP TABLE cohort_metadata") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("missing required tables for v15: cohort_metadata" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("missing required tables for v16: cohort_metadata" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_column() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("ALTER TABLE results DROP COLUMN funding") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any("table 'results' missing required columns: funding" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_template_evidence() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("ALTER TABLE results DROP COLUMN template_evidence") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any("table 'results' missing required columns: template_evidence" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_view() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("DROP VIEW result_detail_metrics") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("missing required views for v15: result_detail_metrics" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("missing required views for v16: result_detail_metrics" in err for err in errors) finally: con.close() def test_validate_database_schema_detects_missing_view_column() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("CREATE OR REPLACE VIEW result_detail_metrics AS SELECT result_id FROM results") - errors = checker.validate_database_schema(con, expected_version=15) + errors = checker.validate_database_schema(con, expected_version=16) assert any( "view 'result_detail_metrics' missing required columns:" in err and "override_rules" in err for err in errors @@ -177,11 +177,11 @@ def test_validate_database_schema_detects_missing_view_column() -> None: def test_validate_database_schema_version_mismatch() -> None: - con = checker.create_in_memory_schema(15) + con = checker.create_in_memory_schema(16) try: con.execute("UPDATE metadata SET read_model_version = 8") - errors = checker.validate_database_schema(con, expected_version=15) - assert any("read_model_version mismatch: expected 15, got 8" in err for err in errors) + errors = checker.validate_database_schema(con, expected_version=16) + assert any("read_model_version mismatch: expected 16, got 8" in err for err in errors) finally: con.close() @@ -345,7 +345,7 @@ def test_cli_default_schema_checks(capsys: pytest.CaptureFixture[str]) -> None: assert exit_code == 0 captured = capsys.readouterr() assert "Results Explorer Compatibility" in captured.out - assert "Schema v15" in captured.out + assert "Schema v16" in captured.out assert "Schema v12" not in captured.out assert "Schema v11" not in captured.out assert "Schema v10" not in captured.out @@ -359,18 +359,18 @@ def test_cli_json_mode(capsys: pytest.CaptureFixture[str]) -> None: captured = capsys.readouterr() data = json.loads(captured.out) assert data["status"] == "passed" - assert data["current_version"] == 15 - assert "v15" in data["schema_checks"] + assert data["current_version"] == 16 + assert "v16" in data["schema_checks"] assert "v12" not in data["schema_checks"] assert "v11" not in data["schema_checks"] assert "v10" not in data["schema_checks"] def test_cli_specific_schema_version(capsys: pytest.CaptureFixture[str]) -> None: - exit_code = checker.main(["--schema-only", "--schema-versions", "15"]) + exit_code = checker.main(["--schema-only", "--schema-versions", "16"]) assert exit_code == 0 captured = capsys.readouterr() - assert "Schema v15" in captured.out + assert "Schema v16" in captured.out assert "Schema v12" not in captured.out assert "Schema v11" not in captured.out assert "Schema v10" not in captured.out @@ -438,9 +438,9 @@ def test_cli_db_path_check(tmp_path: Path, capsys: pytest.CaptureFixture[str]) - db_file = tmp_path / "test.duckdb" with duckdb.connect(str(db_file)) as con: - for stmt in checker.generate_schema_ddl(15): + for stmt in checker.generate_schema_ddl(16): con.execute(stmt) - con.execute("INSERT INTO metadata VALUES (15)") + con.execute("INSERT INTO metadata VALUES (16)") exit_code = checker.main(["--db-path", str(db_file)]) assert exit_code == 0 diff --git a/tests/unit/scripts/test_corpus_cohort_depth.py b/tests/unit/scripts/test_corpus_cohort_depth.py index 2f721ef739..4d1322a04c 100644 --- a/tests/unit/scripts/test_corpus_cohort_depth.py +++ b/tests/unit/scripts/test_corpus_cohort_depth.py @@ -67,6 +67,7 @@ def _write_bundle( platform: str, platform_version: str | None = None, execution_version: str | None = None, + platform_variant: str | None = None, run_timestamp: str = "2026-08-01T12:00:00", test_type: str | None = "power", ) -> None: @@ -74,6 +75,8 @@ def _write_bundle( platform_payload = {"name": platform} if platform_version is not None: platform_payload["version"] = platform_version + if platform_variant is not None: + platform_payload["variant"] = platform_variant payload = { "benchmark": {"id": benchmark, "scale_factor": scale}, "platform": platform_payload, @@ -111,6 +114,113 @@ def test_the_gate_detects_a_one_platform_cohort(tmp_path: Path) -> None: assert shallow == {("tpcds", "10.0"): {"DuckDB"}} +def test_engine_variants_count_as_distinct_cohort_identities(tmp_path: Path) -> None: + validator = _load_validator() + _write_bundle( + tmp_path, + "default.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+default", + ) + _write_bundle( + tmp_path, + "streaming.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + _write_bundle(tmp_path, "duckdb.json", benchmark="tpch", scale=1.0, platform="DuckDB") + + cohorts = validator.cohort_platforms(validator.discover_bundles(tmp_path)) + + assert cohorts == {("tpch", "1.0"): {"polars+default", "polars+streaming", "DuckDB"}} + assert validator.shallow_cohorts(cohorts) == {} + + +def test_repeated_variant_bundles_do_not_pad_cohort_depth(tmp_path: Path) -> None: + validator = _load_validator() + _write_bundle( + tmp_path, + "a.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + _write_bundle( + tmp_path, + "b.json", + benchmark="tpch", + scale=1.0, + platform="Polars", + platform_variant="polars+streaming", + ) + + assert validator.shallow_cohorts(validator.cohort_platforms(validator.discover_bundles(tmp_path))) == { + ("tpch", "1.0"): {"polars+streaming"} + } + + +def test_gateway_routed_bundles_contribute_no_identity(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": { + "name": "Snowflake", + "variant": "snowflake@greybeam", + "gateway": {"name": "greybeam", "routed": True}, + }, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "gateway.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.exclusion_reason(payload) == "gateway_routed" + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): set()} + + +def test_requested_engine_without_variant_string_matches_pipeline(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "constructed.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"polars+streaming"}} + + +def test_blank_variant_falls_through_to_requested_engine(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "variant": " ", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "blank.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"polars+streaming"}} + + +def test_variant_equal_to_platform_name_matches_pipeline(tmp_path: Path) -> None: + validator = _load_validator() + payload = { + "benchmark": {"id": "tpch", "scale_factor": 1.0, "test_type": "power"}, + "platform": {"name": "Polars", "variant": "Polars", "execution_engine": {"requested": "streaming"}}, + "run": {"timestamp": "2026-08-01T12:00:00"}, + **_rankable_body(), + } + (tmp_path / "same.json").write_text(json.dumps(payload), encoding="utf-8") + + assert validator.cohort_platforms(validator.discover_bundles(tmp_path)) == {("tpch", "1.0"): {"Polars"}} + + def _write_phase_bundle( directory: Path, name: str, diff --git a/tests/unit/scripts/test_explorer_build_contract.py b/tests/unit/scripts/test_explorer_build_contract.py index 95a275ca29..201e55c4c1 100644 --- a/tests/unit/scripts/test_explorer_build_contract.py +++ b/tests/unit/scripts/test_explorer_build_contract.py @@ -28,7 +28,7 @@ def test_explorer_build_contract_command_emits_expected_json() -> None: def test_explorer_build_contract_matches_duckdb_only_output_contract() -> None: outputs = EXPLORER_BUILD_CONTRACT["outputs"] - current_read_model_version = 15 + current_read_model_version = 16 assert EXPLORER_BUILD_CONTRACT["version"] == "6" assert EXPLORER_BUILD_CONTRACT["read_model_version"] == current_read_model_version