diff --git a/CHANGELOG.md b/CHANGELOG.md index e86d451289..780f559310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 excluded from native ranking with `ranking_exclusion_reason=gateway_routed`. Existing corpus values are unchanged: bundles without variant data keep `variant_id` equal to the platform id. +- The new-platform acceptance checklist and both adding-platform guides describe the reserved option words, + the deprecated-alias rule and the manifest `execution_engines` declaration with a `polars-df` worked example. + A scaffold script (`scripts/new_platform_scaffold.py --kind sql|dataframe`) prints the expected file plan + with manifest and adapter stubs; its output carries no reserved canonical keys and resolves the default + engine receipt through the inherited adapter hook. - The Polars version-matrix reproduction instructions now use `--execution-engine` for default, in-memory and streaming setups, and identify diff --git a/docs/development/adding-dataframe-platform.md b/docs/development/adding-dataframe-platform.md index ff1b322ce9..7028fffb16 100644 --- a/docs/development/adding-dataframe-platform.md +++ b/docs/development/adding-dataframe-platform.md @@ -212,6 +212,13 @@ DATAFRAME_ADAPTERS = { Before implementation, maintainers can print the expected file plan with the platform scaffold script (`--kind dataframe`). +Declare execution engines when the platform offers more than one way to execute a query, following the same +manifest `capabilities.execution_engines` block, closed class list, value-naming rule and reserved-word ban as +SQL platforms (see `docs/development/adding-new-platforms.md`). The worked example is `polars-df`, which +declares `auto` (adaptive), `in-memory` and `streaming` under the platform's own documented names. DataFrame +adapters inherit `resolve_execution_engine` from the adapter base and map the resolved engine into their +collect or compute call; unsupported values fail closed. + ## Step 5: Add Tests ### Unit Tests diff --git a/docs/development/adding-new-platforms.md b/docs/development/adding-new-platforms.md index fb1f3fc1ed..13961ace27 100644 --- a/docs/development/adding-new-platforms.md +++ b/docs/development/adding-new-platforms.md @@ -768,6 +768,50 @@ make platform-manifest make platform-manifest-check ``` +Declare execution engines when the platform offers more than one way to execute a query. Add a manifest +`capabilities.execution_engines` block where each value carries `class`, `display_name`, `description`, +`dependencies` and `selectable`. The class comes from the closed core list (`adaptive`, `in-memory`, `streaming`, +`gpu`, `native-vectorized`, `standard`, `distributed`, `jit`, `delegated`); the vocabulary parity test rejects +unknown classes and enforces that one engine name maps to one class on every platform. Name each value with +the platform's own documented term when one exists, otherwise the class name. `default` is a reserved sentinel +and is never declared. The worked example is `polars-df`: + +```python +{ + "key": "polars", + "capabilities": { + "execution_engines": { + "auto": { + "class": "adaptive", + "display_name": "Auto", + "description": "Polars chooses the engine for each query", + "dependencies": ["polars"], + "selectable": True, + }, + "in-memory": { + "class": "in-memory", + "display_name": "In-memory", + "description": "Materializes each query in memory", + "dependencies": ["polars"], + "selectable": True, + }, + "streaming": { + "class": "streaming", + "display_name": "Streaming", + "description": "Processes each query in batches to bound memory", + "dependencies": ["polars"], + "selectable": True, + }, + }, + }, +} +``` + +Adapters inherit `resolve_execution_engine` from the adapter base and map the resolved engine into their +query path; unsupported values fail closed. Never name a canonical option `engine`, `mode`, `backend`, +`runtime`, `executor` or `warehouse`; a reserved-word spelling survives only as a deprecated alias registered +in the core `DEPRECATED_OPTION_ALIASES` table with its removal release. + ### Step 2: Add SQL Dialect Support If your platform has a unique SQL dialect, add SQL compatibility rules or an explicit exemption under the phase-aware SQL compatibility system. Do not hide CREATE TABLE rewrites inside the adapter without registering or exempting them in the DDL governance inventory. Put the rules in `benchbox/sql_compat/rules/ddl_optimize/newdatabase_ddl_rewrites.py`: diff --git a/docs/development/new-platform-acceptance-checklist.md b/docs/development/new-platform-acceptance-checklist.md index 6458c01386..247b8f066d 100644 --- a/docs/development/new-platform-acceptance-checklist.md +++ b/docs/development/new-platform-acceptance-checklist.md @@ -41,6 +41,17 @@ Use this checklist before opening a platform-support PR. It is deliberately conc restrict per-query `execution_engine` to `routes_to` targets plus `unknown`, and observe `mixed` when queries differ. Comparisons and regression baseline selection fail closed across different gateway variants unless `--allow-heterogeneous-comparison` is provided. +- Declare execution engines when the platform offers more than one way to execute a query: add a manifest + `capabilities.execution_engines` block with `{name: {class, display_name, description, dependencies, selectable}}. + The class comes from the closed core list (`adaptive`, `in-memory`, `streaming`, `gpu`, `native-vectorized`, + `standard`, `distributed`, `jit`, `delegated`). Name each value with the platform's own documented term when one + exists, otherwise the class name; the same name must map to the same class on every platform (the vocabulary + parity test enforces this). `default` is a reserved sentinel meaning "pass nothing" and is never declared. + Platforms without engines omit the block and accept only `default` through the inherited + `resolve_execution_engine` hook; override the hook only to map native settings into a receipt, and fail closed + on unsupported values. Never name a canonical option `engine`, `mode`, `backend`, `runtime`, `executor` or + `warehouse`; a reserved-word spelling survives only as a deprecated alias registered in the core + `DEPRECATED_OPTION_ALIASES` table with its removal release. - Add `driver_package` and `installation_command`; dependency extras belong in `pyproject.toml`. - Add DDL/query compatibility rules or an explicit exemption in the SQL compatibility inventory when the adapter rewrites SQL or DDL. - Add unit tests for adapter construction, SQL rendering/loading behavior, registry metadata, optional import failure behavior, and any deployment-mode routing. @@ -56,6 +67,12 @@ Use this checklist before opening a platform-support PR. It is deliberately conc `_build_platform_metadata()` paths are live pass-throughs and should not be treated as independent registration authorities. - Add the `-df` CLI path or confirm the existing `adapter_factory` mapping routes to the new adapter. +- Declare execution engines when the platform offers more than one way to execute a query, following the same + manifest `capabilities.execution_engines` block, closed class list, value-naming rule and reserved-word ban as + SQL platforms above. The worked example is `polars-df`, which declares `auto` (adaptive), `in-memory` and + `streaming` under the platform's own documented names. DataFrame adapters inherit `resolve_execution_engine` + from the adapter base and map the resolved engine into their collect or compute call; unsupported values fail + closed. - Add native DataFrame query implementations or document unsupported benchmarks with benchmark-gate rules. - Add lazy availability checks and dependency extras; GPU or native-stack dependencies must not load during base import. - Add unit tests for family behavior, adapter lifecycle, query execution primitives, and registry capability alignment. @@ -87,5 +104,12 @@ Target for a straightforward DataFrame platform is **8-12 files** because native ## Scaffold Helper +Print the expected file plan with manifest and adapter stubs before implementation: + +```bash +uv run -- python scripts/new_platform_scaffold.py --name myengine --kind sql +uv run -- python scripts/new_platform_scaffold.py --name myengine --kind dataframe +``` + A maintainer scaffold script prints the expected file plan for a new SQL or DataFrame platform. The helper emits a checklist and file plan only. It is an architecture health aid, not a framework that owns platform implementation. diff --git a/scripts/new_platform_scaffold.py b/scripts/new_platform_scaffold.py new file mode 100644 index 0000000000..3a17147703 --- /dev/null +++ b/scripts/new_platform_scaffold.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import yaml + +from benchbox.core.execution_engine import EXECUTION_ENGINE_CLASSES + +SQL_BASE_IMPORT = "from benchbox.platforms.base import PlatformAdapter" +DATAFRAME_BASE_IMPORT = "from benchbox.platforms.base import PlatformAdapter" + +FILE_PLAN_SQL = ( + "benchbox/platforms/_platform.py", + "benchbox/core/platform_manifest.py (canonical entry)", + "pyproject.toml (optional dependency extra)", + "SQL compatibility rule or explicit exemption", + "tests/unit/platforms/test__platform.py", + "docs/platforms/.md", +) + +FILE_PLAN_DATAFRAME = ( + "benchbox/platforms/dataframe/_df.py", + "benchbox/core/platform_manifest.py (canonical entry)", + "pyproject.toml (optional dependency extra)", + "tests/unit/platforms/test__df.py", + "docs/platforms/-df.md", +) + + +def manifest_snippet(*, key: str, display_name: str, mode: str) -> dict: + return { + "key": key, + "display_name": display_name, + "support_status": "experimental", + "adapter": {"module": f"benchbox.platforms.{key}_platform"}, + "capabilities": { + "supports_sql": mode == "sql", + "supports_dataframe": mode == "dataframe", + "execution_engines": {}, + }, + } + + +def adapter_stub(*, key: str, display_name: str, mode: str) -> str: + base_import = SQL_BASE_IMPORT if mode == "sql" else DATAFRAME_BASE_IMPORT + class_name = "".join(part.capitalize() for part in key.split("_")) + "Adapter" + lines = [ + "from __future__ import annotations", + "", + base_import, + "", + "", + f"class {class_name}(PlatformAdapter):", + f' platform_name = "{key}"', + f' display_name = "{display_name}"', + "", + ' driver_package = ""', + ' installation_command = "uv add "', + "", + ] + return "\n".join(lines) + + +def file_plan(*, mode: str) -> tuple[str, ...]: + return FILE_PLAN_SQL if mode == "sql" else FILE_PLAN_DATAFRAME + + +def render(*, key: str, display_name: str, mode: str) -> str: + snippet = yaml.safe_dump(manifest_snippet(key=key, display_name=display_name, mode=mode), sort_keys=False) + sections = [ + f"platform: {key} ({mode})", + "", + "manifest entry:", + snippet.rstrip(), + "", + "adapter stub:", + adapter_stub(key=key, display_name=display_name, mode=mode).rstrip(), + "", + "file plan:", + *[f"- {entry.replace('', key)}" for entry in file_plan(mode=mode)], + ] + return "\n".join(sections) + "\n" + + +def validate_snippet(snippet: dict) -> list[str]: + problems: list[str] = [] + capabilities = snippet.get("capabilities", {}) + engines = capabilities.get("execution_engines", {}) + for name, engine in engines.items(): + if engine.get("class") not in EXECUTION_ENGINE_CLASSES: + problems.append(f"unknown engine class for {name!r}") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Print the new-platform file plan with manifest and adapter stubs.") + parser.add_argument("--name", required=True, help="Platform registry key, for example myengine.") + parser.add_argument("--display-name", default="", help="Human-readable platform name.") + parser.add_argument("--kind", choices=("sql", "dataframe"), default="sql") + parser.add_argument("--output-dir", default="", help="Write manifest.yaml and adapter stub files here.") + args = parser.parse_args(argv) + display_name = args.display_name or args.name.replace("_", " ").title() + snippet = manifest_snippet(key=args.name, display_name=display_name, mode=args.kind) + problems = validate_snippet(snippet) + if problems: + print("\n".join(problems), file=sys.stderr) + return 1 + if args.output_dir: + out = Path(args.output_dir) + out.mkdir(parents=True, exist_ok=True) + (out / "manifest.yaml").write_text(yaml.safe_dump(snippet, sort_keys=False)) + (out / f"{args.name}_platform.py").write_text( + adapter_stub(key=args.name, display_name=display_name, mode=args.kind) + ) + sys.stdout.write(render(key=args.name, display_name=display_name, mode=args.kind)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/scripts/test_new_platform_scaffold.py b/tests/unit/scripts/test_new_platform_scaffold.py new file mode 100644 index 0000000000..4c04182b32 --- /dev/null +++ b/tests/unit/scripts/test_new_platform_scaffold.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +from benchbox.core.execution_engine import ( + DEFAULT_EXECUTION_ENGINE, + EXECUTION_ENGINE_CLASSES, + RESERVED_OPTION_KEYS, +) +from scripts.new_platform_scaffold import adapter_stub, manifest_snippet, render, validate_snippet + +pytestmark = [pytest.mark.unit, pytest.mark.fast] + +REPO_ROOT = Path(__file__).resolve().parents[3] +SCRIPT = REPO_ROOT / "scripts" / "new_platform_scaffold.py" + + +def _load_stub(tmp_path: Path, *, key: str = "examplengine", mode: str = "sql"): + out = tmp_path / key + out.mkdir() + (out / "manifest.yaml").write_text( + yaml.safe_dump(manifest_snippet(key=key, display_name="Examplengine", mode=mode), sort_keys=False) + ) + stub_path = out / f"{key}_platform.py" + stub_path.write_text(adapter_stub(key=key, display_name="Examplengine", mode=mode)) + spec = importlib.util.spec_from_file_location(f"{key}_platform", stub_path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_scaffold_manifest_declares_no_reserved_canonical_keys(tmp_path: Path) -> None: + snippet = manifest_snippet(key="examplengine", display_name="Examplengine", mode="sql") + assert validate_snippet(snippet) == [] + assert snippet["capabilities"]["execution_engines"] == {} + assert snippet["support_status"] == "experimental" + + +def test_scaffold_stub_resolves_default_engine_receipt(tmp_path: Path) -> None: + module = _load_stub(tmp_path) + + class ConcreteAdapter(module.ExamplengineAdapter): + @classmethod + def add_cli_arguments(cls, parser): + return parser + + @classmethod + def from_config(cls, **config): + return cls(**config) + + def apply_constraint_configuration(self, *args, **kwargs): + return None + + def apply_platform_optimizations(self, *args, **kwargs): + return None + + def configure_for_benchmark(self, *args, **kwargs): + return None + + def create_connection(self, *args, **kwargs): + raise NotImplementedError + + def create_schema(self, *args, **kwargs): + raise NotImplementedError + + def execute_query(self, *args, **kwargs): + raise NotImplementedError + + def load_data(self, *args, **kwargs): + raise NotImplementedError + + def get_target_dialect(self): + return "generic" + + adapter = ConcreteAdapter() + assert adapter.platform_name == "examplengine" + receipt = adapter.resolve_execution_engine(DEFAULT_EXECUTION_ENGINE) + assert receipt.resolution == "platform_default" + assert receipt.requested == DEFAULT_EXECUTION_ENGINE + + +def test_scaffold_stub_registers_no_option_specs(tmp_path: Path) -> None: + module = _load_stub(tmp_path, key="cleanengine") + source = (tmp_path / "cleanengine" / "cleanengine_platform.py").read_text() + assert "PlatformOptionSpec" not in source + assert module.CleanengineAdapter.platform_name == "cleanengine" + for reserved in RESERVED_OPTION_KEYS: + assert f'"{reserved}"' not in source + + +def test_scaffold_engine_classes_are_core_classes() -> None: + assert set(EXECUTION_ENGINE_CLASSES) >= {"adaptive", "in-memory", "streaming"} + assert DEFAULT_EXECUTION_ENGINE == "default" + + +def test_scaffold_cli_renders_plan_and_stubs(tmp_path: Path) -> None: + rendered = render(key="demoengine", display_name="Demoengine", mode="dataframe") + assert "execution_engines: {}" in rendered + assert "class DemoengineAdapter(PlatformAdapter):" in rendered + assert "benchbox/platforms/dataframe/demoengine_df.py" in rendered + completed = subprocess.run( + [sys.executable, str(SCRIPT), "--name", "demoengine", "--kind", "dataframe", "--output-dir", str(tmp_path)], + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0 + assert (tmp_path / "manifest.yaml").exists() + assert (tmp_path / "demoengine_platform.py").exists()