From 15f49930e6d417583d211fcfd3dba4b6ade40d27 Mon Sep 17 00:00:00 2001 From: Joe Harris Date: Thu, 15 Oct 2026 23:43:27 -0400 Subject: [PATCH] feat(tuning): validate requested tuning against what execution applies Tuning validation now reads the capability registry. A tuning type that the registry marks as not applied at execution is a validation error, and platforms without registry coverage error on unsupported types the same way covered platforms do. Validation, reconciliation and the sample and fallback configuration builders share one registry lookup. The registry gains explicit entries for platforms and tuning types that previously had none (Postgres family, Doris, SingleStore, Databend, Firebolt, QuestDB, Trino, Presto, Athena, Azure Synapse, Fabric Warehouse, BigQuery, Redshift, MotherDuck, DataFusion, SQLite and others). Fabric Warehouse gets its own entry instead of sharing Databricks. Variant platform strings such as duckdb:memory resolve to their base platform's rules. Redshift sorting is validated as applied only when sorted ingestion is on, because the adapter sorts tables only in that mode. Platform aliases such as spark and delta run the same effective-layout validator as the platform they resolve to. Error and drop-reason text names the tuning type and mechanism with a short plain reason. It no longer quotes source files and line numbers. Requested UNIQUE and CHECK constraints, layout tuning on BigQuery, Redshift, Trino and Azure Synapse, Databricks, Spark and Delta partitioning, and primary or foreign keys that an adapter removes now fail validation. The changelog lists them under "Before you upgrade". Packaged DuckDB and Snowflake templates and the matching examples disable UNIQUE and CHECK. A table in the registry parity test pins which tuning types each platform applies; changing the registry requires changing that table. Behavioral tests cover benchmark DDL with the unrendered flags and the adapter rewrites the registry entries rest on. --- CHANGELOG.md | 45 +- benchbox/cli/commands/tuning_group.py | 8 - benchbox/cli/config.py | 11 + benchbox/cli/tuning_runtime.py | 10 +- benchbox/core/tuning/capability_registry.py | 468 ++++++++++- benchbox/core/tuning/interface.py | 29 +- benchbox/core/tuning/reconciliation.py | 11 +- .../templates/duckdb/joinorder_tuned.yaml | 2 +- .../duckdb/read_primitives_tuned.yaml | 2 +- .../tuning/templates/duckdb/ssb_tuned.yaml | 2 +- .../tuning/templates/duckdb/tpcds_tuned.yaml | 2 +- .../tuning/templates/duckdb/tpch_tuned.yaml | 2 +- .../templates/duckdb/tpchavoc_tuned.yaml | 2 +- .../templates/snowflake/tpcds_tuned.yaml | 4 +- .../templates/snowflake/tpch_tuned.yaml | 4 +- examples/tunings/duckdb/joinorder_tuned.yaml | 2 +- .../tunings/duckdb/read_primitives_tuned.yaml | 2 +- examples/tunings/duckdb/ssb_tuned.yaml | 2 +- examples/tunings/duckdb/tpcds_tuned.yaml | 2 +- examples/tunings/duckdb/tpch_tuned.yaml | 2 +- examples/tunings/duckdb/tpchavoc_tuned.yaml | 2 +- examples/tunings/snowflake/tpcds_tuned.yaml | 4 +- examples/tunings/snowflake/tpch_tuned.yaml | 4 +- scripts/generate_cloud_tpc_templates.py | 4 +- .../commands/test_run_tuning_mode_guard.py | 7 +- .../cli/commands/test_tuning_group_cli.py | 24 +- tests/unit/cli/test_config_coverage.py | 8 + tests/unit/cli/test_tuning_runtime.py | 37 +- .../tuning/test_capability_registry_parity.py | 767 ++++++++++++++++++ .../tuning/test_capability_source_drift.py | 11 + .../core/tuning/test_clickhouse_pk_prefix.py | 2 + .../core/tuning/test_fallback_description.py | 2 +- .../tuning/test_platform_identity_keys.py | 46 +- .../core/tuning/test_tuning_reconciliation.py | 24 +- .../test_tuning_reconciliation_adapters.py | 38 +- 35 files changed, 1469 insertions(+), 123 deletions(-) create mode 100644 tests/unit/core/tuning/test_capability_registry_parity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0be6542687..8bf312e5ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -332,12 +332,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mismatched keys are recorded as failed ledger statements and warned. Result metadata reports the live tuned values for shuffle partitions and adaptive execution. +- Tuning validation now rejects tuning requests the platform cannot execute, + and platforms without tuning coverage error on unsupported types like + covered platforms. The registry corrects several platforms' coverage + (Postgres family, Doris, SingleStore, Databend, Starburst, MotherDuck, + DataFusion, and fabric-warehouse, which now has its own entry instead of + sharing Databricks'), and the sample and fallback configuration builders + only enable tuning the target platform supports. Layout tuning on + BigQuery, Redshift, Trino and Synapse now errors until their execution + support lands. - Tuned load-only runs (`--phases generate,load`) now go through the same apply → load → reconcile → metadata → ledger/receipt → export lifecycle as full runs, so a load-only bundle never carries a tuned mode without `platform.tuning`. Adapters without the full lifecycle refuse tuned load-only runs with an actionable error instead. - - Result metadata records genuine engine versions (`engine_version`, `engine_version_source: sql_query`) for DuckDB, SQLite and DataFusion queried directly from the active connection rather than package versions. - ClickHouse tuning introspection bounds table queries in SQL by ledger tables and measures snapshot truncation directly on raw query results. @@ -690,6 +698,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 how to choose the engine at run time; pass `--execution-engine` (for example `--execution-engine streaming`). Tuning files that already set `engine_affinity` keep working and record `resolution` `tuning_profile`. +- Tuning validation now fails when a configuration asks for tuning that the + platform does not apply during a run. Earlier releases accepted these + configurations and ran without the tuning. Run `benchbox tuning validate + --platform ` on your saved tuning files before you upgrade; each + error names the tuning type and why it is not applied. Remove the setting + or use a platform that applies it. Platforms that apply tuning in part are + checked type by type. The new errors are: + - `unique_constraints` and `check_constraints` fail on every platform with + registry coverage, because no benchmark schema carries those clauses. The + packaged DuckDB and Snowflake templates and the matching + `examples/tunings` files now disable both. + - Layout tuning fails on BigQuery (`partitioning`, `clustering`), Redshift + (`partitioning`, `distribution`), Trino and + Azure Synapse (all four layout types). These adapters build the clauses + for dry-run previews but do not apply them when tables are created. Other + platforms that only preview layout tuning fail the same way (for example + the Postgres family, Doris, Firebolt, QuestDB, SingleStore and + TimescaleDB). + - Redshift `sorting` fails while `sorted_ingestion_mode` is `off`, which is + the default, because the adapter sorts tables only when sorted ingestion + is on. Set `sorted_ingestion_mode` to `auto` or `force` to keep the + request. + - `partitioning` fails on Databricks, and so on the `spark` and `delta` + platform names that resolve to it. The adapter builds the clause for + previews only and never adds it when tables are created. Fabric Warehouse + no longer shares the Databricks rules and also rejects `partitioning`. + - `primary_keys` and `foreign_keys` fail where the adapter removes that + constraint from table definitions: primary keys on Databend, Doris, + Firebolt, QuestDB, Trino and Presto; foreign keys on ClickHouse, Databend, + Doris, Firebolt, QuestDB, SingleStore and StarRocks. Constraints that + DataFusion and MotherDuck do not create fail the same way. + - Platforms without registry coverage now report unsupported tuning types as + errors where earlier releases reported warnings, and default tuning + configurations on platforms that drop tuning report errors where they + reported warnings. ### Added diff --git a/benchbox/cli/commands/tuning_group.py b/benchbox/cli/commands/tuning_group.py index 2116eb277c..0dd20ffead 100644 --- a/benchbox/cli/commands/tuning_group.py +++ b/benchbox/cli/commands/tuning_group.py @@ -38,7 +38,6 @@ from benchbox.core.platform_manifest import get_platform_aliases from benchbox.core.tuning.capability_registry import ( PLATFORM_ALIASES, - get_capability, known_registry_platforms, resolve_platform_key, ) @@ -343,13 +342,6 @@ def _validate_sql_config(config_file: str, platform: str) -> None: errors, warnings = config.validate_for_platform_detailed(platform_key) enabled = sorted(config.get_enabled_tuning_types(), key=lambda tuning_type: tuning_type.value) - for tuning_type in enabled: - capability = get_capability(platform_key, tuning_type) - if capability is not None and capability.rendered_via == "none": - warnings.append( - f"Tuning type '{tuning_type.value}' is accepted by platform '{platform_key}' " - f"but not rendered yet ({capability.mechanism_id})" - ) for message in errors: console.print(f"[red]Error: {message}[/red]") diff --git a/benchbox/cli/config.py b/benchbox/cli/config.py index 8d0a5ea24a..463849a8d3 100644 --- a/benchbox/cli/config.py +++ b/benchbox/cli/config.py @@ -647,6 +647,17 @@ def create_sample_unified_tuning_config(self, path: Optional[Path] = None, platf TuningColumn(name="linenumber", type="INTEGER", order=2), ] + from benchbox.core.tuning.capability_registry import get_capability, unapplied_reason + + platform_known = TuningType.is_known_platform(platform) + for tuning_type in list(sample_config.get_enabled_tuning_types()): + capability = get_capability(platform, tuning_type) + if capability is not None: + if unapplied_reason(capability, sample_config.platform_optimizations.sorted_ingestion_mode): + sample_config.disable_tuning_type(tuning_type) + elif platform_known and not tuning_type.is_compatible_with_platform(platform): + sample_config.disable_tuning_type(tuning_type) + self.save_unified_tuning_config(sample_config, path) console.print( f"[green]✅ Sample unified tuning configuration created for platform '{platform}' at {path}[/green]" diff --git a/benchbox/cli/tuning_runtime.py b/benchbox/cli/tuning_runtime.py index 0f9049ae95..5f98ab913e 100644 --- a/benchbox/cli/tuning_runtime.py +++ b/benchbox/cli/tuning_runtime.py @@ -9,8 +9,16 @@ def build_default_unified_config(platform: str | None) -> UnifiedTuningConfiguration: + from benchbox.core.tuning.capability_registry import get_capability, is_not_rendered_at_execution + config = UnifiedTuningConfiguration() - if platform and resolve_platform_key(str(platform).split(":", 1)[0]) == "duckdb": + if platform: + platform_key = resolve_platform_key(str(platform)) + for tuning_type in config.get_enabled_tuning_types(): + capability = get_capability(platform_key, tuning_type) + if capability is not None and is_not_rendered_at_execution(capability): + config.disable_tuning_type(tuning_type) + if platform and resolve_platform_key(str(platform)) == "duckdb": config.check_constraints.enabled = False return config diff --git a/benchbox/core/tuning/capability_registry.py b/benchbox/core/tuning/capability_registry.py index c32ee8463f..43696076bf 100644 --- a/benchbox/core/tuning/capability_registry.py +++ b/benchbox/core/tuning/capability_registry.py @@ -17,14 +17,20 @@ class TuningCapability: rendered_via: RenderedVia mechanism_id: str notes: str = "" + requires_sorted_ingestion: bool = False def _ddl(mechanism_id: str, notes: str = "") -> TuningCapability: return TuningCapability(rendered_via="ddl", mechanism_id=mechanism_id, notes=notes) -def _post_load(mechanism_id: str, notes: str = "") -> TuningCapability: - return TuningCapability(rendered_via="post_load", mechanism_id=mechanism_id, notes=notes) +def _post_load(mechanism_id: str, notes: str = "", requires_sorted_ingestion: bool = False) -> TuningCapability: + return TuningCapability( + rendered_via="post_load", + mechanism_id=mechanism_id, + notes=notes, + requires_sorted_ingestion=requires_sorted_ingestion, + ) def _session(mechanism_id: str, notes: str = "") -> TuningCapability: @@ -40,12 +46,26 @@ def _none(mechanism_id: str, notes: str = "") -> TuningCapability: _CONSTRAINT_NOTE = "Rendered as an inline column/table constraint clause at CREATE TABLE time." -def _constraint_entries() -> dict[TuningType, TuningCapability]: +def _pk_fk_entries() -> dict[TuningType, TuningCapability]: return { _T.PRIMARY_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.UNIQUE_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.CHECK_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + } + + +_UNRENDERED_UNIQUE_CHECK_NOTE = ( + "Benchmark DDL never carries UNIQUE/CHECK clauses from tuning flags: no in-tree benchmark " + "consumes those flags (pinned tree-wide by " + "tests/unit/core/tuning/test_capability_registry_parity.py " + "test_unique_check_rendering_is_vacuous_in_tree), so requesting them renders nothing at execution." +) + + +def _no_unique_check_entries(mechanism_id: str, notes: str = "") -> dict[TuningType, TuningCapability]: + detail = f"{notes} {_UNRENDERED_UNIQUE_CHECK_NOTE}".strip() + return { + _T.UNIQUE_CONSTRAINTS: _none(mechanism_id, detail), + _T.CHECK_CONSTRAINTS: _none(mechanism_id, detail), } @@ -56,8 +76,8 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "chdb": "clickhouse", "spark": "databricks", "delta": "databricks", - "fabric-warehouse": "databricks", "synapse": "azure-synapse", + "starburst": "trino", } @@ -68,21 +88,25 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "CTAS reorder (CREATE OR REPLACE TABLE ... AS SELECT * FROM ... ORDER BY ...) run after data " "load; DuckDB has no inline CREATE TABLE ORDER BY syntax. Consumed via " "core.tuning.generators.duckdb.DuckDBDDLGenerator.generate_ctas_ddl by both dry-run preview " - "and real execution as of the w2 duckdb migration.", + "and real execution since the DuckDB DDL migration.", ), _T.PARTITIONING: _none( "duckdb_copy_to_hint_only", "DuckDBDDLGenerator.generate_tuning_clauses logs a COPY TO Hive-partitioning hint for " "partition columns but no BenchBox code path applies it to the physical benchmark schema. " - "Documented gap, not migrated this round (compatible per the legacy map; no rendering yet).", + "Documented gap: compatible per the legacy map, but nothing renders it yet.", + ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "duckdb_no_unique_check_rendering", + "DuckDBAdapter.apply_constraint_configuration only logs UNIQUE/CHECK flag state (duckdb.py).", ), - **_constraint_entries(), }, "clickhouse": { _T.PARTITIONING: _ddl( "clickhouse_ddl_generator:PARTITION_BY", "core.tuning.generators.clickhouse.ClickHouseDDLGenerator renders PARTITION BY at CREATE " - "TABLE time. Consumed by dry-run preview and (as of the w2 clickhouse migration) real " + "TABLE time. Consumed by dry-run preview and (since the ClickHouse DDL migration) real " "execution in ClickHouseWorkloadMixin._optimize_table_definition.", ), _T.SORTING: _ddl( @@ -111,18 +135,30 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Compatible per the legacy map (downgraded to a warning regardless, as a constraint type); " "ClickHouse has no UNIQUE constraint syntax and no adapter code renders one.", ), + _T.FOREIGN_KEYS: _none( + "clickhouse_drops_foreign_keys", + "ClickHouseWorkloadMixin._optimize_table_definition preserves FOREIGN KEY clauses it " + "cannot execute (clickhouse/workload.py); the engine has no FOREIGN KEY syntax, so a " + "requested foreign key fails at execution.", + ), + _T.CHECK_CONSTRAINTS: _none( + "clickhouse_no_unique_check_rendering", + "The ClickHouse workload column parser terminates column types at UNIQUE/CHECK keywords " + "without rendering them (clickhouse/workload.py).", + ), _T.MATERIALIZED_VIEWS: _none( "unimplemented", "Compatible per the legacy map; no adapter code creates a materialized view.", ), }, "databricks": { - _T.PARTITIONING: _ddl( - "delta_partitioned_by", - "Rendered as Delta PARTITIONED BY at CREATE TABLE time via the cloud_spark DDL mixin " - "(third renderer per ADR-3; not migrated to core.tuning.generators this round -- Databricks " - "migration is explicitly out of scope, owned by the blocked databricks-liquid-clustering " - "TODO).", + _T.PARTITIONING: _none( + "databricks_partitioning_preview_only", + "DatabricksAdapter.generate_tuning_clause builds PARTITIONED BY from tuned columns " + "(databricks/adapter.py:3005-3048) but has no production call site tree-wide, so no " + "PARTITIONED BY is rendered at preview or execution. Real execution builds Delta DDL in " + "_convert_to_delta_table (databricks/adapter.py:2672-2729), which never adds partitioning " + "columns, and apply_table_tunings only logs them via _log_partitioning_and_sorting.", ), _T.CLUSTERING: _post_load( "databricks_z_order_or_liquid", @@ -155,7 +191,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "unimplemented", "Compatible per the legacy map; no adapter code creates a materialized view.", ), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "databricks_no_unique_check_rendering", + "DatabricksAdapter executes tuning only through ZORDER, OPTIMIZE, Liquid and CLUSTER " + "statements (databricks/adapter.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Declared keys are informational and unenforced on Databricks. The adapter never emits RELY " @@ -182,7 +223,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "partition clause. See CLUSTERING entry for the full mechanism/evidence.", ), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "snowflake_no_unique_check_rendering", + "SnowflakeAdapter tuning executes ALTER TABLE ... CLUSTER BY and constraint-bearing " + "CREATE TABLE carrying PK/FK only (snowflake.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Primary keys on standard Snowflake tables are informational and unenforced (Snowflake table " @@ -219,7 +265,10 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "tests/unit/platforms/test_bigquery_adapter.py.", ), _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.CHECK_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "bigquery_no_unique_check_rendering", + "BigQuery DDL emits PRIMARY KEY (...) NOT ENFORCED plus FOREIGN KEY clauses only (bigquery.py).", + ), }, "redshift": { _T.DISTRIBUTION: _none( @@ -245,6 +294,7 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "apply_table_tunings only logs the sort-key mismatch (redshift.py:2599-2605) because it cannot " "rewrite the key without table recreation. With sorted ingestion off, _build_ctas_sort_sql " "returns None and nothing is rendered.", + requires_sorted_ingestion=True, ), _T.PARTITIONING: _none( "redshift_no_partition_rendering", @@ -253,7 +303,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "(redshift.py:2629-2636). Nothing is rendered at execution.", ), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "redshift_no_unique_check_rendering", + "RedshiftAdapter tuning executes CTAS-sort, VACUUM and maintenance statements only (redshift.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Redshift documents primary and foreign keys as informational and unenforced, but states that " @@ -264,7 +318,14 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "matter here because the planner trusts them.", ), }, - "sqlite": _constraint_entries(), + "sqlite": { + **_pk_fk_entries(), + **_no_unique_check_entries( + "sqlite_no_unique_check_rendering", + "The shared tuning-aware DDL path forwards tuning_config; SQLite execution builds no " + "UNIQUE/CHECK clause from the flags.", + ), + }, "postgresql": { _T.PARTITIONING: _none( "postgresql_ddl_generator:preview_only", @@ -285,7 +346,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: ), _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "postgresql_no_unique_check_rendering", + "PostgreSQLAdapter renders constraint-bearing CREATE TABLE from PK/FK flags only (postgresql.py).", + ), }, "mysql": { _T.PARTITIONING: _none( @@ -298,7 +363,200 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "execution. Corrects the prior false " "_ddl(adapter_mixin:MySQLAdapter.generate_tuning_clause) claim.", ), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "mysql_no_unique_check_rendering", + "The mysql-wire tuning-aware DDL path renders PK/FK from flags only (base/mysql_wire.py).", + ), + }, + "singlestore": { + _T.PARTITIONING: _none( + "singlestore_no_partition_rendering", + "SingleStoreAdapter inherits NoOpTableTuningMixin (singlestore.py:75): " + "generate_tuning_clause returns '' and apply_table_tunings is a no-op. Tuned " + "constraints flow through the shared mysql-wire tuning-aware DDL path, as on MySQL.", + ), + _T.PRIMARY_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + _T.FOREIGN_KEYS: _none( + "singlestore_strips_foreign_keys", + "SingleStore strips FOREIGN KEY clauses from every CREATE TABLE at execution " + "(singlestore.py singlestore_strip_foreign_keys at 379-380, wired by the DDL_OPTIMIZE " + "strip_foreign_keys rule in sql_compat/rules/ddl_optimize/singlestore_ddl_rewrites.py:6-16). " + "Requested foreign keys never reach the engine (SingleStore error 2752).", + ), + **_no_unique_check_entries( + "singlestore_no_unique_check_rendering", + "SingleStoreAdapter inherits the mysql-wire tuning-aware DDL path rendering PK from " + "flags only; FOREIGN KEY clauses are stripped (singlestore.py).", + ), + }, + "cedardb": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "CedarDBAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "cedardb_no_unique_check_rendering", + "CedarDBAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (cedardb.py).", + ), + }, + "citus": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "CitusAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "citus_no_unique_check_rendering", + "CitusAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (citus.py).", + ), + }, + "paradedb": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "ParadeDBAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "paradedb_no_unique_check_rendering", + "ParadeDBAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (paradedb.py).", + ), + }, + "fabric-warehouse": { + _T.PARTITIONING: _none( + "delta_ddl_generator:preview_only_tsql_ignores", + "The Delta preview generator computes partitioning, but FabricWarehouseAdapter executes " + "T-SQL (FABRIC_DIALECT='tsql'): the shared builder's tsql output ignores tuned partition " + "columns, generate_tuning_clause returns '' and _optimize_table_definition only adds the " + "schema (fabric_warehouse.py:857-883). Previously aliased to databricks (rendered _ddl) " + "-- that claim was false for the warehouse execution path.", + ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "fabric_warehouse_no_unique_check_rendering", + "FabricWarehouseAdapter executes T-SQL carrying PK/FK only (fabric_warehouse.py).", + ), + }, + "databend": { + _T.CLUSTERING: _post_load( + "databend_alter_cluster_by", + "DatabendAdapter.apply_table_tunings executes ALTER TABLE ... CLUSTER BY from tuned " + "clustering columns (databend/adapter.py:632-651).", + ), + _T.PARTITIONING: _none( + "unimplemented", + "DatabendAdapter.generate_tuning_clause only handles CLUSTERING; nothing renders " + "partitioning at execution.", + ), + _T.SORTING: _none( + "unimplemented", + "Same gap as PARTITIONING: only CLUSTERING is rendered at execution.", + ), + _T.DISTRIBUTION: _none( + "unimplemented", + "Same gap as PARTITIONING: only CLUSTERING is rendered at execution.", + ), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.PRIMARY_KEYS: _none( + "databend_strips_primary_keys", + "DatabendAdapter strips PRIMARY KEY clauses from every executed CREATE TABLE via " + "inline regex (databend/adapter.py:678), alongside the foreign-key strip. Tuned keys " + "reach the DDL builder but never reach the engine.", + ), + _T.FOREIGN_KEYS: _none( + "databend_strips_foreign_keys", + "DatabendAdapter strips foreign keys from every CREATE TABLE (databend/adapter.py:680); " + "remaining constraint configuration is informational log-only.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "databend_no_unique_check_rendering", + "DatabendAdapter strips PK/FK and only renders CLUSTERING; UNIQUE/CHECK clauses are " + "never produced or consumed from tuning.", + ), + _T.CHECK_CONSTRAINTS: _none( + "databend_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: never produced or consumed from tuning.", + ), + }, + "datafusion": { + _T.PRIMARY_KEYS: _none( + "datafusion_unconstrained_schema", + "DataFusionAdapter builds its schema explicitly without constraints even when enabled " + "(datafusion.py:558-564).", + ), + _T.FOREIGN_KEYS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + _T.CHECK_CONSTRAINTS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + }, + "motherduck": { + _T.SORTING: _post_load( + "duckdb_ctas_sort", + "MotherDuckAdapter runs the shared CTAS-sort path from tuned sorting columns during load " + "(motherduck.py:337, via base/sorted_ingestion.py).", + ), + _T.PARTITIONING: _none( + "motherduck_external_views_unpartitioned", + "MotherDuckAdapter.create_schema executes benchmark DDL verbatim without tuning_config " + "(motherduck.py:307-318) and loads through external views; tuned partitioning columns " + "reshape nothing.", + ), + _T.CLUSTERING: _none( + "motherduck_no_cluster_rendering", + "Same execution gap as PARTITIONING: no post-create CLUSTER step consumes tuning.", + ), + _T.DISTRIBUTION: _none( + "motherduck_not_applicable_single_writer", + "MotherDuck is single-writer cloud DuckDB; nothing renders distribution columns.", + ), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.PRIMARY_KEYS: _none( + "motherduck_drops_tuning_config", + "create_schema drops tuning_config and apply_constraint_configuration is a pass-through " + "no-op (motherduck.py:354-355).", + ), + _T.FOREIGN_KEYS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), + _T.CHECK_CONSTRAINTS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), }, "starrocks": { _T.PARTITIONING: _ddl( @@ -326,16 +584,60 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "tables -- byte-identical to the previous bespoke injection (BUCKETS 8). Preview and " "execution render through the same generator.", ), + _T.PRIMARY_KEYS: _ddl( + _INLINE_CONSTRAINT, + "Tuned primary keys choose the StarRocks key model: the workload rewrite reads the " + "PK columns from the tuned DDL and emits PRIMARY KEY when they lead the table, else " + "DUPLICATE KEY (starrocks/workload.py:122-152).", + ), + _T.FOREIGN_KEYS: _none( + "starrocks_strips_foreign_keys", + "StarRocksWorkloadMixin strips foreign keys from every CREATE TABLE (starrocks/workload.py:120).", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "starrocks_no_unique_check_rendering", + "StarRocksWorkloadMixin._optimize_table_definition handles PARTITION/DISTRIBUTION/" + "ORDER BY plus PK-model selection and foreign-key stripping only (starrocks/workload.py:" + "109-198); nothing renders, strips, or validates UNIQUE/CHECK clauses.", + ), + _T.CHECK_CONSTRAINTS: _none( + "starrocks_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: no UNIQUE/CHECK handling in the workload rewrite.", + ), }, "doris": { - _T.PARTITIONING: _ddl( - "doris_ddl_generator", - "core.tuning.generators.doris.DorisDDLGenerator exists and is reachable via " - "get_ddl_generator('doris') for dry-run preview. Adapter execution-path parity was not " - "audited this round (Doris is not in this TODO's migration order).", + _T.PARTITIONING: _none( + "doris_ddl_generator:preview_only", + "core.tuning.generators.doris.DorisDDLGenerator computes PARTITION BY from tuned columns " + "for dry-run preview, but DorisAdapter inherits NoOpTableTuningMixin (doris.py:187) and " + "renders nothing at execution. Corrects the prior unaudited _ddl claim.", + ), + _T.SORTING: _none( + "doris_ddl_generator:preview_only", + "Same execution gap as PARTITIONING: preview only, NoOp at execution.", + ), + _T.DISTRIBUTION: _none( + "doris_ddl_generator:preview_only", + "Same execution gap as PARTITIONING: preview only, NoOp at execution.", + ), + _T.PRIMARY_KEYS: _none( + "doris_strips_primary_keys", + "The shared tuning-aware DDL path would emit tuned primary keys, but DorisAdapter " + "strips PRIMARY KEY clauses from every CREATE TABLE (doris.py:803-805).", + ), + _T.FOREIGN_KEYS: _none( + "doris_strips_foreign_keys", + "DorisAdapter strips foreign keys from every CREATE TABLE (doris.py:802).", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "doris_no_unique_check_rendering", + "DorisAdapter renders nothing at execution (NoOpTableTuningMixin) and strips PK/FK; " + "UNIQUE/CHECK clauses are never produced or consumed from tuning.", + ), + _T.CHECK_CONSTRAINTS: _none( + "doris_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: never produced or consumed from tuning.", ), - _T.SORTING: _ddl("doris_ddl_generator", "See PARTITIONING entry."), - _T.DISTRIBUTION: _ddl("doris_ddl_generator", "See PARTITIONING entry."), }, "trino": { _T.PARTITIONING: _none( @@ -369,6 +671,16 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Trino has no separate clustering clause; the generator folds clustering columns into " "sorted_by (trino.py:202-210, see SORTING entry), and the same execution-path gap applies.", ), + _T.PRIMARY_KEYS: _none( + "trino_strips_primary_keys", + "PrestoTrinoAdapterBase._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (presto_trino_adapter_base.py).", + ), + _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "trino_no_unique_check_rendering", + "TrinoAdapter renders no UNIQUE/CHECK clause at execution (trino.py).", + ), }, "presto": { _T.PARTITIONING: _none( @@ -397,6 +709,16 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Presto has no separate clustering clause; folded into sorted_by upstream (see SORTING " "entry), and the same execution-path gap applies.", ), + _T.PRIMARY_KEYS: _none( + "presto_strips_primary_keys", + "PrestoTrinoAdapterBase._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (presto_trino_adapter_base.py).", + ), + _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "presto_no_unique_check_rendering", + "PrestoAdapter renders no UNIQUE/CHECK clause at execution (presto.py).", + ), }, "athena": { _T.PARTITIONING: _none( @@ -432,6 +754,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Athena has no separate clustering clause; folded into sorted_by upstream, which is dropped " "(see SORTING entry).", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "athena_no_unique_check_rendering", + "AthenaAdapter._convert_to_external_table builds CREATE EXTERNAL TABLE without UNIQUE/CHECK " + "clauses (athena.py).", + ), }, "firebolt": { _T.DISTRIBUTION: _none( @@ -462,6 +790,20 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "clustering hints (firebolt.py:135-142) instead of computing a clause, even for dry-run " "preview.", ), + _T.PRIMARY_KEYS: _none( + "firebolt_strips_primary_keys", + "FireboltAdapter._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (firebolt.py).", + ), + _T.FOREIGN_KEYS: _none( + "firebolt_strips_foreign_keys", + "FireboltAdapter._optimize_table_definition strips FOREIGN KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_foreign_keys (firebolt.py).", + ), + **_no_unique_check_entries( + "firebolt_no_unique_check_rendering", + "FireboltAdapter renders no UNIQUE/CHECK clause at execution (firebolt.py).", + ), }, "azure-synapse": { _T.DISTRIBUTION: _none( @@ -496,6 +838,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "clustering hints (azure_synapse.py:183-189) instead of computing a clause, even for dry-run " "preview.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "azure_synapse_no_unique_check_rendering", + "AzureSynapseAdapter real DDL carries PK/FK with the engine-default DISTRIBUTION only (azure_synapse.py).", + ), }, "timescaledb": { _T.PARTITIONING: _none( @@ -531,6 +878,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Merged with DISTRIBUTION into compress_segmentby by the generator (timescaledb.py:177-186); " "same real-execution gap -- compress_segmentby is hardcoded empty regardless of tuning.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "timescaledb_no_unique_check_rendering", + "TimescaleDBAdapter hypertable and compression paths consume no UNIQUE/CHECK tuning (timescaledb.py).", + ), }, "questdb": { _T.PARTITIONING: _none( @@ -557,6 +909,19 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "QuestDB is single-node; the generator only warning-logs distribution hints and ignores the " "columns (questdb.py:193-199) instead of computing a clause, even for dry-run preview.", ), + _T.PRIMARY_KEYS: _none( + "questdb_strips_primary_keys", + "QuestDBAdapter.create_schema strips PRIMARY KEY clauses via _strip_pk_constraints (questdb.py).", + ), + _T.FOREIGN_KEYS: _none( + "questdb_strips_foreign_keys", + "QuestDBAdapter.create_schema strips FOREIGN KEY clauses via ddl_helpers strip_foreign_keys (questdb.py).", + ), + **_no_unique_check_entries( + "questdb_no_unique_check_rendering", + "QuestDB schema enhancement maps hardcoded timestamp/partition columns only and builds no " + "UNIQUE/CHECK clause (questdb.py).", + ), }, "pg-duckdb": { _T.PARTITIONING: _none( @@ -594,6 +959,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "post_create_statements via the inherited no-op apply_table_tunings/generate_tuning_clause " "path.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "pg_duckdb_no_unique_check_rendering", + "PgDuckDBAdapter inherits PostgreSQLAdapter execution carrying PK/FK only (pg_duckdb.py).", + ), }, "pg-mooncake": { _T.PARTITIONING: _none( @@ -617,6 +987,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Same as PARTITIONING: the generator always returns empty clauses, and " "supports_tuning_type() is unconditionally False.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "pg_mooncake_no_unique_check_rendering", + "PgMooncakeAdapter inherits PostgreSQLAdapter execution carrying PK/FK only (pg_mooncake.py).", + ), }, } @@ -641,7 +1016,7 @@ def interface_compatibility_map() -> dict[str, frozenset[TuningType]]: def resolve_platform_key(platform: str) -> str: - platform_key = platform.lower().replace("_", "-") + platform_key = platform.split(":", 1)[0].strip().lower().replace("_", "-") return PLATFORM_ALIASES.get(platform_key, platform_key) @@ -653,6 +1028,37 @@ def get_capability(platform: str, tuning_type: TuningType) -> TuningCapability | return entries.get(tuning_type) +def is_not_rendered_at_execution(capability: TuningCapability | None) -> bool: + if capability is None: + return False + return capability.rendered_via == "none" or ":preview_only" in capability.mechanism_id + + +def not_rendered_reason(capability: TuningCapability) -> str: + mechanism_id = capability.mechanism_id + if "strips" in mechanism_id: + reason = "the platform adapter removes it from CREATE TABLE statements" + elif "unconstrained" in mechanism_id: + reason = "the schema is created without constraints" + elif "no_unique_check_rendering" in mechanism_id: + reason = "table DDL never carries this constraint" + elif "preview" in mechanism_id or mechanism_id.endswith("hint_only"): + reason = "it is generated only for dry-run previews and is not applied when tables are created" + elif "log_only" in mechanism_id: + reason = "the adapter only logs it and does not apply it" + else: + reason = "BenchBox does not apply it on this platform" + return f"{reason} ({mechanism_id})" + + +def unapplied_reason(capability: TuningCapability, sorted_ingestion_mode: str | None) -> str | None: + if is_not_rendered_at_execution(capability): + return not_rendered_reason(capability) + if capability.requires_sorted_ingestion and sorted_ingestion_mode in (None, "off"): + return "it runs only when sorted ingestion is enabled; set sorted_ingestion_mode to auto or force" + return None + + def known_registry_platforms() -> frozenset[str]: return frozenset(PLATFORM_TUNING_CAPABILITIES) diff --git a/benchbox/core/tuning/interface.py b/benchbox/core/tuning/interface.py index 4c18544ef4..741d706dce 100644 --- a/benchbox/core/tuning/interface.py +++ b/benchbox/core/tuning/interface.py @@ -1158,6 +1158,13 @@ def disable_platform_optimization(self, optimization_type: TuningType) -> None: elif optimization_type == TuningType.MATERIALIZED_VIEWS: self.platform_optimizations.materialized_views_enabled = False + def disable_tuning_type(self, tuning_type: TuningType) -> None: + for attr, checked_type in self._CONSTRAINT_CHECKS: + if checked_type == tuning_type: + getattr(self, attr).enabled = False + return + self.disable_platform_optimization(tuning_type) + _CONSTRAINT_CHECKS = [ ("primary_keys", TuningType.PRIMARY_KEYS), ("foreign_keys", TuningType.FOREIGN_KEYS), @@ -1208,13 +1215,24 @@ def validate_for_platform_detailed( errors: list[str] = [] warnings: list[str] = [] enabled_types = self.get_enabled_tuning_types() - platform_known = TuningType.is_known_platform(platform) + + from benchbox.core.tuning.capability_registry import ( + get_capability, + resolve_platform_key, + unapplied_reason, + ) for tuning_type in enabled_types: + capability = get_capability(platform, tuning_type) + if capability is not None: + reason = unapplied_reason(capability, self.platform_optimizations.sorted_ingestion_mode) + if reason is not None: + errors.append(f"requested {tuning_type.value} is not applied at execution on {platform}: {reason}") + continue if tuning_type.is_compatible_with_platform(platform): continue message = f"Tuning type '{tuning_type.value}' is not supported by platform '{platform}'" - if tuning_type in _CONSTRAINT_TUNING_TYPES or not platform_known: + if tuning_type in _CONSTRAINT_TUNING_TYPES: warnings.append(message) else: errors.append(message) @@ -1225,13 +1243,12 @@ def validate_for_platform_detailed( f"'spark' session-config map is not applied by platform '{platform}'; " "run on spark, velox, or lakesail to apply it" ) - validator = _get_effective_layout_validator(platform_key) + resolved_key = resolve_platform_key(platform_key) + validator = _get_effective_layout_validator(resolved_key) if validator is not None: errors.extend(validator(self)) if schema_primary_keys: - from benchbox.core.tuning.capability_registry import resolve_platform_key - - if resolve_platform_key(platform_key) == "clickhouse": + if resolved_key == "clickhouse": errors.extend(_clickhouse_primary_key_prefix_errors(self, schema_primary_keys)) return errors, warnings diff --git a/benchbox/core/tuning/reconciliation.py b/benchbox/core/tuning/reconciliation.py index bd2d55c98a..c12ede442b 100644 --- a/benchbox/core/tuning/reconciliation.py +++ b/benchbox/core/tuning/reconciliation.py @@ -13,7 +13,12 @@ AppliedStatement, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import TuningCapability, get_capability +from benchbox.core.tuning.capability_registry import ( + TuningCapability, + get_capability, + is_not_rendered_at_execution, + not_rendered_reason, +) from benchbox.core.tuning.interface import TuningType, UnifiedTuningConfiguration from benchbox.core.tuning.introspection import normalize_identifier, statement_index_columns, statement_table @@ -234,8 +239,8 @@ def _realized_in_ddl(capability: TuningCapability | None, statement: AppliedStat def _drop_reason(capability: TuningCapability | None) -> str: if capability is None: return NOT_RENDERED_REASON - if capability.rendered_via == "none" or ":preview_only" in capability.mechanism_id: - return capability.notes or f"not rendered at execution ({capability.mechanism_id})" + if is_not_rendered_at_execution(capability): + return f"not rendered at execution: {not_rendered_reason(capability)}" return NOT_RENDERED_REASON diff --git a/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml b/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml index a9020c2e95..e544227573 100644 --- a/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml b/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml index d161385e6a..cdc3c2b886 100644 --- a/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml b/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml index 0fb33bc1e0..65e36458ae 100644 --- a/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml index 894bca10b8..22f5d468c9 100644 --- a/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml index 53328c6589..65781c2969 100644 --- a/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml index 6c6ae3c6aa..c0cccc78ff 100644 --- a/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml b/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml index 882d9e1996..db13f0084c 100644 --- a/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml +++ b/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml b/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml index 9a9a875028..c50c549007 100644 --- a/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml +++ b/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/examples/tunings/duckdb/joinorder_tuned.yaml b/examples/tunings/duckdb/joinorder_tuned.yaml index a9020c2e95..e544227573 100644 --- a/examples/tunings/duckdb/joinorder_tuned.yaml +++ b/examples/tunings/duckdb/joinorder_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/read_primitives_tuned.yaml b/examples/tunings/duckdb/read_primitives_tuned.yaml index d161385e6a..cdc3c2b886 100644 --- a/examples/tunings/duckdb/read_primitives_tuned.yaml +++ b/examples/tunings/duckdb/read_primitives_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/ssb_tuned.yaml b/examples/tunings/duckdb/ssb_tuned.yaml index 0fb33bc1e0..65e36458ae 100644 --- a/examples/tunings/duckdb/ssb_tuned.yaml +++ b/examples/tunings/duckdb/ssb_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpcds_tuned.yaml b/examples/tunings/duckdb/tpcds_tuned.yaml index 894bca10b8..22f5d468c9 100644 --- a/examples/tunings/duckdb/tpcds_tuned.yaml +++ b/examples/tunings/duckdb/tpcds_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpch_tuned.yaml b/examples/tunings/duckdb/tpch_tuned.yaml index 53328c6589..65781c2969 100644 --- a/examples/tunings/duckdb/tpch_tuned.yaml +++ b/examples/tunings/duckdb/tpch_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpchavoc_tuned.yaml b/examples/tunings/duckdb/tpchavoc_tuned.yaml index 6c6ae3c6aa..c0cccc78ff 100644 --- a/examples/tunings/duckdb/tpchavoc_tuned.yaml +++ b/examples/tunings/duckdb/tpchavoc_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/snowflake/tpcds_tuned.yaml b/examples/tunings/snowflake/tpcds_tuned.yaml index 882d9e1996..db13f0084c 100644 --- a/examples/tunings/snowflake/tpcds_tuned.yaml +++ b/examples/tunings/snowflake/tpcds_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/examples/tunings/snowflake/tpch_tuned.yaml b/examples/tunings/snowflake/tpch_tuned.yaml index 9a9a875028..c50c549007 100644 --- a/examples/tunings/snowflake/tpch_tuned.yaml +++ b/examples/tunings/snowflake/tpch_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/scripts/generate_cloud_tpc_templates.py b/scripts/generate_cloud_tpc_templates.py index 3bafa8a023..89607e8f1b 100644 --- a/scripts/generate_cloud_tpc_templates.py +++ b/scripts/generate_cloud_tpc_templates.py @@ -140,11 +140,11 @@ def snowflake_table_candidates(benchmark: str, table: str, candidates: list) -> on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/tests/unit/cli/commands/test_run_tuning_mode_guard.py b/tests/unit/cli/commands/test_run_tuning_mode_guard.py index 5dda4d6845..8b1c76da86 100644 --- a/tests/unit/cli/commands/test_run_tuning_mode_guard.py +++ b/tests/unit/cli/commands/test_run_tuning_mode_guard.py @@ -52,8 +52,8 @@ def test_dataframe_run_loads_without_the_sql_guard(tmp_path): ("mode", "source"), [(TuningMode.TUNED, TuningSource.FALLBACK), (TuningMode.AUTO, TuningSource.SMART_DEFAULTS)], ) -@pytest.mark.parametrize(("platform", "check_enabled"), [("duckdb", False), ("postgresql", True)]) -def test_fallback_and_auto_request_check_constraints_except_on_duckdb(mode, source, platform, check_enabled): +@pytest.mark.parametrize("platform", ["duckdb", "postgresql"]) +def test_fallback_and_auto_do_not_request_unrendered_check_constraints(mode, source, platform): s = SimpleNamespace( tuning_resolution=SimpleNamespace(config_file=None, source=source, mode=mode), resolved_mode="sql", @@ -67,5 +67,6 @@ def test_fallback_and_auto_request_check_constraints_except_on_duckdb(mode, sour _load_unified_tuning_config(s) - assert s.loaded_unified_config.check_constraints.enabled is check_enabled + assert s.loaded_unified_config.check_constraints.enabled is False + assert s.loaded_unified_config.unique_constraints.enabled is False assert s.loaded_unified_config.primary_keys.enabled is True diff --git a/tests/unit/cli/commands/test_tuning_group_cli.py b/tests/unit/cli/commands/test_tuning_group_cli.py index 8a2b5ed5ca..63cf6da6f2 100644 --- a/tests/unit/cli/commands/test_tuning_group_cli.py +++ b/tests/unit/cli/commands/test_tuning_group_cli.py @@ -315,7 +315,7 @@ def test_shipped_duckdb_template_validates(self): assert result.exit_code == 0, output assert "Configuration is valid for duckdb" in output - assert "Enabled tuning types: 4" in output + assert "Enabled tuning types: 3" in output def test_sql_validation_goes_through_validate_for_platform_detailed(self): template = REPO_ROOT / "examples/tunings/duckdb/tpch_tuned.yaml" @@ -340,7 +340,7 @@ def test_incompatible_tuning_type_is_an_error(self): assert "Tuning type 'sorting' is not supported by platform 'bigquery'" in output assert "Configuration has errors" in output - def test_registry_unrendered_type_is_reported_as_warning(self, tmp_path): + def test_registry_unrendered_type_is_an_error(self, tmp_path): template = tmp_path / "databricks_distribution.yaml" template.write_text( "primary_keys:\n" @@ -363,9 +363,9 @@ def test_registry_unrendered_type_is_reported_as_warning(self, tmp_path): result, output = _run_validate([str(template), "--platform", "databricks"]) - assert result.exit_code == 0, output - assert "'distribution' is accepted by platform 'databricks' but not rendered yet" in output - assert "Configuration is valid but has warnings" in output + assert result.exit_code != 0 + assert "requested distribution is not applied at execution on databricks" in output + assert "Configuration has errors" in output def test_platform_alias_resolves_to_registry_key(self): template = REPO_ROOT / "examples/tunings/duckdb/tpch_tuned.yaml" @@ -409,12 +409,22 @@ def test_empty_file_is_rejected(self, tmp_path): class TestTuningValidateMode: def test_sql_datafusion_file_validates_with_mode_sql(self, tmp_path): target = tmp_path / "datafusion_tuned.yaml" - target.write_text("table_tunings: {}\n") + target.write_text( + "primary_keys:\n" + " enabled: false\n" + "foreign_keys:\n" + " enabled: false\n" + "unique_constraints:\n" + " enabled: false\n" + "check_constraints:\n" + " enabled: false\n" + "table_tunings: {}\n" + ) result, output = _run_validate([str(target), "--platform", "datafusion", "--mode", "sql"]) assert result.exit_code == 0, output - assert "Enabled tuning types: 4" in output + assert "Enabled tuning types: 0" in output def test_dataframe_file_rejected_with_mode_sql(self): template = REPO_ROOT / "examples/tunings/dataframe/polars_optimized.yaml" diff --git a/tests/unit/cli/test_config_coverage.py b/tests/unit/cli/test_config_coverage.py index 2599f37e02..9c676ac1a2 100644 --- a/tests/unit/cli/test_config_coverage.py +++ b/tests/unit/cli/test_config_coverage.py @@ -136,11 +136,19 @@ def test_tuning_config_parse_save_and_validate(monkeypatch: pytest.MonkeyPatch, u = cfg.UnifiedTuningConfiguration() u.primary_keys.enabled = True u.foreign_keys.enabled = True + u.unique_constraints.enabled = False + u.check_constraints.enabled = False unified = tmp_path / "unified.yaml" mgr.save_unified_tuning_config(u, unified, format="yaml") loaded = mgr.load_unified_tuning_config(unified, platform="duckdb") assert loaded.primary_keys.enabled is True + bare = cfg.UnifiedTuningConfiguration() + bare_path = tmp_path / "bare.yaml" + mgr.save_unified_tuning_config(bare, bare_path, format="yaml") + with pytest.raises(ValueError): + mgr.load_unified_tuning_config(bare_path, platform="duckdb") + def test_show_config_prints_path_and_yaml(tmp_path: Path) -> None: mgr = cfg.ConfigManager(config_path=tmp_path / "config.yaml") diff --git a/tests/unit/cli/test_tuning_runtime.py b/tests/unit/cli/test_tuning_runtime.py index 6943258492..b23d82d989 100644 --- a/tests/unit/cli/test_tuning_runtime.py +++ b/tests/unit/cli/test_tuning_runtime.py @@ -59,16 +59,41 @@ def test_build_baseline_unified_config_reports_no_clustering_strategy() -> None: assert config.platform_optimizations.liquid_clustering_enabled is False -@pytest.mark.parametrize("platform", ["duckdb", "DuckDB", "duckdb:memory"]) -def test_default_unified_config_disables_check_constraints_on_duckdb(platform: str) -> None: +@pytest.mark.parametrize("platform", ["duckdb", "DuckDB"]) +def test_default_unified_config_disables_unique_and_check_constraints_on_duckdb(platform: str) -> None: config = build_default_unified_config(platform) assert config.check_constraints.enabled is False assert config.primary_keys.enabled is True assert config.foreign_keys.enabled is True - assert config.unique_constraints.enabled is True + assert config.unique_constraints.enabled is False -@pytest.mark.parametrize("platform", [None, "motherduck", "postgresql", "snowflake"]) -def test_default_unified_config_keeps_check_constraints_elsewhere(platform: str | None) -> None: - assert build_default_unified_config(platform).to_dict() == UnifiedTuningConfiguration().to_dict() +def test_default_unified_config_shares_base_rules_on_duckdb_memory_variant() -> None: + config = build_default_unified_config("duckdb:memory") + + assert config.check_constraints.enabled is False + assert config.unique_constraints.enabled is False + + +def test_default_unified_config_without_platform_keeps_all_constraints() -> None: + assert build_default_unified_config(None).to_dict() == UnifiedTuningConfiguration().to_dict() + + +@pytest.mark.parametrize("platform", ["postgresql", "snowflake"]) +def test_default_unified_config_disables_unrendered_unique_and_check(platform: str) -> None: + config = build_default_unified_config(platform) + + assert config.primary_keys.enabled is True + assert config.foreign_keys.enabled is True + assert config.unique_constraints.enabled is False + assert config.check_constraints.enabled is False + + +def test_default_unified_config_disables_unrendered_constraints_on_motherduck() -> None: + config = build_default_unified_config("motherduck") + + assert config.primary_keys.enabled is False + assert config.foreign_keys.enabled is False + assert config.unique_constraints.enabled is False + assert config.check_constraints.enabled is False diff --git a/tests/unit/core/tuning/test_capability_registry_parity.py b/tests/unit/core/tuning/test_capability_registry_parity.py new file mode 100644 index 0000000000..4eceb0fedd --- /dev/null +++ b/tests/unit/core/tuning/test_capability_registry_parity.py @@ -0,0 +1,767 @@ +from __future__ import annotations + +import inspect +import re + +import pytest +import yaml + +from benchbox.core.tuning.capability_registry import ( + PLATFORM_ALIASES, + PLATFORM_TUNING_CAPABILITIES, + get_capability, + is_not_rendered_at_execution, + known_registry_platforms, + not_rendered_reason, + resolve_platform_key, +) +from benchbox.core.tuning.interface import TableTuning, TuningColumn, TuningType, UnifiedTuningConfiguration + +pytestmark = [ + pytest.mark.unit, + pytest.mark.fast, +] + + +def _partitioned_config() -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.table_tunings["orders"] = TableTuning( + table_name="orders", + partitioning=[TuningColumn(name="o_orderdate", type="DATE", order=1)], + ) + return config + + +def _distributed_config() -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.table_tunings["lineitem"] = TableTuning( + table_name="lineitem", + distribution=[TuningColumn(name="l_orderkey", type="INTEGER", order=1)], + ) + return config + + +class TestRequestedButNotRenderedIsAnError: + def test_partitioning_on_postgresql_errors(self) -> None: + errors, _warnings = _partitioned_config().validate_for_platform_detailed("postgresql") + assert any("requested partitioning is not applied at execution on postgresql" in e for e in errors) + + def test_partitioning_on_postgres_family_errors(self) -> None: + for platform in ("cedardb", "citus", "paradedb", "pg-duckdb"): + errors, _warnings = _partitioned_config().validate_for_platform_detailed(platform) + assert any("is not applied at execution" in e for e in errors), platform + + def test_distribution_on_redshift_errors(self) -> None: + errors, _warnings = _distributed_config().validate_for_platform_detailed("redshift") + assert any("requested distribution is not applied at execution on redshift" in e for e in errors) + + def test_layout_on_trino_and_athena_errors(self) -> None: + for platform in ("trino", "athena"): + errors, _warnings = _partitioned_config().validate_for_platform_detailed(platform) + assert any("is not applied at execution" in e for e in errors), platform + + def test_starburst_alias_resolves_to_trino_rules(self) -> None: + assert resolve_platform_key("Starburst") == "trino" + errors, _warnings = _partitioned_config().validate_for_platform_detailed("starburst") + assert any("is not applied at execution" in e for e in errors) + + def test_rendering_registry_entry_does_not_error(self) -> None: + errors, warnings = _distributed_config().validate_for_platform_detailed("starrocks") + assert errors == [] + assert warnings == [] + + def test_doris_layout_and_constraints_error(self) -> None: + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("doris") + assert any("requested primary_keys is not applied at execution" in e for e in errors) + assert any("requested foreign_keys is not applied at execution" in e for e in errors) + + def test_datafusion_constraints_error(self) -> None: + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("datafusion") + assert len(errors) == 4 + + def test_motherduck_sorting_renders_but_constraints_error(self) -> None: + sort_config = UnifiedTuningConfiguration() + sort_config.disable_all_constraints() + sort_config.table_tunings["orders"] = TableTuning( + table_name="orders", + sorting=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + ) + errors, _warnings = sort_config.validate_for_platform_detailed("motherduck") + assert errors == [] + + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("motherduck") + assert any("is not applied at execution" in e for e in errors) + + def test_fabric_warehouse_partitioning_errors(self) -> None: + errors, _warnings = _partitioned_config().validate_for_platform_detailed("fabric-warehouse") + assert any("is not applied at execution" in e for e in errors) + + def test_stripped_constraint_types_error(self) -> None: + for platform in ("doris", "databend", "starrocks"): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.unique_constraints.enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any("requested unique_constraints is not applied at execution" in e for e in errors), platform + + def test_vacuous_unique_check_types_error(self) -> None: + for platform in ( + "duckdb", + "databricks", + "snowflake", + "sqlite", + "postgresql", + "mysql", + "singlestore", + "cedardb", + "citus", + "paradedb", + "fabric-warehouse", + "athena", + "azure-synapse", + "timescaledb", + "redshift", + "firebolt", + "questdb", + "pg-duckdb", + "pg-mooncake", + "trino", + "presto", + ): + for tuning_attr in ("unique_constraints", "check_constraints"): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_bigquery_unique_and_clickhouse_fk_check_error(self) -> None: + for platform, tuning_attr in ( + ("bigquery", "unique_constraints"), + ("clickhouse", "foreign_keys"), + ("clickhouse", "check_constraints"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_dropped_pk_fk_types_error(self) -> None: + for platform, tuning_attr in ( + ("firebolt", "primary_keys"), + ("firebolt", "foreign_keys"), + ("questdb", "primary_keys"), + ("questdb", "foreign_keys"), + ("trino", "primary_keys"), + ("presto", "primary_keys"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_rendered_pk_fk_types_do_not_error(self) -> None: + for platform, tuning_attr in ( + ("pg-duckdb", "primary_keys"), + ("pg-duckdb", "foreign_keys"), + ("pg-mooncake", "primary_keys"), + ("pg-mooncake", "foreign_keys"), + ("trino", "foreign_keys"), + ("presto", "foreign_keys"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], (platform, tuning_attr, errors) + + +_NOT_RENDERED: dict[str, frozenset[str]] = { + "athena": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "azure-synapse": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "bigquery": frozenset( + {"check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "cedardb": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "citus": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "clickhouse": frozenset({"check_constraints", "foreign_keys", "materialized_views", "unique_constraints"}), + "databend": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "materialized_views", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "databricks": frozenset( + { + "bloom_filters", + "check_constraints", + "distribution", + "materialized_views", + "partitioning", + "unique_constraints", + } + ), + "datafusion": frozenset({"check_constraints", "foreign_keys", "primary_keys", "unique_constraints"}), + "doris": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "duckdb": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "fabric-warehouse": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "firebolt": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "motherduck": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "foreign_keys", + "materialized_views", + "partitioning", + "primary_keys", + "unique_constraints", + } + ), + "mysql": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "paradedb": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "pg-duckdb": frozenset({"check_constraints", "clustering", "partitioning", "sorting", "unique_constraints"}), + "pg-mooncake": frozenset({"check_constraints", "clustering", "partitioning", "sorting", "unique_constraints"}), + "postgresql": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "presto": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "questdb": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "redshift": frozenset( + {"check_constraints", "distribution", "materialized_views", "partitioning", "unique_constraints"} + ), + "singlestore": frozenset({"check_constraints", "foreign_keys", "partitioning", "unique_constraints"}), + "snowflake": frozenset({"check_constraints", "materialized_views", "unique_constraints"}), + "sqlite": frozenset({"check_constraints", "unique_constraints"}), + "starrocks": frozenset({"check_constraints", "foreign_keys", "unique_constraints"}), + "timescaledb": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "trino": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), +} +_RENDERED: dict[str, frozenset[str]] = { + "athena": frozenset({"foreign_keys", "primary_keys"}), + "azure-synapse": frozenset({"foreign_keys", "primary_keys"}), + "bigquery": frozenset({"foreign_keys", "primary_keys"}), + "cedardb": frozenset({"foreign_keys", "primary_keys"}), + "citus": frozenset({"foreign_keys", "primary_keys"}), + "clickhouse": frozenset({"clustering", "partitioning", "primary_keys", "sorting"}), + "databend": frozenset({"clustering"}), + "databricks": frozenset( + { + "auto_compact", + "auto_optimize", + "clustering", + "foreign_keys", + "liquid_clustering", + "primary_keys", + "z_ordering", + } + ), + "datafusion": frozenset({}), + "doris": frozenset({}), + "duckdb": frozenset({"foreign_keys", "primary_keys", "sorting"}), + "fabric-warehouse": frozenset({"foreign_keys", "primary_keys"}), + "firebolt": frozenset({}), + "motherduck": frozenset({"sorting"}), + "mysql": frozenset({"foreign_keys", "primary_keys"}), + "paradedb": frozenset({"foreign_keys", "primary_keys"}), + "pg-duckdb": frozenset({"foreign_keys", "primary_keys"}), + "pg-mooncake": frozenset({"foreign_keys", "primary_keys"}), + "postgresql": frozenset({"foreign_keys", "primary_keys"}), + "presto": frozenset({"foreign_keys"}), + "questdb": frozenset({}), + "redshift": frozenset({"foreign_keys", "primary_keys", "sorting"}), + "singlestore": frozenset({"primary_keys"}), + "snowflake": frozenset({"clustering", "foreign_keys", "partitioning", "primary_keys"}), + "sqlite": frozenset({"foreign_keys", "primary_keys"}), + "starrocks": frozenset({"distribution", "partitioning", "primary_keys", "sorting"}), + "timescaledb": frozenset({"foreign_keys", "primary_keys"}), + "trino": frozenset({"foreign_keys"}), +} + + +_SOURCE_LOCATION = re.compile(r"\.py:\d+") + +_LAYOUT_TYPES = frozenset({TuningType.PARTITIONING, TuningType.CLUSTERING, TuningType.DISTRIBUTION, TuningType.SORTING}) + + +def _config_requesting(tuning_type: TuningType) -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.platform_optimizations.sorted_ingestion_mode = "auto" + for attribute, _checked in UnifiedTuningConfiguration._OPT_CHECKS: + setattr(config.platform_optimizations, attribute, False) + if tuning_type in _LAYOUT_TYPES: + column = [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + config.table_tunings["orders"] = TableTuning(table_name="orders", **{tuning_type.value: column}) + return config + for attribute, checked in UnifiedTuningConfiguration._CONSTRAINT_CHECKS: + if checked == tuning_type: + getattr(config, attribute).enabled = True + return config + for attribute, checked in UnifiedTuningConfiguration._OPT_CHECKS: + if checked == tuning_type: + setattr(config.platform_optimizations, attribute, True) + return config + raise AssertionError(f"no way to request {tuning_type}") + + +class TestMechanismPreconditions: + def test_redshift_sorting_requires_sorted_ingestion(self) -> None: + config = _config_requesting(TuningType.SORTING) + config.platform_optimizations.sorted_ingestion_mode = "off" + errors, _warnings = config.validate_for_platform_detailed("redshift") + assert any("sorted_ingestion_mode" in error and error.startswith("requested sorting") for error in errors) + + config.platform_optimizations.sorted_ingestion_mode = "auto" + errors, _warnings = config.validate_for_platform_detailed("redshift") + assert not any(error.startswith("requested sorting") for error in errors) + + def test_only_redshift_sorting_is_gated_on_sorted_ingestion(self) -> None: + gated = { + (platform, tuning_type.value) + for platform, entries in PLATFORM_TUNING_CAPABILITIES.items() + for tuning_type, capability in entries.items() + if capability.requires_sorted_ingestion + } + assert gated == {("redshift", "sorting")} + + @pytest.mark.parametrize("alias", ["spark", "delta", "databricks:sql", "Databricks"]) + def test_databricks_aliases_run_the_effective_layout_validator(self, alias: str) -> None: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.platform_optimizations.databricks_clustering_strategy = "liquid_clustering" + config.platform_optimizations.liquid_clustering_enabled = True + config.table_tunings["orders"] = TableTuning( + table_name="orders", + clustering=[TuningColumn(name=f"c{index}", type="INTEGER", order=index) for index in range(1, 6)], + ) + reference, _ = config.validate_for_platform_detailed("databricks") + errors, _warnings = config.validate_for_platform_detailed(alias) + assert any("four" in error or "4" in error for error in reference), reference + assert [error for error in errors if "four" in error or "4" in error] == [ + error for error in reference if "four" in error or "4" in error + ] + + +class TestRegistryTable: + def test_table_covers_every_registry_platform(self) -> None: + assert set(_NOT_RENDERED) == set(_RENDERED) == known_registry_platforms() + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_registry_matches_table(self, platform: str) -> None: + entries = PLATFORM_TUNING_CAPABILITIES[platform] + not_rendered = frozenset( + tuning_type.value for tuning_type, capability in entries.items() if is_not_rendered_at_execution(capability) + ) + rendered = frozenset(tuning_type.value for tuning_type in entries) - not_rendered + assert not_rendered == _NOT_RENDERED[platform], f"{platform}: not-rendered set changed" + assert rendered == _RENDERED[platform], f"{platform}: rendered set changed" + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_requested_types_error_exactly_when_table_says_not_rendered(self, platform: str) -> None: + for tuning_value in sorted(_NOT_RENDERED[platform] | _RENDERED[platform]): + errors, _warnings = _config_requesting(TuningType(tuning_value)).validate_for_platform_detailed(platform) + refused = any(error.startswith(f"requested {tuning_value} is not applied at execution") for error in errors) + assert refused == (tuning_value in _NOT_RENDERED[platform]), (platform, tuning_value, errors) + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_error_text_names_mechanism_without_source_locations(self, platform: str) -> None: + for tuning_value in sorted(_NOT_RENDERED[platform]): + tuning_type = TuningType(tuning_value) + capability = get_capability(platform, tuning_type) + assert capability is not None + errors, _warnings = _config_requesting(tuning_type).validate_for_platform_detailed(platform) + refusal = next(error for error in errors if error.startswith(f"requested {tuning_value} is not applied")) + assert capability.mechanism_id in refusal + assert not _SOURCE_LOCATION.search(refusal), refusal + assert not _SOURCE_LOCATION.search(not_rendered_reason(capability)) + + def test_alias_platforms_resolve_to_registry_entries(self) -> None: + for alias, target in PLATFORM_ALIASES.items(): + if target in known_registry_platforms(): + assert resolve_platform_key(alias) == target + + +_GENERATOR_QUIET_FIELDS: dict[tuple[str, str], str] = { + ("databricks", "distribution"): "distribute_by", + ("firebolt", "sorting"): "sort_by", + ("firebolt", "clustering"): "cluster_by", + ("questdb", "sorting"): "sort_by", + ("questdb", "distribution"): "distribute_by", + ("azure-synapse", "sorting"): "sort_by", + ("azure-synapse", "clustering"): "cluster_by", +} + + +_DATABEND_QUIET_KINDS: frozenset[str] = frozenset({"partitioning", "sorting", "distribution"}) + + +_VACUITY_SIGNATURE_ONLY: dict[tuple[str, str], str] = { + ("tpcds_obt", "TPCDSOBT"): "get_create_tables_sql accepts no tuning_config", + ("core.joinorder.benchmark", "JoinOrderSchema"): "schema helper; get_create_tables_sql accepts no tuning_config", +} + + +_DISCOVERY_IMPORT_SKIPS: dict[str, str] = {} + + +def _benchmark_classes_in(module: object, module_key: str, seen: set[int]) -> list[tuple[str, str, type]]: + found: list[tuple[str, str, type]] = [] + for attribute in sorted(vars(module)): + value = vars(module)[attribute] + if ( + inspect.isclass(value) + and hasattr(value, "get_create_tables_sql") + and value.__module__.split(".")[0] == "benchbox" + and id(value) not in seen + ): + seen.add(id(value)) + found.append((module_key, attribute, value)) + return found + + +def _discovered_benchmarks() -> list[tuple[str, str, type]]: + import importlib + import pkgutil + + import benchbox + import benchbox.core + + found: list[tuple[str, str, type]] = [] + seen: set[int] = set() + for module in sorted(pkgutil.iter_modules(benchbox.__path__), key=lambda entry: entry.name): + imported = importlib.import_module(f"benchbox.{module.name}") + found.extend(_benchmark_classes_in(imported, module.name, seen)) + failures: dict[str, str] = {} + for module in sorted(pkgutil.walk_packages(benchbox.core.__path__, "benchbox.core."), key=lambda e: e.name): + if module.name in _DISCOVERY_IMPORT_SKIPS: + continue + try: + imported = importlib.import_module(module.name) + except ImportError as exc: + failures[module.name] = str(exc) + continue + found.extend(_benchmark_classes_in(imported, module.name.removeprefix("benchbox."), seen)) + assert not failures, f"benchbox.core modules failed to import; fix or list in _DISCOVERY_IMPORT_SKIPS: {failures}" + return found + + +def _vacuous_capable_benchmarks() -> list[tuple[str, str, object]]: + capable: list[tuple[str, str, object]] = [] + for module_name, class_name, benchmark_cls in _discovered_benchmarks(): + if (module_name, class_name) in _VACUITY_SIGNATURE_ONLY: + parameters = inspect.signature(benchmark_cls.get_create_tables_sql).parameters + assert "tuning_config" not in parameters, ( + f"{module_name}.{class_name} now accepts tuning_config; cover it behaviorally" + ) + continue + try: + benchmark = benchmark_cls() + except Exception as exc: + pytest.fail( + f"{module_name}.{class_name} needs explicit construction ({exc}); construct it in " + "_vacuous_capable_benchmarks or classify it in _VACUITY_SIGNATURE_ONLY" + ) + parameters = inspect.signature(benchmark.get_create_tables_sql).parameters + assert "tuning_config" in parameters, ( + f"{module_name}.{class_name} accepts no tuning_config; classify it in _VACUITY_SIGNATURE_ONLY" + ) + capable.append((module_name, class_name, benchmark)) + assert capable, "no behaviorally covered benchmarks; the vacuity proof is empty" + return capable + + +class TestBenchmarkDdlAndAdapterBehavior: + def test_unique_check_rendering_is_vacuous_in_tree(self) -> None: + for module_name, class_name, benchmark in _vacuous_capable_benchmarks(): + plain = UnifiedTuningConfiguration() + plain.disable_all_constraints() + toggled = UnifiedTuningConfiguration() + toggled.disable_all_constraints() + toggled.unique_constraints.enabled = True + toggled.check_constraints.enabled = True + assert benchmark.get_create_tables_sql(tuning_config=plain) == benchmark.get_create_tables_sql( + tuning_config=toggled + ), ( + f"{module_name}.{class_name} DDL now varies with UNIQUE/CHECK flags; the not-rendered " + "UNIQUE/CHECK registry entries must be re-audited against execution" + ) + + def test_benchmark_ddl_ignores_layout_flags_in_tree(self) -> None: + for module_name, class_name, benchmark in _vacuous_capable_benchmarks(): + plain = UnifiedTuningConfiguration() + plain.disable_all_constraints() + toggled = UnifiedTuningConfiguration() + toggled.disable_all_constraints() + toggled.table_tunings["orders"] = TableTuning( + table_name="orders", + partitioning=[TuningColumn(name="o_orderdate", type="DATE", order=1)], + sorting=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + clustering=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + distribution=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + ) + assert benchmark.get_create_tables_sql(tuning_config=plain) == benchmark.get_create_tables_sql( + tuning_config=toggled + ), ( + f"{module_name}.{class_name} DDL now varies with layout tuning flags; the not-rendered " + "layout registry entries must be re-audited against execution" + ) + + def test_vacuity_discovery_covers_all_benchmarks(self) -> None: + found = {(module_name, class_name) for module_name, class_name, _ in _discovered_benchmarks()} + assert found, "benchmark discovery is empty; the vacuity proof covers nothing" + known = { + ("tpch", "TPCH"), + ("tpcds", "TPCDS"), + ("ssb", "SSB"), + ("clickbench", "ClickBench"), + ("core.ai_primitives", "AIPrimitivesBenchmark"), + } + assert known <= found, f"discovery lost known benchmarks: {sorted(known - found)}" + for key in _VACUITY_SIGNATURE_ONLY: + assert key in found, f"signature-only benchmark {key} disappeared from discovery" + for module_name, class_name in sorted(found - set(_VACUITY_SIGNATURE_ONLY)): + assert any( + module_name == seen_module and class_name == seen_class + for seen_module, seen_class, _ in _vacuous_capable_benchmarks() + ), ( + f"{module_name}.{class_name} is not covered by the vacuity proof; construct it in " + "_vacuous_capable_benchmarks or classify it in _VACUITY_SIGNATURE_ONLY" + ) + + def test_questdb_create_schema_strips_constraints(self) -> None: + from benchbox.platforms.base.ddl_helpers import strip_foreign_keys + from benchbox.platforms.questdb import QuestDBAdapter + + adapter = QuestDBAdapter.__new__(QuestDBAdapter) + create = "CREATE TABLE orders (o_orderkey INTEGER, o_custkey INTEGER, PRIMARY KEY (o_orderkey));" + assert "PRIMARY KEY" not in QuestDBAdapter._strip_pk_constraints(adapter, create) + assert "o_orderkey INTEGER" in QuestDBAdapter._strip_pk_constraints(adapter, create) + foreign = "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x));" + assert "FOREIGN KEY" not in strip_foreign_keys(foreign) + + def test_trino_base_optimize_strips_primary_keys(self) -> None: + from benchbox.platforms.presto_trino_adapter_base import PrestoTrinoAdapterBase + + adapter = PrestoTrinoAdapterBase.__new__(PrestoTrinoAdapterBase) + adapter.table_format = "hive" + adapter.catalog = "catalog" + create = "CREATE TABLE orders (o_orderkey INTEGER, PRIMARY KEY (o_orderkey));" + assert "PRIMARY KEY" not in PrestoTrinoAdapterBase._optimize_table_definition(adapter, create) + foreign = "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x));" + assert "FOREIGN KEY" in PrestoTrinoAdapterBase._optimize_table_definition(adapter, foreign) + + def test_clickhouse_optimize_preserves_foreign_keys(self) -> None: + from benchbox.platforms.clickhouse.workload import ClickHouseWorkloadMixin + + mixin = ClickHouseWorkloadMixin.__new__(ClickHouseWorkloadMixin) + create = ( + "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x)) " + "ENGINE = MergeTree() ORDER BY tuple();" + ) + assert "FOREIGN KEY" in ClickHouseWorkloadMixin._optimize_table_definition(mixin, create, None) + + def test_azure_synapse_optimize_uses_engine_default(self) -> None: + from benchbox.platforms.azure_synapse import AzureSynapseAdapter + + adapter = AzureSynapseAdapter.__new__(AzureSynapseAdapter) + adapter.schema = "dbo" + adapter.distribution_default = "ROUND_ROBIN" + create = "CREATE TABLE orders (o_orderkey INTEGER, tuned_dist INTEGER);" + rendered = AzureSynapseAdapter._optimize_table_definition(adapter, create) + assert "DISTRIBUTION = ROUND_ROBIN" in rendered + assert "tuned_dist" not in rendered.split("WITH (DISTRIBUTION")[1] + assert "PARTITION" not in rendered.upper() + + def test_questdb_enhancement_takes_no_tuning(self) -> None: + from benchbox.platforms.questdb import QuestDBAdapter + + parameters = inspect.signature(QuestDBAdapter._apply_questdb_schema_enhancements).parameters + assert "tuning_config" not in parameters and "table_tuning" not in parameters + adapter = QuestDBAdapter.__new__(QuestDBAdapter) + adapter.partition_by = None + create = "CREATE TABLE lineitem (l_orderkey INTEGER, l_shipdate DATE);" + rendered = QuestDBAdapter._apply_questdb_schema_enhancements(adapter, create) + assert "PARTITION BY MONTH" in rendered.upper() + + def test_quiet_generators_emit_no_clause(self) -> None: + from benchbox.core.tuning.ddl_generator import get_ddl_generator + + for (platform_key, tuning_value), field in sorted(_GENERATOR_QUIET_FIELDS.items()): + tuning: dict[str, list[TuningColumn]] = { + tuning_value: [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + } + clauses = get_ddl_generator(platform_key).generate_tuning_clauses(TableTuning(table_name="t", **tuning)) + assert getattr(clauses, field) is None, ( + f"{platform_key}/{tuning_value} generator emits a {field} clause from tuned columns" + ) + + def test_databend_generate_ignores_non_clustering(self) -> None: + from benchbox.platforms.databend.adapter import DatabendAdapter + + for tuning_value in sorted(_DATABEND_QUIET_KINDS): + tuning: dict[str, list[TuningColumn]] = { + tuning_value: [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + } + assert DatabendAdapter.generate_tuning_clause(None, TableTuning(table_name="t", **tuning)) == "" + + def test_pk_fk_flags_drive_ddl_content(self) -> None: + from benchbox.ssb import SSBBenchmark + from benchbox.tpcds import TPCDSBenchmark + from benchbox.tpch import TPCHBenchmark + + for benchmark in (TPCHBenchmark(), TPCDSBenchmark(), SSBBenchmark()): + name = type(benchmark).__name__ + disabled = UnifiedTuningConfiguration() + disabled.disable_all_constraints() + pk_only = UnifiedTuningConfiguration() + pk_only.disable_all_constraints() + pk_only.primary_keys.enabled = True + plain = benchmark.get_create_tables_sql(tuning_config=disabled) + keyed = benchmark.get_create_tables_sql(tuning_config=pk_only) + assert keyed != plain, f"{name} DDL ignores the primary_keys flag" + assert "PRIMARY KEY" in keyed, f"{name} DDL carries no PRIMARY KEY clause" + assert "PRIMARY KEY" not in plain, f"{name} DDL carries PRIMARY KEY with flags off" + for benchmark in (TPCHBenchmark(), TPCDSBenchmark()): + name = type(benchmark).__name__ + disabled = UnifiedTuningConfiguration() + disabled.disable_all_constraints() + fk_only = UnifiedTuningConfiguration() + fk_only.disable_all_constraints() + fk_only.foreign_keys.enabled = True + plain = benchmark.get_create_tables_sql(tuning_config=disabled) + keyed = benchmark.get_create_tables_sql(tuning_config=fk_only) + assert keyed != plain, f"{name} DDL ignores the foreign_keys flag" + assert "FOREIGN KEY" in keyed, f"{name} DDL carries no FOREIGN KEY clause" + + def test_athena_rewrite_takes_no_tuning(self) -> None: + from benchbox.platforms.athena import AthenaAdapter + + parameters = inspect.signature(AthenaAdapter._convert_to_external_table).parameters + assert "tuning_config" not in parameters and "table_tuning" not in parameters + + def test_datafusion_schema_ignores_constraints(self) -> None: + from benchbox.platforms.datafusion import DataFusionAdapter + + parameters = inspect.signature(DataFusionAdapter._get_benchmark_schema).parameters + assert "tuning_config" not in parameters and "enable_primary_keys" not in parameters + assert "enable_foreign_keys" not in parameters + + +class TestShippedTemplatesValidateClean: + @pytest.mark.parametrize("platform", ["clickhouse", "databricks", "duckdb", "snowflake"]) + def test_packaged_templates_validate_clean(self, platform: str) -> None: + from benchbox.core.tuning.packaged_templates import list_packaged_templates + + templates = list_packaged_templates(platform) + assert templates.get(platform), f"no packaged templates found for {platform}" + for template_path in templates[platform]: + config = UnifiedTuningConfiguration.from_dict(yaml.safe_load(template_path.read_text(encoding="utf-8"))) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"{template_path}: {errors}" + + @pytest.mark.parametrize( + "platform", + ["duckdb", "clickhouse", "snowflake", "bigquery", "postgresql", "databricks", "redshift"], + ) + def test_sample_configs_validate_clean(self, tmp_path, platform: str) -> None: + from benchbox.cli.config import ConfigManager + + target = tmp_path / f"{platform}_tuning.yaml" + ConfigManager().create_sample_unified_tuning_config(target, platform) + config = UnifiedTuningConfiguration.from_dict(yaml.safe_load(target.read_text(encoding="utf-8"))) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"sample for {platform}: {errors}" + + @pytest.mark.parametrize("platform", sorted(known_registry_platforms() | {"spark", "starburst"})) + def test_fallback_config_validates_clean(self, tmp_path, platform: str) -> None: + from benchbox.cli.tuning_runtime import build_default_unified_config + + config = build_default_unified_config(platform) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"fallback for {platform}: {errors}" diff --git a/tests/unit/core/tuning/test_capability_source_drift.py b/tests/unit/core/tuning/test_capability_source_drift.py index a84a080496..e11589e37f 100644 --- a/tests/unit/core/tuning/test_capability_source_drift.py +++ b/tests/unit/core/tuning/test_capability_source_drift.py @@ -66,6 +66,7 @@ "clustering", "primary_keys", "foreign_keys", + "unique_constraints", "check_constraints", "materialized_views", } @@ -88,7 +89,9 @@ "sorting", "clustering", "primary_keys", + "foreign_keys", "unique_constraints", + "check_constraints", "materialized_views", } ), @@ -184,6 +187,14 @@ def test_registry_has_platforms_interface_deliberately_excludes(): "questdb", "pg-duckdb", "pg-mooncake", + "cedardb", + "citus", + "paradedb", + "singlestore", + "databend", + "datafusion", + "motherduck", + "fabric-warehouse", } diff --git a/tests/unit/core/tuning/test_clickhouse_pk_prefix.py b/tests/unit/core/tuning/test_clickhouse_pk_prefix.py index 6cce5bbbd0..ba3ed741df 100644 --- a/tests/unit/core/tuning/test_clickhouse_pk_prefix.py +++ b/tests/unit/core/tuning/test_clickhouse_pk_prefix.py @@ -86,6 +86,8 @@ def _config(table_tuning: TableTuning, primary_keys_enabled: bool) -> UnifiedTun config = UnifiedTuningConfiguration() config.primary_keys.enabled = primary_keys_enabled config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False config.table_tunings[table_tuning.table_name] = table_tuning return config diff --git a/tests/unit/core/tuning/test_fallback_description.py b/tests/unit/core/tuning/test_fallback_description.py index beaeebdf79..68295077aa 100644 --- a/tests/unit/core/tuning/test_fallback_description.py +++ b/tests/unit/core/tuning/test_fallback_description.py @@ -46,7 +46,7 @@ def test_descriptions_are_not_one_fixed_string(): def test_sql_override_keys_are_canonical_registry_platforms(): sql_overrides = {key for key in PLATFORM_FALLBACK_DESCRIPTIONS if key in PLATFORM_TUNING_CAPABILITIES} - assert sql_overrides == {"clickhouse"} + assert sql_overrides == {"clickhouse", "datafusion"} @pytest.mark.parametrize( diff --git a/tests/unit/core/tuning/test_platform_identity_keys.py b/tests/unit/core/tuning/test_platform_identity_keys.py index 8051371f28..feb1250154 100644 --- a/tests/unit/core/tuning/test_platform_identity_keys.py +++ b/tests/unit/core/tuning/test_platform_identity_keys.py @@ -73,17 +73,33 @@ class TestDisplayNameValidationNoLongerErrors: @pytest.mark.parametrize("display_name", DISPLAY_NAMES) def test_default_config_produces_no_errors_for_display_names(self, display_name): config = UnifiedTuningConfiguration() + config.disable_all_constraints() assert config.validate_for_platform(display_name) == [] - @pytest.mark.parametrize("display_name", DISPLAY_NAMES) - def test_display_name_mismatches_are_warnings_not_errors(self, display_name): + def test_doris_display_name_applies_rendered_via_rules(self): + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("Doris") + assert any("requested primary_keys is not applied at execution" in error for error in errors) + assert any("requested foreign_keys is not applied at execution" in error for error in errors) + + def test_starrocks_display_name_validates_clean(self): + config = UnifiedTuningConfiguration() + config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False + errors, warnings = config.validate_for_platform_detailed("StarRocks") + assert errors == [] + assert warnings == [] + + @pytest.mark.parametrize("display_name", ["ClickHouse (Local)", "Apache Doris"]) + def test_unresolvable_display_name_mismatches_are_warnings_not_errors(self, display_name): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed(display_name) assert errors == [] assert len(warnings) == 4 assert all("is not supported by platform" in w for w in warnings) - def test_table_layout_tuning_on_unknown_platform_warns_not_errors(self): + def test_table_layout_tuning_on_rendering_unknown_platform_is_silent(self): config = UnifiedTuningConfiguration() config.disable_all_constraints() config.table_tunings["orders"] = TableTuning( @@ -93,7 +109,7 @@ def test_table_layout_tuning_on_unknown_platform_warns_not_errors(self): errors, warnings = config.validate_for_platform_detailed("starrocks") assert errors == [] - assert any("'distribution'" in w for w in warnings) + assert warnings == [] class TestKnownPlatformSemanticsPreserved: @@ -132,10 +148,11 @@ def test_known_platform_set_stays_in_sync_with_compatibility_map(self): for tuning_type in tuning_types: assert tuning_type.is_compatible_with_platform(platform) is True - def test_duckdb_default_config_still_validates_clean(self): + def test_duckdb_default_config_errors_on_unrendered_constraints(self): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed("duckdb") - assert errors == [] + assert any("requested unique_constraints is not applied at execution" in e for e in errors) + assert any("requested check_constraints is not applied at execution" in e for e in errors) assert warnings == [] def test_layout_incompatibility_on_known_platform_still_errors(self): @@ -158,12 +175,13 @@ def test_platform_specific_opt_on_known_platform_still_errors(self): assert any("'z_ordering'" in e for e in errors) assert config.validate_for_platform("databricks") == [] - def test_constraint_mismatch_on_known_platform_is_warning(self): + def test_requested_but_not_rendered_constraint_is_an_error(self): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed("clickhouse") - assert errors == [] - assert any("'foreign_keys'" in w for w in warnings) - assert any("'check_constraints'" in w for w in warnings) + assert any("requested unique_constraints is not applied at execution" in e for e in errors) + assert any("requested foreign_keys is not applied at execution" in e for e in errors) + assert any("requested check_constraints is not applied at execution" in e for e in errors) + assert warnings == [] def test_databricks_liquid_layout_validation_still_errors(self): config = UnifiedTuningConfiguration() @@ -220,9 +238,13 @@ def test_registry_constructed_adapter_resolves_aliases(self, tmp_path): def test_validate_tuning_configuration_uses_canonical_key(self): adapter = _StubAdapter(display_name="StarRocks", type="starrocks") - adapter.unified_tuning_configuration = UnifiedTuningConfiguration() + config = UnifiedTuningConfiguration() + config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False + adapter.unified_tuning_configuration = config assert adapter.validate_tuning_configuration_for_platform() == [] - assert adapter.validate_tuning_configuration(UnifiedTuningConfiguration()) == [] + assert adapter.validate_tuning_configuration(config) == [] class _MetadataStubAdapter: diff --git a/tests/unit/core/tuning/test_tuning_reconciliation.py b/tests/unit/core/tuning/test_tuning_reconciliation.py index d09c1ae5ce..8e02c65bc2 100644 --- a/tests/unit/core/tuning/test_tuning_reconciliation.py +++ b/tests/unit/core/tuning/test_tuning_reconciliation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from unittest.mock import Mock import pytest @@ -15,7 +16,7 @@ STATEMENT_FAILED, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import get_capability +from benchbox.core.tuning.capability_registry import get_capability, not_rendered_reason from benchbox.core.tuning.interface import ( PlatformOptimizationConfiguration, TableTuning, @@ -266,7 +267,8 @@ def test_backtick_quoted_starrocks_clauses_are_evidence(self): assert ledger.dropped == [] assert [(item.intent, item.satisfied_by) for item in ledger.satisfied] == [ - ("distribution:LINEITEM (L_ORDERKEY)", 0) + ("distribution:LINEITEM (L_ORDERKEY)", 0), + ("primary_keys", 1), ] def test_property_style_columns_inside_literals_are_evidence(self): @@ -330,7 +332,7 @@ class TestDropReasons: ("duckdb", "partitioning", TuningType.PARTITIONING), ], ) - def test_registry_none_or_preview_only_drops_with_registry_notes(self, platform, slot, tuning_type): + def test_registry_none_or_preview_only_drops_with_registry_reason(self, platform, slot, tuning_type): ledger = AppliedTuningLedger() _reconcile(_config(**{slot: _columns("L_SHIPDATE")}), ledger, platform) @@ -338,7 +340,8 @@ def test_registry_none_or_preview_only_drops_with_registry_notes(self, platform, [dropped] = ledger.dropped capability = get_capability(platform, tuning_type) assert capability is not None and capability.rendered_via == "none" - assert dropped.reason == capability.notes + assert dropped.reason == f"not rendered at execution: {not_rendered_reason(capability)}" + assert not re.search(r"\.py:\d+", dropped.reason) @pytest.mark.parametrize("platform", ["datafusion", "emr-serverless", "cedardb", "influxdb"]) def test_platform_without_registry_entry_drops_as_not_rendered(self, platform): @@ -609,10 +612,15 @@ def test_declared_keys_that_were_not_rendered_still_drop(self, platform): declared_constraints=frozenset({TuningType.PRIMARY_KEYS, TuningType.FOREIGN_KEYS}), ) - assert {item.intent: item.reason for item in ledger.dropped} == { - "primary_keys": NOT_RENDERED_REASON, - "foreign_keys": NOT_RENDERED_REASON, - } + expected_reasons = {"primary_keys": NOT_RENDERED_REASON, "foreign_keys": NOT_RENDERED_REASON} + if platform == "datafusion": + expected_reasons = { + kind.value: "not rendered at execution: " + "the schema is created without constraints (datafusion_unconstrained_schema)" + for kind in (TuningType.PRIMARY_KEYS, TuningType.FOREIGN_KEYS) + } + + assert {item.intent: item.reason for item in ledger.dropped} == expected_reasons def test_unknown_declarations_reconcile_every_enabled_toggle(self): ledger = AppliedTuningLedger() diff --git a/tests/unit/platforms/test_tuning_reconciliation_adapters.py b/tests/unit/platforms/test_tuning_reconciliation_adapters.py index ac37cc9b3c..20e4f09229 100644 --- a/tests/unit/platforms/test_tuning_reconciliation_adapters.py +++ b/tests/unit/platforms/test_tuning_reconciliation_adapters.py @@ -20,7 +20,7 @@ PHASE_POST_LOAD, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import get_capability +from benchbox.core.tuning.capability_registry import get_capability, not_rendered_reason from benchbox.core.tuning.interface import TableTuning, TuningColumn, TuningType, UnifiedTuningConfiguration from benchbox.core.tuning.reconciliation import NOT_RENDERED_REASON, RECONCILIATION_FAILED_INTENT from benchbox.platforms.base import PlatformAdapter, tuning_trust @@ -89,14 +89,19 @@ def _tuned(adapter: PlatformAdapter) -> PlatformAdapter: return adapter +_DATAFUSION_REASON = ( + "not rendered at execution: the schema is created without constraints (datafusion_unconstrained_schema)" +) + + def _drops(adapter: PlatformAdapter) -> dict[str, str]: return {item.intent: item.reason for item in adapter._applied_tuning_ledger.dropped} -def _registry_notes(platform: str, tuning_type: TuningType) -> str: +def _registry_reason(platform: str, tuning_type: TuningType) -> str: capability = get_capability(platform, tuning_type) assert capability is not None and capability.rendered_via == "none" - return capability.notes + return f"not rendered at execution: {not_rendered_reason(capability)}" def _status(adapter: PlatformAdapter) -> str: @@ -117,7 +122,7 @@ def save_unified_tunings(self, config): class TestFamilies: - def test_postgres_no_op_apply_drops_layout_with_registry_notes(self, monkeypatch, tmp_path): + def test_postgres_no_op_apply_drops_layout_with_registry_reason(self, monkeypatch, tmp_path): monkeypatch.setattr(postgresql_module, "psycopg", Mock(__version__="3.1.0")) monkeypatch.setattr("benchbox.core.tuning.metadata.TuningMetadataManager", FakeMetadataManager) config = _layout_config(partitioning=_columns("L_SHIPDATE"), clustering=_columns("L_ORDERKEY")) @@ -133,8 +138,8 @@ def test_postgres_no_op_apply_drops_layout_with_registry_notes(self, monkeypatch adapter._reconcile_requested_tuning(config) assert _drops(adapter) == { - "partitioning:LINEITEM (L_SHIPDATE)": _registry_notes("postgresql", TuningType.PARTITIONING), - "clustering:LINEITEM (L_ORDERKEY)": _registry_notes("postgresql", TuningType.CLUSTERING), + "partitioning:LINEITEM (L_SHIPDATE)": _registry_reason("postgresql", TuningType.PARTITIONING), + "clustering:LINEITEM (L_ORDERKEY)": _registry_reason("postgresql", TuningType.CLUSTERING), } assert _status(adapter) == NOOP @@ -146,7 +151,12 @@ def test_postgres_family_variants_drop_every_layout_intent(self, platform): [reason] = _drops(adapter).values() capability = get_capability(platform, TuningType.PARTITIONING) - assert reason == (capability.notes if capability is not None else NOT_RENDERED_REASON) + expected = ( + f"not rendered at execution: {not_rendered_reason(capability)}" + if capability is not None + else NOT_RENDERED_REASON + ) + assert reason == expected def test_postgres_constraints_rendered_in_create_table_are_satisfied(self): adapter = _tuned(_Adapter(type="postgresql")) @@ -165,7 +175,7 @@ def test_postgres_constraints_rendered_in_create_table_are_satisfied(self): ] assert _drops(adapter) == {"foreign_keys": NOT_RENDERED_REASON} - def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_notes(self): + def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_reason(self): adapter = _tuned(_Adapter(type="bigquery")) adapter._reconcile_requested_tuning( @@ -173,8 +183,8 @@ def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_n ) assert _drops(adapter) == { - "partitioning:LINEITEM (L_SHIPDATE)": _registry_notes("bigquery", TuningType.PARTITIONING), - "clustering:LINEITEM (L_ORDERKEY)": _registry_notes("bigquery", TuningType.CLUSTERING), + "partitioning:LINEITEM (L_SHIPDATE)": _registry_reason("bigquery", TuningType.PARTITIONING), + "clustering:LINEITEM (L_ORDERKEY)": _registry_reason("bigquery", TuningType.CLUSTERING), } def test_redshift_analyze_only_drops_distribution_and_unrendered_sort(self): @@ -188,7 +198,7 @@ def test_redshift_analyze_only_drops_distribution_and_unrendered_sort(self): ) assert _drops(adapter) == { - "distribution:LINEITEM (L_ORDERKEY)": _registry_notes("redshift", TuningType.DISTRIBUTION), + "distribution:LINEITEM (L_ORDERKEY)": _registry_reason("redshift", TuningType.DISTRIBUTION), "sorting:LINEITEM (L_SHIPDATE)": NOT_RENDERED_REASON, } assert _status(adapter) == APPLIED_UNVERIFIED @@ -221,8 +231,8 @@ def test_datafusion_requested_constraints_are_dropped_not_noop_with_empty_drops( adapter._reconcile_requested_tuning(config) assert _drops(adapter) == { - "primary_keys": NOT_RENDERED_REASON, - "foreign_keys": NOT_RENDERED_REASON, + "primary_keys": _DATAFUSION_REASON, + "foreign_keys": _DATAFUSION_REASON, } assert adapter._applied_tuning_ledger.statements == [] assert _status(adapter) == NOOP @@ -325,7 +335,7 @@ def test_constraint_toggles_follow_what_the_run_benchmark_declares(self, tmp_pat adapter._reconcile_requested_tuning(UnifiedTuningConfiguration()) - assert _drops(adapter) == {"primary_keys": NOT_RENDERED_REASON} + assert _drops(adapter) == {"primary_keys": _DATAFUSION_REASON} def test_reconciliation_error_fails_closed_with_a_blocking_drop(self, monkeypatch): adapter = self._adapter()