diff --git a/CHANGELOG.md b/CHANGELOG.md index 0be6542687..8bf312e5ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -332,12 +332,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 mismatched keys are recorded as failed ledger statements and warned. Result metadata reports the live tuned values for shuffle partitions and adaptive execution. +- Tuning validation now rejects tuning requests the platform cannot execute, + and platforms without tuning coverage error on unsupported types like + covered platforms. The registry corrects several platforms' coverage + (Postgres family, Doris, SingleStore, Databend, Starburst, MotherDuck, + DataFusion, and fabric-warehouse, which now has its own entry instead of + sharing Databricks'), and the sample and fallback configuration builders + only enable tuning the target platform supports. Layout tuning on + BigQuery, Redshift, Trino and Synapse now errors until their execution + support lands. - Tuned load-only runs (`--phases generate,load`) now go through the same apply → load → reconcile → metadata → ledger/receipt → export lifecycle as full runs, so a load-only bundle never carries a tuned mode without `platform.tuning`. Adapters without the full lifecycle refuse tuned load-only runs with an actionable error instead. - - Result metadata records genuine engine versions (`engine_version`, `engine_version_source: sql_query`) for DuckDB, SQLite and DataFusion queried directly from the active connection rather than package versions. - ClickHouse tuning introspection bounds table queries in SQL by ledger tables and measures snapshot truncation directly on raw query results. @@ -690,6 +698,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 how to choose the engine at run time; pass `--execution-engine` (for example `--execution-engine streaming`). Tuning files that already set `engine_affinity` keep working and record `resolution` `tuning_profile`. +- Tuning validation now fails when a configuration asks for tuning that the + platform does not apply during a run. Earlier releases accepted these + configurations and ran without the tuning. Run `benchbox tuning validate + --platform ` on your saved tuning files before you upgrade; each + error names the tuning type and why it is not applied. Remove the setting + or use a platform that applies it. Platforms that apply tuning in part are + checked type by type. The new errors are: + - `unique_constraints` and `check_constraints` fail on every platform with + registry coverage, because no benchmark schema carries those clauses. The + packaged DuckDB and Snowflake templates and the matching + `examples/tunings` files now disable both. + - Layout tuning fails on BigQuery (`partitioning`, `clustering`), Redshift + (`partitioning`, `distribution`), Trino and + Azure Synapse (all four layout types). These adapters build the clauses + for dry-run previews but do not apply them when tables are created. Other + platforms that only preview layout tuning fail the same way (for example + the Postgres family, Doris, Firebolt, QuestDB, SingleStore and + TimescaleDB). + - Redshift `sorting` fails while `sorted_ingestion_mode` is `off`, which is + the default, because the adapter sorts tables only when sorted ingestion + is on. Set `sorted_ingestion_mode` to `auto` or `force` to keep the + request. + - `partitioning` fails on Databricks, and so on the `spark` and `delta` + platform names that resolve to it. The adapter builds the clause for + previews only and never adds it when tables are created. Fabric Warehouse + no longer shares the Databricks rules and also rejects `partitioning`. + - `primary_keys` and `foreign_keys` fail where the adapter removes that + constraint from table definitions: primary keys on Databend, Doris, + Firebolt, QuestDB, Trino and Presto; foreign keys on ClickHouse, Databend, + Doris, Firebolt, QuestDB, SingleStore and StarRocks. Constraints that + DataFusion and MotherDuck do not create fail the same way. + - Platforms without registry coverage now report unsupported tuning types as + errors where earlier releases reported warnings, and default tuning + configurations on platforms that drop tuning report errors where they + reported warnings. ### Added diff --git a/benchbox/cli/commands/tuning_group.py b/benchbox/cli/commands/tuning_group.py index 2116eb277c..0dd20ffead 100644 --- a/benchbox/cli/commands/tuning_group.py +++ b/benchbox/cli/commands/tuning_group.py @@ -38,7 +38,6 @@ from benchbox.core.platform_manifest import get_platform_aliases from benchbox.core.tuning.capability_registry import ( PLATFORM_ALIASES, - get_capability, known_registry_platforms, resolve_platform_key, ) @@ -343,13 +342,6 @@ def _validate_sql_config(config_file: str, platform: str) -> None: errors, warnings = config.validate_for_platform_detailed(platform_key) enabled = sorted(config.get_enabled_tuning_types(), key=lambda tuning_type: tuning_type.value) - for tuning_type in enabled: - capability = get_capability(platform_key, tuning_type) - if capability is not None and capability.rendered_via == "none": - warnings.append( - f"Tuning type '{tuning_type.value}' is accepted by platform '{platform_key}' " - f"but not rendered yet ({capability.mechanism_id})" - ) for message in errors: console.print(f"[red]Error: {message}[/red]") diff --git a/benchbox/cli/config.py b/benchbox/cli/config.py index 8d0a5ea24a..463849a8d3 100644 --- a/benchbox/cli/config.py +++ b/benchbox/cli/config.py @@ -647,6 +647,17 @@ def create_sample_unified_tuning_config(self, path: Optional[Path] = None, platf TuningColumn(name="linenumber", type="INTEGER", order=2), ] + from benchbox.core.tuning.capability_registry import get_capability, unapplied_reason + + platform_known = TuningType.is_known_platform(platform) + for tuning_type in list(sample_config.get_enabled_tuning_types()): + capability = get_capability(platform, tuning_type) + if capability is not None: + if unapplied_reason(capability, sample_config.platform_optimizations.sorted_ingestion_mode): + sample_config.disable_tuning_type(tuning_type) + elif platform_known and not tuning_type.is_compatible_with_platform(platform): + sample_config.disable_tuning_type(tuning_type) + self.save_unified_tuning_config(sample_config, path) console.print( f"[green]✅ Sample unified tuning configuration created for platform '{platform}' at {path}[/green]" diff --git a/benchbox/cli/tuning_runtime.py b/benchbox/cli/tuning_runtime.py index 0f9049ae95..5f98ab913e 100644 --- a/benchbox/cli/tuning_runtime.py +++ b/benchbox/cli/tuning_runtime.py @@ -9,8 +9,16 @@ def build_default_unified_config(platform: str | None) -> UnifiedTuningConfiguration: + from benchbox.core.tuning.capability_registry import get_capability, is_not_rendered_at_execution + config = UnifiedTuningConfiguration() - if platform and resolve_platform_key(str(platform).split(":", 1)[0]) == "duckdb": + if platform: + platform_key = resolve_platform_key(str(platform)) + for tuning_type in config.get_enabled_tuning_types(): + capability = get_capability(platform_key, tuning_type) + if capability is not None and is_not_rendered_at_execution(capability): + config.disable_tuning_type(tuning_type) + if platform and resolve_platform_key(str(platform)) == "duckdb": config.check_constraints.enabled = False return config diff --git a/benchbox/core/tuning/capability_registry.py b/benchbox/core/tuning/capability_registry.py index c32ee8463f..43696076bf 100644 --- a/benchbox/core/tuning/capability_registry.py +++ b/benchbox/core/tuning/capability_registry.py @@ -17,14 +17,20 @@ class TuningCapability: rendered_via: RenderedVia mechanism_id: str notes: str = "" + requires_sorted_ingestion: bool = False def _ddl(mechanism_id: str, notes: str = "") -> TuningCapability: return TuningCapability(rendered_via="ddl", mechanism_id=mechanism_id, notes=notes) -def _post_load(mechanism_id: str, notes: str = "") -> TuningCapability: - return TuningCapability(rendered_via="post_load", mechanism_id=mechanism_id, notes=notes) +def _post_load(mechanism_id: str, notes: str = "", requires_sorted_ingestion: bool = False) -> TuningCapability: + return TuningCapability( + rendered_via="post_load", + mechanism_id=mechanism_id, + notes=notes, + requires_sorted_ingestion=requires_sorted_ingestion, + ) def _session(mechanism_id: str, notes: str = "") -> TuningCapability: @@ -40,12 +46,26 @@ def _none(mechanism_id: str, notes: str = "") -> TuningCapability: _CONSTRAINT_NOTE = "Rendered as an inline column/table constraint clause at CREATE TABLE time." -def _constraint_entries() -> dict[TuningType, TuningCapability]: +def _pk_fk_entries() -> dict[TuningType, TuningCapability]: return { _T.PRIMARY_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.UNIQUE_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.CHECK_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + } + + +_UNRENDERED_UNIQUE_CHECK_NOTE = ( + "Benchmark DDL never carries UNIQUE/CHECK clauses from tuning flags: no in-tree benchmark " + "consumes those flags (pinned tree-wide by " + "tests/unit/core/tuning/test_capability_registry_parity.py " + "test_unique_check_rendering_is_vacuous_in_tree), so requesting them renders nothing at execution." +) + + +def _no_unique_check_entries(mechanism_id: str, notes: str = "") -> dict[TuningType, TuningCapability]: + detail = f"{notes} {_UNRENDERED_UNIQUE_CHECK_NOTE}".strip() + return { + _T.UNIQUE_CONSTRAINTS: _none(mechanism_id, detail), + _T.CHECK_CONSTRAINTS: _none(mechanism_id, detail), } @@ -56,8 +76,8 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "chdb": "clickhouse", "spark": "databricks", "delta": "databricks", - "fabric-warehouse": "databricks", "synapse": "azure-synapse", + "starburst": "trino", } @@ -68,21 +88,25 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "CTAS reorder (CREATE OR REPLACE TABLE ... AS SELECT * FROM ... ORDER BY ...) run after data " "load; DuckDB has no inline CREATE TABLE ORDER BY syntax. Consumed via " "core.tuning.generators.duckdb.DuckDBDDLGenerator.generate_ctas_ddl by both dry-run preview " - "and real execution as of the w2 duckdb migration.", + "and real execution since the DuckDB DDL migration.", ), _T.PARTITIONING: _none( "duckdb_copy_to_hint_only", "DuckDBDDLGenerator.generate_tuning_clauses logs a COPY TO Hive-partitioning hint for " "partition columns but no BenchBox code path applies it to the physical benchmark schema. " - "Documented gap, not migrated this round (compatible per the legacy map; no rendering yet).", + "Documented gap: compatible per the legacy map, but nothing renders it yet.", + ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "duckdb_no_unique_check_rendering", + "DuckDBAdapter.apply_constraint_configuration only logs UNIQUE/CHECK flag state (duckdb.py).", ), - **_constraint_entries(), }, "clickhouse": { _T.PARTITIONING: _ddl( "clickhouse_ddl_generator:PARTITION_BY", "core.tuning.generators.clickhouse.ClickHouseDDLGenerator renders PARTITION BY at CREATE " - "TABLE time. Consumed by dry-run preview and (as of the w2 clickhouse migration) real " + "TABLE time. Consumed by dry-run preview and (since the ClickHouse DDL migration) real " "execution in ClickHouseWorkloadMixin._optimize_table_definition.", ), _T.SORTING: _ddl( @@ -111,18 +135,30 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Compatible per the legacy map (downgraded to a warning regardless, as a constraint type); " "ClickHouse has no UNIQUE constraint syntax and no adapter code renders one.", ), + _T.FOREIGN_KEYS: _none( + "clickhouse_drops_foreign_keys", + "ClickHouseWorkloadMixin._optimize_table_definition preserves FOREIGN KEY clauses it " + "cannot execute (clickhouse/workload.py); the engine has no FOREIGN KEY syntax, so a " + "requested foreign key fails at execution.", + ), + _T.CHECK_CONSTRAINTS: _none( + "clickhouse_no_unique_check_rendering", + "The ClickHouse workload column parser terminates column types at UNIQUE/CHECK keywords " + "without rendering them (clickhouse/workload.py).", + ), _T.MATERIALIZED_VIEWS: _none( "unimplemented", "Compatible per the legacy map; no adapter code creates a materialized view.", ), }, "databricks": { - _T.PARTITIONING: _ddl( - "delta_partitioned_by", - "Rendered as Delta PARTITIONED BY at CREATE TABLE time via the cloud_spark DDL mixin " - "(third renderer per ADR-3; not migrated to core.tuning.generators this round -- Databricks " - "migration is explicitly out of scope, owned by the blocked databricks-liquid-clustering " - "TODO).", + _T.PARTITIONING: _none( + "databricks_partitioning_preview_only", + "DatabricksAdapter.generate_tuning_clause builds PARTITIONED BY from tuned columns " + "(databricks/adapter.py:3005-3048) but has no production call site tree-wide, so no " + "PARTITIONED BY is rendered at preview or execution. Real execution builds Delta DDL in " + "_convert_to_delta_table (databricks/adapter.py:2672-2729), which never adds partitioning " + "columns, and apply_table_tunings only logs them via _log_partitioning_and_sorting.", ), _T.CLUSTERING: _post_load( "databricks_z_order_or_liquid", @@ -155,7 +191,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "unimplemented", "Compatible per the legacy map; no adapter code creates a materialized view.", ), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "databricks_no_unique_check_rendering", + "DatabricksAdapter executes tuning only through ZORDER, OPTIMIZE, Liquid and CLUSTER " + "statements (databricks/adapter.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Declared keys are informational and unenforced on Databricks. The adapter never emits RELY " @@ -182,7 +223,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "partition clause. See CLUSTERING entry for the full mechanism/evidence.", ), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "snowflake_no_unique_check_rendering", + "SnowflakeAdapter tuning executes ALTER TABLE ... CLUSTER BY and constraint-bearing " + "CREATE TABLE carrying PK/FK only (snowflake.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Primary keys on standard Snowflake tables are informational and unenforced (Snowflake table " @@ -219,7 +265,10 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "tests/unit/platforms/test_bigquery_adapter.py.", ), _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), - _T.CHECK_CONSTRAINTS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "bigquery_no_unique_check_rendering", + "BigQuery DDL emits PRIMARY KEY (...) NOT ENFORCED plus FOREIGN KEY clauses only (bigquery.py).", + ), }, "redshift": { _T.DISTRIBUTION: _none( @@ -245,6 +294,7 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "apply_table_tunings only logs the sort-key mismatch (redshift.py:2599-2605) because it cannot " "rewrite the key without table recreation. With sorted ingestion off, _build_ctas_sort_sql " "returns None and nothing is rendered.", + requires_sorted_ingestion=True, ), _T.PARTITIONING: _none( "redshift_no_partition_rendering", @@ -253,7 +303,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "(redshift.py:2629-2636). Nothing is rendered at execution.", ), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "redshift_no_unique_check_rendering", + "RedshiftAdapter tuning executes CTAS-sort, VACUUM and maintenance statements only (redshift.py).", + ), _T.PRIMARY_KEYS: _ddl( _INLINE_CONSTRAINT, "Redshift documents primary and foreign keys as informational and unenforced, but states that " @@ -264,7 +318,14 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "matter here because the planner trusts them.", ), }, - "sqlite": _constraint_entries(), + "sqlite": { + **_pk_fk_entries(), + **_no_unique_check_entries( + "sqlite_no_unique_check_rendering", + "The shared tuning-aware DDL path forwards tuning_config; SQLite execution builds no " + "UNIQUE/CHECK clause from the flags.", + ), + }, "postgresql": { _T.PARTITIONING: _none( "postgresql_ddl_generator:preview_only", @@ -285,7 +346,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: ), _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "postgresql_no_unique_check_rendering", + "PostgreSQLAdapter renders constraint-bearing CREATE TABLE from PK/FK flags only (postgresql.py).", + ), }, "mysql": { _T.PARTITIONING: _none( @@ -298,7 +363,200 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "execution. Corrects the prior false " "_ddl(adapter_mixin:MySQLAdapter.generate_tuning_clause) claim.", ), - **_constraint_entries(), + **_pk_fk_entries(), + **_no_unique_check_entries( + "mysql_no_unique_check_rendering", + "The mysql-wire tuning-aware DDL path renders PK/FK from flags only (base/mysql_wire.py).", + ), + }, + "singlestore": { + _T.PARTITIONING: _none( + "singlestore_no_partition_rendering", + "SingleStoreAdapter inherits NoOpTableTuningMixin (singlestore.py:75): " + "generate_tuning_clause returns '' and apply_table_tunings is a no-op. Tuned " + "constraints flow through the shared mysql-wire tuning-aware DDL path, as on MySQL.", + ), + _T.PRIMARY_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + _T.FOREIGN_KEYS: _none( + "singlestore_strips_foreign_keys", + "SingleStore strips FOREIGN KEY clauses from every CREATE TABLE at execution " + "(singlestore.py singlestore_strip_foreign_keys at 379-380, wired by the DDL_OPTIMIZE " + "strip_foreign_keys rule in sql_compat/rules/ddl_optimize/singlestore_ddl_rewrites.py:6-16). " + "Requested foreign keys never reach the engine (SingleStore error 2752).", + ), + **_no_unique_check_entries( + "singlestore_no_unique_check_rendering", + "SingleStoreAdapter inherits the mysql-wire tuning-aware DDL path rendering PK from " + "flags only; FOREIGN KEY clauses are stripped (singlestore.py).", + ), + }, + "cedardb": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "CedarDBAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "cedardb_no_unique_check_rendering", + "CedarDBAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (cedardb.py).", + ), + }, + "citus": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "CitusAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "citus_no_unique_check_rendering", + "CitusAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (citus.py).", + ), + }, + "paradedb": { + _T.PARTITIONING: _none( + "postgresql_ddl_generator:preview_only", + "ParadeDBAdapter subclasses PostgreSQLAdapter with no DDL override: identical " + "execution gap to postgresql (preview computes PARTITION BY, execution renders nothing).", + ), + _T.CLUSTERING: _none( + "postgresql_ddl_generator:preview_only", + "Same inherited execution gap as PARTITIONING.", + ), + _T.BLOOM_FILTERS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + **_pk_fk_entries(), + **_no_unique_check_entries( + "paradedb_no_unique_check_rendering", + "ParadeDBAdapter inherits PostgreSQLAdapter execution rendering PK/FK from flags only (paradedb.py).", + ), + }, + "fabric-warehouse": { + _T.PARTITIONING: _none( + "delta_ddl_generator:preview_only_tsql_ignores", + "The Delta preview generator computes partitioning, but FabricWarehouseAdapter executes " + "T-SQL (FABRIC_DIALECT='tsql'): the shared builder's tsql output ignores tuned partition " + "columns, generate_tuning_clause returns '' and _optimize_table_definition only adds the " + "schema (fabric_warehouse.py:857-883). Previously aliased to databricks (rendered _ddl) " + "-- that claim was false for the warehouse execution path.", + ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "fabric_warehouse_no_unique_check_rendering", + "FabricWarehouseAdapter executes T-SQL carrying PK/FK only (fabric_warehouse.py).", + ), + }, + "databend": { + _T.CLUSTERING: _post_load( + "databend_alter_cluster_by", + "DatabendAdapter.apply_table_tunings executes ALTER TABLE ... CLUSTER BY from tuned " + "clustering columns (databend/adapter.py:632-651).", + ), + _T.PARTITIONING: _none( + "unimplemented", + "DatabendAdapter.generate_tuning_clause only handles CLUSTERING; nothing renders " + "partitioning at execution.", + ), + _T.SORTING: _none( + "unimplemented", + "Same gap as PARTITIONING: only CLUSTERING is rendered at execution.", + ), + _T.DISTRIBUTION: _none( + "unimplemented", + "Same gap as PARTITIONING: only CLUSTERING is rendered at execution.", + ), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.PRIMARY_KEYS: _none( + "databend_strips_primary_keys", + "DatabendAdapter strips PRIMARY KEY clauses from every executed CREATE TABLE via " + "inline regex (databend/adapter.py:678), alongside the foreign-key strip. Tuned keys " + "reach the DDL builder but never reach the engine.", + ), + _T.FOREIGN_KEYS: _none( + "databend_strips_foreign_keys", + "DatabendAdapter strips foreign keys from every CREATE TABLE (databend/adapter.py:680); " + "remaining constraint configuration is informational log-only.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "databend_no_unique_check_rendering", + "DatabendAdapter strips PK/FK and only renders CLUSTERING; UNIQUE/CHECK clauses are " + "never produced or consumed from tuning.", + ), + _T.CHECK_CONSTRAINTS: _none( + "databend_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: never produced or consumed from tuning.", + ), + }, + "datafusion": { + _T.PRIMARY_KEYS: _none( + "datafusion_unconstrained_schema", + "DataFusionAdapter builds its schema explicitly without constraints even when enabled " + "(datafusion.py:558-564).", + ), + _T.FOREIGN_KEYS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + _T.CHECK_CONSTRAINTS: _none( + "datafusion_unconstrained_schema", + "Same as PRIMARY_KEYS: schema is created without constraints.", + ), + }, + "motherduck": { + _T.SORTING: _post_load( + "duckdb_ctas_sort", + "MotherDuckAdapter runs the shared CTAS-sort path from tuned sorting columns during load " + "(motherduck.py:337, via base/sorted_ingestion.py).", + ), + _T.PARTITIONING: _none( + "motherduck_external_views_unpartitioned", + "MotherDuckAdapter.create_schema executes benchmark DDL verbatim without tuning_config " + "(motherduck.py:307-318) and loads through external views; tuned partitioning columns " + "reshape nothing.", + ), + _T.CLUSTERING: _none( + "motherduck_no_cluster_rendering", + "Same execution gap as PARTITIONING: no post-create CLUSTER step consumes tuning.", + ), + _T.DISTRIBUTION: _none( + "motherduck_not_applicable_single_writer", + "MotherDuck is single-writer cloud DuckDB; nothing renders distribution columns.", + ), + _T.MATERIALIZED_VIEWS: _none("unimplemented", "Compatible per the legacy map; no adapter implementation."), + _T.PRIMARY_KEYS: _none( + "motherduck_drops_tuning_config", + "create_schema drops tuning_config and apply_constraint_configuration is a pass-through " + "no-op (motherduck.py:354-355).", + ), + _T.FOREIGN_KEYS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), + _T.CHECK_CONSTRAINTS: _none( + "motherduck_drops_tuning_config", + "Same as PRIMARY_KEYS: tuning never reaches DDL or the adapter.", + ), }, "starrocks": { _T.PARTITIONING: _ddl( @@ -326,16 +584,60 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "tables -- byte-identical to the previous bespoke injection (BUCKETS 8). Preview and " "execution render through the same generator.", ), + _T.PRIMARY_KEYS: _ddl( + _INLINE_CONSTRAINT, + "Tuned primary keys choose the StarRocks key model: the workload rewrite reads the " + "PK columns from the tuned DDL and emits PRIMARY KEY when they lead the table, else " + "DUPLICATE KEY (starrocks/workload.py:122-152).", + ), + _T.FOREIGN_KEYS: _none( + "starrocks_strips_foreign_keys", + "StarRocksWorkloadMixin strips foreign keys from every CREATE TABLE (starrocks/workload.py:120).", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "starrocks_no_unique_check_rendering", + "StarRocksWorkloadMixin._optimize_table_definition handles PARTITION/DISTRIBUTION/" + "ORDER BY plus PK-model selection and foreign-key stripping only (starrocks/workload.py:" + "109-198); nothing renders, strips, or validates UNIQUE/CHECK clauses.", + ), + _T.CHECK_CONSTRAINTS: _none( + "starrocks_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: no UNIQUE/CHECK handling in the workload rewrite.", + ), }, "doris": { - _T.PARTITIONING: _ddl( - "doris_ddl_generator", - "core.tuning.generators.doris.DorisDDLGenerator exists and is reachable via " - "get_ddl_generator('doris') for dry-run preview. Adapter execution-path parity was not " - "audited this round (Doris is not in this TODO's migration order).", + _T.PARTITIONING: _none( + "doris_ddl_generator:preview_only", + "core.tuning.generators.doris.DorisDDLGenerator computes PARTITION BY from tuned columns " + "for dry-run preview, but DorisAdapter inherits NoOpTableTuningMixin (doris.py:187) and " + "renders nothing at execution. Corrects the prior unaudited _ddl claim.", + ), + _T.SORTING: _none( + "doris_ddl_generator:preview_only", + "Same execution gap as PARTITIONING: preview only, NoOp at execution.", + ), + _T.DISTRIBUTION: _none( + "doris_ddl_generator:preview_only", + "Same execution gap as PARTITIONING: preview only, NoOp at execution.", + ), + _T.PRIMARY_KEYS: _none( + "doris_strips_primary_keys", + "The shared tuning-aware DDL path would emit tuned primary keys, but DorisAdapter " + "strips PRIMARY KEY clauses from every CREATE TABLE (doris.py:803-805).", + ), + _T.FOREIGN_KEYS: _none( + "doris_strips_foreign_keys", + "DorisAdapter strips foreign keys from every CREATE TABLE (doris.py:802).", + ), + _T.UNIQUE_CONSTRAINTS: _none( + "doris_no_unique_check_rendering", + "DorisAdapter renders nothing at execution (NoOpTableTuningMixin) and strips PK/FK; " + "UNIQUE/CHECK clauses are never produced or consumed from tuning.", + ), + _T.CHECK_CONSTRAINTS: _none( + "doris_no_unique_check_rendering", + "Same gap as UNIQUE_CONSTRAINTS: never produced or consumed from tuning.", ), - _T.SORTING: _ddl("doris_ddl_generator", "See PARTITIONING entry."), - _T.DISTRIBUTION: _ddl("doris_ddl_generator", "See PARTITIONING entry."), }, "trino": { _T.PARTITIONING: _none( @@ -369,6 +671,16 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Trino has no separate clustering clause; the generator folds clustering columns into " "sorted_by (trino.py:202-210, see SORTING entry), and the same execution-path gap applies.", ), + _T.PRIMARY_KEYS: _none( + "trino_strips_primary_keys", + "PrestoTrinoAdapterBase._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (presto_trino_adapter_base.py).", + ), + _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "trino_no_unique_check_rendering", + "TrinoAdapter renders no UNIQUE/CHECK clause at execution (trino.py).", + ), }, "presto": { _T.PARTITIONING: _none( @@ -397,6 +709,16 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Presto has no separate clustering clause; folded into sorted_by upstream (see SORTING " "entry), and the same execution-path gap applies.", ), + _T.PRIMARY_KEYS: _none( + "presto_strips_primary_keys", + "PrestoTrinoAdapterBase._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (presto_trino_adapter_base.py).", + ), + _T.FOREIGN_KEYS: _ddl(_INLINE_CONSTRAINT, _CONSTRAINT_NOTE), + **_no_unique_check_entries( + "presto_no_unique_check_rendering", + "PrestoAdapter renders no UNIQUE/CHECK clause at execution (presto.py).", + ), }, "athena": { _T.PARTITIONING: _none( @@ -432,6 +754,12 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Athena has no separate clustering clause; folded into sorted_by upstream, which is dropped " "(see SORTING entry).", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "athena_no_unique_check_rendering", + "AthenaAdapter._convert_to_external_table builds CREATE EXTERNAL TABLE without UNIQUE/CHECK " + "clauses (athena.py).", + ), }, "firebolt": { _T.DISTRIBUTION: _none( @@ -462,6 +790,20 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "clustering hints (firebolt.py:135-142) instead of computing a clause, even for dry-run " "preview.", ), + _T.PRIMARY_KEYS: _none( + "firebolt_strips_primary_keys", + "FireboltAdapter._optimize_table_definition strips PRIMARY KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_primary_keys (firebolt.py).", + ), + _T.FOREIGN_KEYS: _none( + "firebolt_strips_foreign_keys", + "FireboltAdapter._optimize_table_definition strips FOREIGN KEY clauses from every " + "CREATE TABLE via ddl_helpers strip_foreign_keys (firebolt.py).", + ), + **_no_unique_check_entries( + "firebolt_no_unique_check_rendering", + "FireboltAdapter renders no UNIQUE/CHECK clause at execution (firebolt.py).", + ), }, "azure-synapse": { _T.DISTRIBUTION: _none( @@ -496,6 +838,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "clustering hints (azure_synapse.py:183-189) instead of computing a clause, even for dry-run " "preview.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "azure_synapse_no_unique_check_rendering", + "AzureSynapseAdapter real DDL carries PK/FK with the engine-default DISTRIBUTION only (azure_synapse.py).", + ), }, "timescaledb": { _T.PARTITIONING: _none( @@ -531,6 +878,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Merged with DISTRIBUTION into compress_segmentby by the generator (timescaledb.py:177-186); " "same real-execution gap -- compress_segmentby is hardcoded empty regardless of tuning.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "timescaledb_no_unique_check_rendering", + "TimescaleDBAdapter hypertable and compression paths consume no UNIQUE/CHECK tuning (timescaledb.py).", + ), }, "questdb": { _T.PARTITIONING: _none( @@ -557,6 +909,19 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "QuestDB is single-node; the generator only warning-logs distribution hints and ignores the " "columns (questdb.py:193-199) instead of computing a clause, even for dry-run preview.", ), + _T.PRIMARY_KEYS: _none( + "questdb_strips_primary_keys", + "QuestDBAdapter.create_schema strips PRIMARY KEY clauses via _strip_pk_constraints (questdb.py).", + ), + _T.FOREIGN_KEYS: _none( + "questdb_strips_foreign_keys", + "QuestDBAdapter.create_schema strips FOREIGN KEY clauses via ddl_helpers strip_foreign_keys (questdb.py).", + ), + **_no_unique_check_entries( + "questdb_no_unique_check_rendering", + "QuestDB schema enhancement maps hardcoded timestamp/partition columns only and builds no " + "UNIQUE/CHECK clause (questdb.py).", + ), }, "pg-duckdb": { _T.PARTITIONING: _none( @@ -594,6 +959,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "post_create_statements via the inherited no-op apply_table_tunings/generate_tuning_clause " "path.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "pg_duckdb_no_unique_check_rendering", + "PgDuckDBAdapter inherits PostgreSQLAdapter execution carrying PK/FK only (pg_duckdb.py).", + ), }, "pg-mooncake": { _T.PARTITIONING: _none( @@ -617,6 +987,11 @@ def _constraint_entries() -> dict[TuningType, TuningCapability]: "Same as PARTITIONING: the generator always returns empty clauses, and " "supports_tuning_type() is unconditionally False.", ), + **_pk_fk_entries(), + **_no_unique_check_entries( + "pg_mooncake_no_unique_check_rendering", + "PgMooncakeAdapter inherits PostgreSQLAdapter execution carrying PK/FK only (pg_mooncake.py).", + ), }, } @@ -641,7 +1016,7 @@ def interface_compatibility_map() -> dict[str, frozenset[TuningType]]: def resolve_platform_key(platform: str) -> str: - platform_key = platform.lower().replace("_", "-") + platform_key = platform.split(":", 1)[0].strip().lower().replace("_", "-") return PLATFORM_ALIASES.get(platform_key, platform_key) @@ -653,6 +1028,37 @@ def get_capability(platform: str, tuning_type: TuningType) -> TuningCapability | return entries.get(tuning_type) +def is_not_rendered_at_execution(capability: TuningCapability | None) -> bool: + if capability is None: + return False + return capability.rendered_via == "none" or ":preview_only" in capability.mechanism_id + + +def not_rendered_reason(capability: TuningCapability) -> str: + mechanism_id = capability.mechanism_id + if "strips" in mechanism_id: + reason = "the platform adapter removes it from CREATE TABLE statements" + elif "unconstrained" in mechanism_id: + reason = "the schema is created without constraints" + elif "no_unique_check_rendering" in mechanism_id: + reason = "table DDL never carries this constraint" + elif "preview" in mechanism_id or mechanism_id.endswith("hint_only"): + reason = "it is generated only for dry-run previews and is not applied when tables are created" + elif "log_only" in mechanism_id: + reason = "the adapter only logs it and does not apply it" + else: + reason = "BenchBox does not apply it on this platform" + return f"{reason} ({mechanism_id})" + + +def unapplied_reason(capability: TuningCapability, sorted_ingestion_mode: str | None) -> str | None: + if is_not_rendered_at_execution(capability): + return not_rendered_reason(capability) + if capability.requires_sorted_ingestion and sorted_ingestion_mode in (None, "off"): + return "it runs only when sorted ingestion is enabled; set sorted_ingestion_mode to auto or force" + return None + + def known_registry_platforms() -> frozenset[str]: return frozenset(PLATFORM_TUNING_CAPABILITIES) diff --git a/benchbox/core/tuning/interface.py b/benchbox/core/tuning/interface.py index 4c18544ef4..741d706dce 100644 --- a/benchbox/core/tuning/interface.py +++ b/benchbox/core/tuning/interface.py @@ -1158,6 +1158,13 @@ def disable_platform_optimization(self, optimization_type: TuningType) -> None: elif optimization_type == TuningType.MATERIALIZED_VIEWS: self.platform_optimizations.materialized_views_enabled = False + def disable_tuning_type(self, tuning_type: TuningType) -> None: + for attr, checked_type in self._CONSTRAINT_CHECKS: + if checked_type == tuning_type: + getattr(self, attr).enabled = False + return + self.disable_platform_optimization(tuning_type) + _CONSTRAINT_CHECKS = [ ("primary_keys", TuningType.PRIMARY_KEYS), ("foreign_keys", TuningType.FOREIGN_KEYS), @@ -1208,13 +1215,24 @@ def validate_for_platform_detailed( errors: list[str] = [] warnings: list[str] = [] enabled_types = self.get_enabled_tuning_types() - platform_known = TuningType.is_known_platform(platform) + + from benchbox.core.tuning.capability_registry import ( + get_capability, + resolve_platform_key, + unapplied_reason, + ) for tuning_type in enabled_types: + capability = get_capability(platform, tuning_type) + if capability is not None: + reason = unapplied_reason(capability, self.platform_optimizations.sorted_ingestion_mode) + if reason is not None: + errors.append(f"requested {tuning_type.value} is not applied at execution on {platform}: {reason}") + continue if tuning_type.is_compatible_with_platform(platform): continue message = f"Tuning type '{tuning_type.value}' is not supported by platform '{platform}'" - if tuning_type in _CONSTRAINT_TUNING_TYPES or not platform_known: + if tuning_type in _CONSTRAINT_TUNING_TYPES: warnings.append(message) else: errors.append(message) @@ -1225,13 +1243,12 @@ def validate_for_platform_detailed( f"'spark' session-config map is not applied by platform '{platform}'; " "run on spark, velox, or lakesail to apply it" ) - validator = _get_effective_layout_validator(platform_key) + resolved_key = resolve_platform_key(platform_key) + validator = _get_effective_layout_validator(resolved_key) if validator is not None: errors.extend(validator(self)) if schema_primary_keys: - from benchbox.core.tuning.capability_registry import resolve_platform_key - - if resolve_platform_key(platform_key) == "clickhouse": + if resolved_key == "clickhouse": errors.extend(_clickhouse_primary_key_prefix_errors(self, schema_primary_keys)) return errors, warnings diff --git a/benchbox/core/tuning/reconciliation.py b/benchbox/core/tuning/reconciliation.py index bd2d55c98a..c12ede442b 100644 --- a/benchbox/core/tuning/reconciliation.py +++ b/benchbox/core/tuning/reconciliation.py @@ -13,7 +13,12 @@ AppliedStatement, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import TuningCapability, get_capability +from benchbox.core.tuning.capability_registry import ( + TuningCapability, + get_capability, + is_not_rendered_at_execution, + not_rendered_reason, +) from benchbox.core.tuning.interface import TuningType, UnifiedTuningConfiguration from benchbox.core.tuning.introspection import normalize_identifier, statement_index_columns, statement_table @@ -234,8 +239,8 @@ def _realized_in_ddl(capability: TuningCapability | None, statement: AppliedStat def _drop_reason(capability: TuningCapability | None) -> str: if capability is None: return NOT_RENDERED_REASON - if capability.rendered_via == "none" or ":preview_only" in capability.mechanism_id: - return capability.notes or f"not rendered at execution ({capability.mechanism_id})" + if is_not_rendered_at_execution(capability): + return f"not rendered at execution: {not_rendered_reason(capability)}" return NOT_RENDERED_REASON diff --git a/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml b/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml index a9020c2e95..e544227573 100644 --- a/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/joinorder_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml b/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml index d161385e6a..cdc3c2b886 100644 --- a/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/read_primitives_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml b/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml index 0fb33bc1e0..65e36458ae 100644 --- a/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/ssb_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml index 894bca10b8..22f5d468c9 100644 --- a/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpcds_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml index 53328c6589..65781c2969 100644 --- a/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpch_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml b/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml index 6c6ae3c6aa..c0cccc78ff 100644 --- a/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml +++ b/benchbox/core/tuning/templates/duckdb/tpchavoc_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml b/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml index 882d9e1996..db13f0084c 100644 --- a/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml +++ b/benchbox/core/tuning/templates/snowflake/tpcds_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml b/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml index 9a9a875028..c50c549007 100644 --- a/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml +++ b/benchbox/core/tuning/templates/snowflake/tpch_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/examples/tunings/duckdb/joinorder_tuned.yaml b/examples/tunings/duckdb/joinorder_tuned.yaml index a9020c2e95..e544227573 100644 --- a/examples/tunings/duckdb/joinorder_tuned.yaml +++ b/examples/tunings/duckdb/joinorder_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/read_primitives_tuned.yaml b/examples/tunings/duckdb/read_primitives_tuned.yaml index d161385e6a..cdc3c2b886 100644 --- a/examples/tunings/duckdb/read_primitives_tuned.yaml +++ b/examples/tunings/duckdb/read_primitives_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/ssb_tuned.yaml b/examples/tunings/duckdb/ssb_tuned.yaml index 0fb33bc1e0..65e36458ae 100644 --- a/examples/tunings/duckdb/ssb_tuned.yaml +++ b/examples/tunings/duckdb/ssb_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpcds_tuned.yaml b/examples/tunings/duckdb/tpcds_tuned.yaml index 894bca10b8..22f5d468c9 100644 --- a/examples/tunings/duckdb/tpcds_tuned.yaml +++ b/examples/tunings/duckdb/tpcds_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpch_tuned.yaml b/examples/tunings/duckdb/tpch_tuned.yaml index 53328c6589..65781c2969 100644 --- a/examples/tunings/duckdb/tpch_tuned.yaml +++ b/examples/tunings/duckdb/tpch_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/duckdb/tpchavoc_tuned.yaml b/examples/tunings/duckdb/tpchavoc_tuned.yaml index 6c6ae3c6aa..c0cccc78ff 100644 --- a/examples/tunings/duckdb/tpchavoc_tuned.yaml +++ b/examples/tunings/duckdb/tpchavoc_tuned.yaml @@ -10,7 +10,7 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: diff --git a/examples/tunings/snowflake/tpcds_tuned.yaml b/examples/tunings/snowflake/tpcds_tuned.yaml index 882d9e1996..db13f0084c 100644 --- a/examples/tunings/snowflake/tpcds_tuned.yaml +++ b/examples/tunings/snowflake/tpcds_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/examples/tunings/snowflake/tpch_tuned.yaml b/examples/tunings/snowflake/tpch_tuned.yaml index 9a9a875028..c50c549007 100644 --- a/examples/tunings/snowflake/tpch_tuned.yaml +++ b/examples/tunings/snowflake/tpch_tuned.yaml @@ -10,11 +10,11 @@ foreign_keys: on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/scripts/generate_cloud_tpc_templates.py b/scripts/generate_cloud_tpc_templates.py index 3bafa8a023..89607e8f1b 100644 --- a/scripts/generate_cloud_tpc_templates.py +++ b/scripts/generate_cloud_tpc_templates.py @@ -140,11 +140,11 @@ def snowflake_table_candidates(benchmark: str, table: str, candidates: list) -> on_update_action: RESTRICT unique_constraints: - enabled: true + enabled: false ignore_nulls: false check_constraints: - enabled: true + enabled: false enforce_on_insert: true enforce_on_update: true diff --git a/tests/unit/cli/commands/test_run_tuning_mode_guard.py b/tests/unit/cli/commands/test_run_tuning_mode_guard.py index 5dda4d6845..8b1c76da86 100644 --- a/tests/unit/cli/commands/test_run_tuning_mode_guard.py +++ b/tests/unit/cli/commands/test_run_tuning_mode_guard.py @@ -52,8 +52,8 @@ def test_dataframe_run_loads_without_the_sql_guard(tmp_path): ("mode", "source"), [(TuningMode.TUNED, TuningSource.FALLBACK), (TuningMode.AUTO, TuningSource.SMART_DEFAULTS)], ) -@pytest.mark.parametrize(("platform", "check_enabled"), [("duckdb", False), ("postgresql", True)]) -def test_fallback_and_auto_request_check_constraints_except_on_duckdb(mode, source, platform, check_enabled): +@pytest.mark.parametrize("platform", ["duckdb", "postgresql"]) +def test_fallback_and_auto_do_not_request_unrendered_check_constraints(mode, source, platform): s = SimpleNamespace( tuning_resolution=SimpleNamespace(config_file=None, source=source, mode=mode), resolved_mode="sql", @@ -67,5 +67,6 @@ def test_fallback_and_auto_request_check_constraints_except_on_duckdb(mode, sour _load_unified_tuning_config(s) - assert s.loaded_unified_config.check_constraints.enabled is check_enabled + assert s.loaded_unified_config.check_constraints.enabled is False + assert s.loaded_unified_config.unique_constraints.enabled is False assert s.loaded_unified_config.primary_keys.enabled is True diff --git a/tests/unit/cli/commands/test_tuning_group_cli.py b/tests/unit/cli/commands/test_tuning_group_cli.py index 8a2b5ed5ca..63cf6da6f2 100644 --- a/tests/unit/cli/commands/test_tuning_group_cli.py +++ b/tests/unit/cli/commands/test_tuning_group_cli.py @@ -315,7 +315,7 @@ def test_shipped_duckdb_template_validates(self): assert result.exit_code == 0, output assert "Configuration is valid for duckdb" in output - assert "Enabled tuning types: 4" in output + assert "Enabled tuning types: 3" in output def test_sql_validation_goes_through_validate_for_platform_detailed(self): template = REPO_ROOT / "examples/tunings/duckdb/tpch_tuned.yaml" @@ -340,7 +340,7 @@ def test_incompatible_tuning_type_is_an_error(self): assert "Tuning type 'sorting' is not supported by platform 'bigquery'" in output assert "Configuration has errors" in output - def test_registry_unrendered_type_is_reported_as_warning(self, tmp_path): + def test_registry_unrendered_type_is_an_error(self, tmp_path): template = tmp_path / "databricks_distribution.yaml" template.write_text( "primary_keys:\n" @@ -363,9 +363,9 @@ def test_registry_unrendered_type_is_reported_as_warning(self, tmp_path): result, output = _run_validate([str(template), "--platform", "databricks"]) - assert result.exit_code == 0, output - assert "'distribution' is accepted by platform 'databricks' but not rendered yet" in output - assert "Configuration is valid but has warnings" in output + assert result.exit_code != 0 + assert "requested distribution is not applied at execution on databricks" in output + assert "Configuration has errors" in output def test_platform_alias_resolves_to_registry_key(self): template = REPO_ROOT / "examples/tunings/duckdb/tpch_tuned.yaml" @@ -409,12 +409,22 @@ def test_empty_file_is_rejected(self, tmp_path): class TestTuningValidateMode: def test_sql_datafusion_file_validates_with_mode_sql(self, tmp_path): target = tmp_path / "datafusion_tuned.yaml" - target.write_text("table_tunings: {}\n") + target.write_text( + "primary_keys:\n" + " enabled: false\n" + "foreign_keys:\n" + " enabled: false\n" + "unique_constraints:\n" + " enabled: false\n" + "check_constraints:\n" + " enabled: false\n" + "table_tunings: {}\n" + ) result, output = _run_validate([str(target), "--platform", "datafusion", "--mode", "sql"]) assert result.exit_code == 0, output - assert "Enabled tuning types: 4" in output + assert "Enabled tuning types: 0" in output def test_dataframe_file_rejected_with_mode_sql(self): template = REPO_ROOT / "examples/tunings/dataframe/polars_optimized.yaml" diff --git a/tests/unit/cli/test_config_coverage.py b/tests/unit/cli/test_config_coverage.py index 2599f37e02..9c676ac1a2 100644 --- a/tests/unit/cli/test_config_coverage.py +++ b/tests/unit/cli/test_config_coverage.py @@ -136,11 +136,19 @@ def test_tuning_config_parse_save_and_validate(monkeypatch: pytest.MonkeyPatch, u = cfg.UnifiedTuningConfiguration() u.primary_keys.enabled = True u.foreign_keys.enabled = True + u.unique_constraints.enabled = False + u.check_constraints.enabled = False unified = tmp_path / "unified.yaml" mgr.save_unified_tuning_config(u, unified, format="yaml") loaded = mgr.load_unified_tuning_config(unified, platform="duckdb") assert loaded.primary_keys.enabled is True + bare = cfg.UnifiedTuningConfiguration() + bare_path = tmp_path / "bare.yaml" + mgr.save_unified_tuning_config(bare, bare_path, format="yaml") + with pytest.raises(ValueError): + mgr.load_unified_tuning_config(bare_path, platform="duckdb") + def test_show_config_prints_path_and_yaml(tmp_path: Path) -> None: mgr = cfg.ConfigManager(config_path=tmp_path / "config.yaml") diff --git a/tests/unit/cli/test_tuning_runtime.py b/tests/unit/cli/test_tuning_runtime.py index 6943258492..b23d82d989 100644 --- a/tests/unit/cli/test_tuning_runtime.py +++ b/tests/unit/cli/test_tuning_runtime.py @@ -59,16 +59,41 @@ def test_build_baseline_unified_config_reports_no_clustering_strategy() -> None: assert config.platform_optimizations.liquid_clustering_enabled is False -@pytest.mark.parametrize("platform", ["duckdb", "DuckDB", "duckdb:memory"]) -def test_default_unified_config_disables_check_constraints_on_duckdb(platform: str) -> None: +@pytest.mark.parametrize("platform", ["duckdb", "DuckDB"]) +def test_default_unified_config_disables_unique_and_check_constraints_on_duckdb(platform: str) -> None: config = build_default_unified_config(platform) assert config.check_constraints.enabled is False assert config.primary_keys.enabled is True assert config.foreign_keys.enabled is True - assert config.unique_constraints.enabled is True + assert config.unique_constraints.enabled is False -@pytest.mark.parametrize("platform", [None, "motherduck", "postgresql", "snowflake"]) -def test_default_unified_config_keeps_check_constraints_elsewhere(platform: str | None) -> None: - assert build_default_unified_config(platform).to_dict() == UnifiedTuningConfiguration().to_dict() +def test_default_unified_config_shares_base_rules_on_duckdb_memory_variant() -> None: + config = build_default_unified_config("duckdb:memory") + + assert config.check_constraints.enabled is False + assert config.unique_constraints.enabled is False + + +def test_default_unified_config_without_platform_keeps_all_constraints() -> None: + assert build_default_unified_config(None).to_dict() == UnifiedTuningConfiguration().to_dict() + + +@pytest.mark.parametrize("platform", ["postgresql", "snowflake"]) +def test_default_unified_config_disables_unrendered_unique_and_check(platform: str) -> None: + config = build_default_unified_config(platform) + + assert config.primary_keys.enabled is True + assert config.foreign_keys.enabled is True + assert config.unique_constraints.enabled is False + assert config.check_constraints.enabled is False + + +def test_default_unified_config_disables_unrendered_constraints_on_motherduck() -> None: + config = build_default_unified_config("motherduck") + + assert config.primary_keys.enabled is False + assert config.foreign_keys.enabled is False + assert config.unique_constraints.enabled is False + assert config.check_constraints.enabled is False diff --git a/tests/unit/core/tuning/test_capability_registry_parity.py b/tests/unit/core/tuning/test_capability_registry_parity.py new file mode 100644 index 0000000000..4eceb0fedd --- /dev/null +++ b/tests/unit/core/tuning/test_capability_registry_parity.py @@ -0,0 +1,767 @@ +from __future__ import annotations + +import inspect +import re + +import pytest +import yaml + +from benchbox.core.tuning.capability_registry import ( + PLATFORM_ALIASES, + PLATFORM_TUNING_CAPABILITIES, + get_capability, + is_not_rendered_at_execution, + known_registry_platforms, + not_rendered_reason, + resolve_platform_key, +) +from benchbox.core.tuning.interface import TableTuning, TuningColumn, TuningType, UnifiedTuningConfiguration + +pytestmark = [ + pytest.mark.unit, + pytest.mark.fast, +] + + +def _partitioned_config() -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.table_tunings["orders"] = TableTuning( + table_name="orders", + partitioning=[TuningColumn(name="o_orderdate", type="DATE", order=1)], + ) + return config + + +def _distributed_config() -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.table_tunings["lineitem"] = TableTuning( + table_name="lineitem", + distribution=[TuningColumn(name="l_orderkey", type="INTEGER", order=1)], + ) + return config + + +class TestRequestedButNotRenderedIsAnError: + def test_partitioning_on_postgresql_errors(self) -> None: + errors, _warnings = _partitioned_config().validate_for_platform_detailed("postgresql") + assert any("requested partitioning is not applied at execution on postgresql" in e for e in errors) + + def test_partitioning_on_postgres_family_errors(self) -> None: + for platform in ("cedardb", "citus", "paradedb", "pg-duckdb"): + errors, _warnings = _partitioned_config().validate_for_platform_detailed(platform) + assert any("is not applied at execution" in e for e in errors), platform + + def test_distribution_on_redshift_errors(self) -> None: + errors, _warnings = _distributed_config().validate_for_platform_detailed("redshift") + assert any("requested distribution is not applied at execution on redshift" in e for e in errors) + + def test_layout_on_trino_and_athena_errors(self) -> None: + for platform in ("trino", "athena"): + errors, _warnings = _partitioned_config().validate_for_platform_detailed(platform) + assert any("is not applied at execution" in e for e in errors), platform + + def test_starburst_alias_resolves_to_trino_rules(self) -> None: + assert resolve_platform_key("Starburst") == "trino" + errors, _warnings = _partitioned_config().validate_for_platform_detailed("starburst") + assert any("is not applied at execution" in e for e in errors) + + def test_rendering_registry_entry_does_not_error(self) -> None: + errors, warnings = _distributed_config().validate_for_platform_detailed("starrocks") + assert errors == [] + assert warnings == [] + + def test_doris_layout_and_constraints_error(self) -> None: + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("doris") + assert any("requested primary_keys is not applied at execution" in e for e in errors) + assert any("requested foreign_keys is not applied at execution" in e for e in errors) + + def test_datafusion_constraints_error(self) -> None: + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("datafusion") + assert len(errors) == 4 + + def test_motherduck_sorting_renders_but_constraints_error(self) -> None: + sort_config = UnifiedTuningConfiguration() + sort_config.disable_all_constraints() + sort_config.table_tunings["orders"] = TableTuning( + table_name="orders", + sorting=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + ) + errors, _warnings = sort_config.validate_for_platform_detailed("motherduck") + assert errors == [] + + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("motherduck") + assert any("is not applied at execution" in e for e in errors) + + def test_fabric_warehouse_partitioning_errors(self) -> None: + errors, _warnings = _partitioned_config().validate_for_platform_detailed("fabric-warehouse") + assert any("is not applied at execution" in e for e in errors) + + def test_stripped_constraint_types_error(self) -> None: + for platform in ("doris", "databend", "starrocks"): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.unique_constraints.enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any("requested unique_constraints is not applied at execution" in e for e in errors), platform + + def test_vacuous_unique_check_types_error(self) -> None: + for platform in ( + "duckdb", + "databricks", + "snowflake", + "sqlite", + "postgresql", + "mysql", + "singlestore", + "cedardb", + "citus", + "paradedb", + "fabric-warehouse", + "athena", + "azure-synapse", + "timescaledb", + "redshift", + "firebolt", + "questdb", + "pg-duckdb", + "pg-mooncake", + "trino", + "presto", + ): + for tuning_attr in ("unique_constraints", "check_constraints"): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_bigquery_unique_and_clickhouse_fk_check_error(self) -> None: + for platform, tuning_attr in ( + ("bigquery", "unique_constraints"), + ("clickhouse", "foreign_keys"), + ("clickhouse", "check_constraints"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_dropped_pk_fk_types_error(self) -> None: + for platform, tuning_attr in ( + ("firebolt", "primary_keys"), + ("firebolt", "foreign_keys"), + ("questdb", "primary_keys"), + ("questdb", "foreign_keys"), + ("trino", "primary_keys"), + ("presto", "primary_keys"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert any(f"requested {tuning_attr} is not applied at execution" in e for e in errors), ( + platform, + tuning_attr, + ) + + def test_rendered_pk_fk_types_do_not_error(self) -> None: + for platform, tuning_attr in ( + ("pg-duckdb", "primary_keys"), + ("pg-duckdb", "foreign_keys"), + ("pg-mooncake", "primary_keys"), + ("pg-mooncake", "foreign_keys"), + ("trino", "foreign_keys"), + ("presto", "foreign_keys"), + ): + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + getattr(config, tuning_attr).enabled = True + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], (platform, tuning_attr, errors) + + +_NOT_RENDERED: dict[str, frozenset[str]] = { + "athena": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "azure-synapse": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "bigquery": frozenset( + {"check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "cedardb": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "citus": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "clickhouse": frozenset({"check_constraints", "foreign_keys", "materialized_views", "unique_constraints"}), + "databend": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "materialized_views", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "databricks": frozenset( + { + "bloom_filters", + "check_constraints", + "distribution", + "materialized_views", + "partitioning", + "unique_constraints", + } + ), + "datafusion": frozenset({"check_constraints", "foreign_keys", "primary_keys", "unique_constraints"}), + "doris": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "duckdb": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "fabric-warehouse": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "firebolt": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "motherduck": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "foreign_keys", + "materialized_views", + "partitioning", + "primary_keys", + "unique_constraints", + } + ), + "mysql": frozenset({"check_constraints", "partitioning", "unique_constraints"}), + "paradedb": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "pg-duckdb": frozenset({"check_constraints", "clustering", "partitioning", "sorting", "unique_constraints"}), + "pg-mooncake": frozenset({"check_constraints", "clustering", "partitioning", "sorting", "unique_constraints"}), + "postgresql": frozenset( + {"bloom_filters", "check_constraints", "clustering", "materialized_views", "partitioning", "unique_constraints"} + ), + "presto": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "questdb": frozenset( + { + "check_constraints", + "distribution", + "foreign_keys", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), + "redshift": frozenset( + {"check_constraints", "distribution", "materialized_views", "partitioning", "unique_constraints"} + ), + "singlestore": frozenset({"check_constraints", "foreign_keys", "partitioning", "unique_constraints"}), + "snowflake": frozenset({"check_constraints", "materialized_views", "unique_constraints"}), + "sqlite": frozenset({"check_constraints", "unique_constraints"}), + "starrocks": frozenset({"check_constraints", "foreign_keys", "unique_constraints"}), + "timescaledb": frozenset( + {"check_constraints", "clustering", "distribution", "partitioning", "sorting", "unique_constraints"} + ), + "trino": frozenset( + { + "check_constraints", + "clustering", + "distribution", + "partitioning", + "primary_keys", + "sorting", + "unique_constraints", + } + ), +} +_RENDERED: dict[str, frozenset[str]] = { + "athena": frozenset({"foreign_keys", "primary_keys"}), + "azure-synapse": frozenset({"foreign_keys", "primary_keys"}), + "bigquery": frozenset({"foreign_keys", "primary_keys"}), + "cedardb": frozenset({"foreign_keys", "primary_keys"}), + "citus": frozenset({"foreign_keys", "primary_keys"}), + "clickhouse": frozenset({"clustering", "partitioning", "primary_keys", "sorting"}), + "databend": frozenset({"clustering"}), + "databricks": frozenset( + { + "auto_compact", + "auto_optimize", + "clustering", + "foreign_keys", + "liquid_clustering", + "primary_keys", + "z_ordering", + } + ), + "datafusion": frozenset({}), + "doris": frozenset({}), + "duckdb": frozenset({"foreign_keys", "primary_keys", "sorting"}), + "fabric-warehouse": frozenset({"foreign_keys", "primary_keys"}), + "firebolt": frozenset({}), + "motherduck": frozenset({"sorting"}), + "mysql": frozenset({"foreign_keys", "primary_keys"}), + "paradedb": frozenset({"foreign_keys", "primary_keys"}), + "pg-duckdb": frozenset({"foreign_keys", "primary_keys"}), + "pg-mooncake": frozenset({"foreign_keys", "primary_keys"}), + "postgresql": frozenset({"foreign_keys", "primary_keys"}), + "presto": frozenset({"foreign_keys"}), + "questdb": frozenset({}), + "redshift": frozenset({"foreign_keys", "primary_keys", "sorting"}), + "singlestore": frozenset({"primary_keys"}), + "snowflake": frozenset({"clustering", "foreign_keys", "partitioning", "primary_keys"}), + "sqlite": frozenset({"foreign_keys", "primary_keys"}), + "starrocks": frozenset({"distribution", "partitioning", "primary_keys", "sorting"}), + "timescaledb": frozenset({"foreign_keys", "primary_keys"}), + "trino": frozenset({"foreign_keys"}), +} + + +_SOURCE_LOCATION = re.compile(r"\.py:\d+") + +_LAYOUT_TYPES = frozenset({TuningType.PARTITIONING, TuningType.CLUSTERING, TuningType.DISTRIBUTION, TuningType.SORTING}) + + +def _config_requesting(tuning_type: TuningType) -> UnifiedTuningConfiguration: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.platform_optimizations.sorted_ingestion_mode = "auto" + for attribute, _checked in UnifiedTuningConfiguration._OPT_CHECKS: + setattr(config.platform_optimizations, attribute, False) + if tuning_type in _LAYOUT_TYPES: + column = [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + config.table_tunings["orders"] = TableTuning(table_name="orders", **{tuning_type.value: column}) + return config + for attribute, checked in UnifiedTuningConfiguration._CONSTRAINT_CHECKS: + if checked == tuning_type: + getattr(config, attribute).enabled = True + return config + for attribute, checked in UnifiedTuningConfiguration._OPT_CHECKS: + if checked == tuning_type: + setattr(config.platform_optimizations, attribute, True) + return config + raise AssertionError(f"no way to request {tuning_type}") + + +class TestMechanismPreconditions: + def test_redshift_sorting_requires_sorted_ingestion(self) -> None: + config = _config_requesting(TuningType.SORTING) + config.platform_optimizations.sorted_ingestion_mode = "off" + errors, _warnings = config.validate_for_platform_detailed("redshift") + assert any("sorted_ingestion_mode" in error and error.startswith("requested sorting") for error in errors) + + config.platform_optimizations.sorted_ingestion_mode = "auto" + errors, _warnings = config.validate_for_platform_detailed("redshift") + assert not any(error.startswith("requested sorting") for error in errors) + + def test_only_redshift_sorting_is_gated_on_sorted_ingestion(self) -> None: + gated = { + (platform, tuning_type.value) + for platform, entries in PLATFORM_TUNING_CAPABILITIES.items() + for tuning_type, capability in entries.items() + if capability.requires_sorted_ingestion + } + assert gated == {("redshift", "sorting")} + + @pytest.mark.parametrize("alias", ["spark", "delta", "databricks:sql", "Databricks"]) + def test_databricks_aliases_run_the_effective_layout_validator(self, alias: str) -> None: + config = UnifiedTuningConfiguration() + config.disable_all_constraints() + config.platform_optimizations.databricks_clustering_strategy = "liquid_clustering" + config.platform_optimizations.liquid_clustering_enabled = True + config.table_tunings["orders"] = TableTuning( + table_name="orders", + clustering=[TuningColumn(name=f"c{index}", type="INTEGER", order=index) for index in range(1, 6)], + ) + reference, _ = config.validate_for_platform_detailed("databricks") + errors, _warnings = config.validate_for_platform_detailed(alias) + assert any("four" in error or "4" in error for error in reference), reference + assert [error for error in errors if "four" in error or "4" in error] == [ + error for error in reference if "four" in error or "4" in error + ] + + +class TestRegistryTable: + def test_table_covers_every_registry_platform(self) -> None: + assert set(_NOT_RENDERED) == set(_RENDERED) == known_registry_platforms() + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_registry_matches_table(self, platform: str) -> None: + entries = PLATFORM_TUNING_CAPABILITIES[platform] + not_rendered = frozenset( + tuning_type.value for tuning_type, capability in entries.items() if is_not_rendered_at_execution(capability) + ) + rendered = frozenset(tuning_type.value for tuning_type in entries) - not_rendered + assert not_rendered == _NOT_RENDERED[platform], f"{platform}: not-rendered set changed" + assert rendered == _RENDERED[platform], f"{platform}: rendered set changed" + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_requested_types_error_exactly_when_table_says_not_rendered(self, platform: str) -> None: + for tuning_value in sorted(_NOT_RENDERED[platform] | _RENDERED[platform]): + errors, _warnings = _config_requesting(TuningType(tuning_value)).validate_for_platform_detailed(platform) + refused = any(error.startswith(f"requested {tuning_value} is not applied at execution") for error in errors) + assert refused == (tuning_value in _NOT_RENDERED[platform]), (platform, tuning_value, errors) + + @pytest.mark.parametrize("platform", sorted(_NOT_RENDERED)) + def test_error_text_names_mechanism_without_source_locations(self, platform: str) -> None: + for tuning_value in sorted(_NOT_RENDERED[platform]): + tuning_type = TuningType(tuning_value) + capability = get_capability(platform, tuning_type) + assert capability is not None + errors, _warnings = _config_requesting(tuning_type).validate_for_platform_detailed(platform) + refusal = next(error for error in errors if error.startswith(f"requested {tuning_value} is not applied")) + assert capability.mechanism_id in refusal + assert not _SOURCE_LOCATION.search(refusal), refusal + assert not _SOURCE_LOCATION.search(not_rendered_reason(capability)) + + def test_alias_platforms_resolve_to_registry_entries(self) -> None: + for alias, target in PLATFORM_ALIASES.items(): + if target in known_registry_platforms(): + assert resolve_platform_key(alias) == target + + +_GENERATOR_QUIET_FIELDS: dict[tuple[str, str], str] = { + ("databricks", "distribution"): "distribute_by", + ("firebolt", "sorting"): "sort_by", + ("firebolt", "clustering"): "cluster_by", + ("questdb", "sorting"): "sort_by", + ("questdb", "distribution"): "distribute_by", + ("azure-synapse", "sorting"): "sort_by", + ("azure-synapse", "clustering"): "cluster_by", +} + + +_DATABEND_QUIET_KINDS: frozenset[str] = frozenset({"partitioning", "sorting", "distribution"}) + + +_VACUITY_SIGNATURE_ONLY: dict[tuple[str, str], str] = { + ("tpcds_obt", "TPCDSOBT"): "get_create_tables_sql accepts no tuning_config", + ("core.joinorder.benchmark", "JoinOrderSchema"): "schema helper; get_create_tables_sql accepts no tuning_config", +} + + +_DISCOVERY_IMPORT_SKIPS: dict[str, str] = {} + + +def _benchmark_classes_in(module: object, module_key: str, seen: set[int]) -> list[tuple[str, str, type]]: + found: list[tuple[str, str, type]] = [] + for attribute in sorted(vars(module)): + value = vars(module)[attribute] + if ( + inspect.isclass(value) + and hasattr(value, "get_create_tables_sql") + and value.__module__.split(".")[0] == "benchbox" + and id(value) not in seen + ): + seen.add(id(value)) + found.append((module_key, attribute, value)) + return found + + +def _discovered_benchmarks() -> list[tuple[str, str, type]]: + import importlib + import pkgutil + + import benchbox + import benchbox.core + + found: list[tuple[str, str, type]] = [] + seen: set[int] = set() + for module in sorted(pkgutil.iter_modules(benchbox.__path__), key=lambda entry: entry.name): + imported = importlib.import_module(f"benchbox.{module.name}") + found.extend(_benchmark_classes_in(imported, module.name, seen)) + failures: dict[str, str] = {} + for module in sorted(pkgutil.walk_packages(benchbox.core.__path__, "benchbox.core."), key=lambda e: e.name): + if module.name in _DISCOVERY_IMPORT_SKIPS: + continue + try: + imported = importlib.import_module(module.name) + except ImportError as exc: + failures[module.name] = str(exc) + continue + found.extend(_benchmark_classes_in(imported, module.name.removeprefix("benchbox."), seen)) + assert not failures, f"benchbox.core modules failed to import; fix or list in _DISCOVERY_IMPORT_SKIPS: {failures}" + return found + + +def _vacuous_capable_benchmarks() -> list[tuple[str, str, object]]: + capable: list[tuple[str, str, object]] = [] + for module_name, class_name, benchmark_cls in _discovered_benchmarks(): + if (module_name, class_name) in _VACUITY_SIGNATURE_ONLY: + parameters = inspect.signature(benchmark_cls.get_create_tables_sql).parameters + assert "tuning_config" not in parameters, ( + f"{module_name}.{class_name} now accepts tuning_config; cover it behaviorally" + ) + continue + try: + benchmark = benchmark_cls() + except Exception as exc: + pytest.fail( + f"{module_name}.{class_name} needs explicit construction ({exc}); construct it in " + "_vacuous_capable_benchmarks or classify it in _VACUITY_SIGNATURE_ONLY" + ) + parameters = inspect.signature(benchmark.get_create_tables_sql).parameters + assert "tuning_config" in parameters, ( + f"{module_name}.{class_name} accepts no tuning_config; classify it in _VACUITY_SIGNATURE_ONLY" + ) + capable.append((module_name, class_name, benchmark)) + assert capable, "no behaviorally covered benchmarks; the vacuity proof is empty" + return capable + + +class TestBenchmarkDdlAndAdapterBehavior: + def test_unique_check_rendering_is_vacuous_in_tree(self) -> None: + for module_name, class_name, benchmark in _vacuous_capable_benchmarks(): + plain = UnifiedTuningConfiguration() + plain.disable_all_constraints() + toggled = UnifiedTuningConfiguration() + toggled.disable_all_constraints() + toggled.unique_constraints.enabled = True + toggled.check_constraints.enabled = True + assert benchmark.get_create_tables_sql(tuning_config=plain) == benchmark.get_create_tables_sql( + tuning_config=toggled + ), ( + f"{module_name}.{class_name} DDL now varies with UNIQUE/CHECK flags; the not-rendered " + "UNIQUE/CHECK registry entries must be re-audited against execution" + ) + + def test_benchmark_ddl_ignores_layout_flags_in_tree(self) -> None: + for module_name, class_name, benchmark in _vacuous_capable_benchmarks(): + plain = UnifiedTuningConfiguration() + plain.disable_all_constraints() + toggled = UnifiedTuningConfiguration() + toggled.disable_all_constraints() + toggled.table_tunings["orders"] = TableTuning( + table_name="orders", + partitioning=[TuningColumn(name="o_orderdate", type="DATE", order=1)], + sorting=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + clustering=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + distribution=[TuningColumn(name="o_orderkey", type="INTEGER", order=1)], + ) + assert benchmark.get_create_tables_sql(tuning_config=plain) == benchmark.get_create_tables_sql( + tuning_config=toggled + ), ( + f"{module_name}.{class_name} DDL now varies with layout tuning flags; the not-rendered " + "layout registry entries must be re-audited against execution" + ) + + def test_vacuity_discovery_covers_all_benchmarks(self) -> None: + found = {(module_name, class_name) for module_name, class_name, _ in _discovered_benchmarks()} + assert found, "benchmark discovery is empty; the vacuity proof covers nothing" + known = { + ("tpch", "TPCH"), + ("tpcds", "TPCDS"), + ("ssb", "SSB"), + ("clickbench", "ClickBench"), + ("core.ai_primitives", "AIPrimitivesBenchmark"), + } + assert known <= found, f"discovery lost known benchmarks: {sorted(known - found)}" + for key in _VACUITY_SIGNATURE_ONLY: + assert key in found, f"signature-only benchmark {key} disappeared from discovery" + for module_name, class_name in sorted(found - set(_VACUITY_SIGNATURE_ONLY)): + assert any( + module_name == seen_module and class_name == seen_class + for seen_module, seen_class, _ in _vacuous_capable_benchmarks() + ), ( + f"{module_name}.{class_name} is not covered by the vacuity proof; construct it in " + "_vacuous_capable_benchmarks or classify it in _VACUITY_SIGNATURE_ONLY" + ) + + def test_questdb_create_schema_strips_constraints(self) -> None: + from benchbox.platforms.base.ddl_helpers import strip_foreign_keys + from benchbox.platforms.questdb import QuestDBAdapter + + adapter = QuestDBAdapter.__new__(QuestDBAdapter) + create = "CREATE TABLE orders (o_orderkey INTEGER, o_custkey INTEGER, PRIMARY KEY (o_orderkey));" + assert "PRIMARY KEY" not in QuestDBAdapter._strip_pk_constraints(adapter, create) + assert "o_orderkey INTEGER" in QuestDBAdapter._strip_pk_constraints(adapter, create) + foreign = "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x));" + assert "FOREIGN KEY" not in strip_foreign_keys(foreign) + + def test_trino_base_optimize_strips_primary_keys(self) -> None: + from benchbox.platforms.presto_trino_adapter_base import PrestoTrinoAdapterBase + + adapter = PrestoTrinoAdapterBase.__new__(PrestoTrinoAdapterBase) + adapter.table_format = "hive" + adapter.catalog = "catalog" + create = "CREATE TABLE orders (o_orderkey INTEGER, PRIMARY KEY (o_orderkey));" + assert "PRIMARY KEY" not in PrestoTrinoAdapterBase._optimize_table_definition(adapter, create) + foreign = "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x));" + assert "FOREIGN KEY" in PrestoTrinoAdapterBase._optimize_table_definition(adapter, foreign) + + def test_clickhouse_optimize_preserves_foreign_keys(self) -> None: + from benchbox.platforms.clickhouse.workload import ClickHouseWorkloadMixin + + mixin = ClickHouseWorkloadMixin.__new__(ClickHouseWorkloadMixin) + create = ( + "CREATE TABLE orders (o_orderkey INTEGER, FOREIGN KEY (o_orderkey) REFERENCES c (x)) " + "ENGINE = MergeTree() ORDER BY tuple();" + ) + assert "FOREIGN KEY" in ClickHouseWorkloadMixin._optimize_table_definition(mixin, create, None) + + def test_azure_synapse_optimize_uses_engine_default(self) -> None: + from benchbox.platforms.azure_synapse import AzureSynapseAdapter + + adapter = AzureSynapseAdapter.__new__(AzureSynapseAdapter) + adapter.schema = "dbo" + adapter.distribution_default = "ROUND_ROBIN" + create = "CREATE TABLE orders (o_orderkey INTEGER, tuned_dist INTEGER);" + rendered = AzureSynapseAdapter._optimize_table_definition(adapter, create) + assert "DISTRIBUTION = ROUND_ROBIN" in rendered + assert "tuned_dist" not in rendered.split("WITH (DISTRIBUTION")[1] + assert "PARTITION" not in rendered.upper() + + def test_questdb_enhancement_takes_no_tuning(self) -> None: + from benchbox.platforms.questdb import QuestDBAdapter + + parameters = inspect.signature(QuestDBAdapter._apply_questdb_schema_enhancements).parameters + assert "tuning_config" not in parameters and "table_tuning" not in parameters + adapter = QuestDBAdapter.__new__(QuestDBAdapter) + adapter.partition_by = None + create = "CREATE TABLE lineitem (l_orderkey INTEGER, l_shipdate DATE);" + rendered = QuestDBAdapter._apply_questdb_schema_enhancements(adapter, create) + assert "PARTITION BY MONTH" in rendered.upper() + + def test_quiet_generators_emit_no_clause(self) -> None: + from benchbox.core.tuning.ddl_generator import get_ddl_generator + + for (platform_key, tuning_value), field in sorted(_GENERATOR_QUIET_FIELDS.items()): + tuning: dict[str, list[TuningColumn]] = { + tuning_value: [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + } + clauses = get_ddl_generator(platform_key).generate_tuning_clauses(TableTuning(table_name="t", **tuning)) + assert getattr(clauses, field) is None, ( + f"{platform_key}/{tuning_value} generator emits a {field} clause from tuned columns" + ) + + def test_databend_generate_ignores_non_clustering(self) -> None: + from benchbox.platforms.databend.adapter import DatabendAdapter + + for tuning_value in sorted(_DATABEND_QUIET_KINDS): + tuning: dict[str, list[TuningColumn]] = { + tuning_value: [TuningColumn(name="tuned_column", type="INTEGER", order=1)] + } + assert DatabendAdapter.generate_tuning_clause(None, TableTuning(table_name="t", **tuning)) == "" + + def test_pk_fk_flags_drive_ddl_content(self) -> None: + from benchbox.ssb import SSBBenchmark + from benchbox.tpcds import TPCDSBenchmark + from benchbox.tpch import TPCHBenchmark + + for benchmark in (TPCHBenchmark(), TPCDSBenchmark(), SSBBenchmark()): + name = type(benchmark).__name__ + disabled = UnifiedTuningConfiguration() + disabled.disable_all_constraints() + pk_only = UnifiedTuningConfiguration() + pk_only.disable_all_constraints() + pk_only.primary_keys.enabled = True + plain = benchmark.get_create_tables_sql(tuning_config=disabled) + keyed = benchmark.get_create_tables_sql(tuning_config=pk_only) + assert keyed != plain, f"{name} DDL ignores the primary_keys flag" + assert "PRIMARY KEY" in keyed, f"{name} DDL carries no PRIMARY KEY clause" + assert "PRIMARY KEY" not in plain, f"{name} DDL carries PRIMARY KEY with flags off" + for benchmark in (TPCHBenchmark(), TPCDSBenchmark()): + name = type(benchmark).__name__ + disabled = UnifiedTuningConfiguration() + disabled.disable_all_constraints() + fk_only = UnifiedTuningConfiguration() + fk_only.disable_all_constraints() + fk_only.foreign_keys.enabled = True + plain = benchmark.get_create_tables_sql(tuning_config=disabled) + keyed = benchmark.get_create_tables_sql(tuning_config=fk_only) + assert keyed != plain, f"{name} DDL ignores the foreign_keys flag" + assert "FOREIGN KEY" in keyed, f"{name} DDL carries no FOREIGN KEY clause" + + def test_athena_rewrite_takes_no_tuning(self) -> None: + from benchbox.platforms.athena import AthenaAdapter + + parameters = inspect.signature(AthenaAdapter._convert_to_external_table).parameters + assert "tuning_config" not in parameters and "table_tuning" not in parameters + + def test_datafusion_schema_ignores_constraints(self) -> None: + from benchbox.platforms.datafusion import DataFusionAdapter + + parameters = inspect.signature(DataFusionAdapter._get_benchmark_schema).parameters + assert "tuning_config" not in parameters and "enable_primary_keys" not in parameters + assert "enable_foreign_keys" not in parameters + + +class TestShippedTemplatesValidateClean: + @pytest.mark.parametrize("platform", ["clickhouse", "databricks", "duckdb", "snowflake"]) + def test_packaged_templates_validate_clean(self, platform: str) -> None: + from benchbox.core.tuning.packaged_templates import list_packaged_templates + + templates = list_packaged_templates(platform) + assert templates.get(platform), f"no packaged templates found for {platform}" + for template_path in templates[platform]: + config = UnifiedTuningConfiguration.from_dict(yaml.safe_load(template_path.read_text(encoding="utf-8"))) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"{template_path}: {errors}" + + @pytest.mark.parametrize( + "platform", + ["duckdb", "clickhouse", "snowflake", "bigquery", "postgresql", "databricks", "redshift"], + ) + def test_sample_configs_validate_clean(self, tmp_path, platform: str) -> None: + from benchbox.cli.config import ConfigManager + + target = tmp_path / f"{platform}_tuning.yaml" + ConfigManager().create_sample_unified_tuning_config(target, platform) + config = UnifiedTuningConfiguration.from_dict(yaml.safe_load(target.read_text(encoding="utf-8"))) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"sample for {platform}: {errors}" + + @pytest.mark.parametrize("platform", sorted(known_registry_platforms() | {"spark", "starburst"})) + def test_fallback_config_validates_clean(self, tmp_path, platform: str) -> None: + from benchbox.cli.tuning_runtime import build_default_unified_config + + config = build_default_unified_config(platform) + errors, _warnings = config.validate_for_platform_detailed(platform) + assert errors == [], f"fallback for {platform}: {errors}" diff --git a/tests/unit/core/tuning/test_capability_source_drift.py b/tests/unit/core/tuning/test_capability_source_drift.py index a84a080496..e11589e37f 100644 --- a/tests/unit/core/tuning/test_capability_source_drift.py +++ b/tests/unit/core/tuning/test_capability_source_drift.py @@ -66,6 +66,7 @@ "clustering", "primary_keys", "foreign_keys", + "unique_constraints", "check_constraints", "materialized_views", } @@ -88,7 +89,9 @@ "sorting", "clustering", "primary_keys", + "foreign_keys", "unique_constraints", + "check_constraints", "materialized_views", } ), @@ -184,6 +187,14 @@ def test_registry_has_platforms_interface_deliberately_excludes(): "questdb", "pg-duckdb", "pg-mooncake", + "cedardb", + "citus", + "paradedb", + "singlestore", + "databend", + "datafusion", + "motherduck", + "fabric-warehouse", } diff --git a/tests/unit/core/tuning/test_clickhouse_pk_prefix.py b/tests/unit/core/tuning/test_clickhouse_pk_prefix.py index 6cce5bbbd0..ba3ed741df 100644 --- a/tests/unit/core/tuning/test_clickhouse_pk_prefix.py +++ b/tests/unit/core/tuning/test_clickhouse_pk_prefix.py @@ -86,6 +86,8 @@ def _config(table_tuning: TableTuning, primary_keys_enabled: bool) -> UnifiedTun config = UnifiedTuningConfiguration() config.primary_keys.enabled = primary_keys_enabled config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False config.table_tunings[table_tuning.table_name] = table_tuning return config diff --git a/tests/unit/core/tuning/test_fallback_description.py b/tests/unit/core/tuning/test_fallback_description.py index beaeebdf79..68295077aa 100644 --- a/tests/unit/core/tuning/test_fallback_description.py +++ b/tests/unit/core/tuning/test_fallback_description.py @@ -46,7 +46,7 @@ def test_descriptions_are_not_one_fixed_string(): def test_sql_override_keys_are_canonical_registry_platforms(): sql_overrides = {key for key in PLATFORM_FALLBACK_DESCRIPTIONS if key in PLATFORM_TUNING_CAPABILITIES} - assert sql_overrides == {"clickhouse"} + assert sql_overrides == {"clickhouse", "datafusion"} @pytest.mark.parametrize( diff --git a/tests/unit/core/tuning/test_platform_identity_keys.py b/tests/unit/core/tuning/test_platform_identity_keys.py index 8051371f28..feb1250154 100644 --- a/tests/unit/core/tuning/test_platform_identity_keys.py +++ b/tests/unit/core/tuning/test_platform_identity_keys.py @@ -73,17 +73,33 @@ class TestDisplayNameValidationNoLongerErrors: @pytest.mark.parametrize("display_name", DISPLAY_NAMES) def test_default_config_produces_no_errors_for_display_names(self, display_name): config = UnifiedTuningConfiguration() + config.disable_all_constraints() assert config.validate_for_platform(display_name) == [] - @pytest.mark.parametrize("display_name", DISPLAY_NAMES) - def test_display_name_mismatches_are_warnings_not_errors(self, display_name): + def test_doris_display_name_applies_rendered_via_rules(self): + config = UnifiedTuningConfiguration() + errors, _warnings = config.validate_for_platform_detailed("Doris") + assert any("requested primary_keys is not applied at execution" in error for error in errors) + assert any("requested foreign_keys is not applied at execution" in error for error in errors) + + def test_starrocks_display_name_validates_clean(self): + config = UnifiedTuningConfiguration() + config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False + errors, warnings = config.validate_for_platform_detailed("StarRocks") + assert errors == [] + assert warnings == [] + + @pytest.mark.parametrize("display_name", ["ClickHouse (Local)", "Apache Doris"]) + def test_unresolvable_display_name_mismatches_are_warnings_not_errors(self, display_name): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed(display_name) assert errors == [] assert len(warnings) == 4 assert all("is not supported by platform" in w for w in warnings) - def test_table_layout_tuning_on_unknown_platform_warns_not_errors(self): + def test_table_layout_tuning_on_rendering_unknown_platform_is_silent(self): config = UnifiedTuningConfiguration() config.disable_all_constraints() config.table_tunings["orders"] = TableTuning( @@ -93,7 +109,7 @@ def test_table_layout_tuning_on_unknown_platform_warns_not_errors(self): errors, warnings = config.validate_for_platform_detailed("starrocks") assert errors == [] - assert any("'distribution'" in w for w in warnings) + assert warnings == [] class TestKnownPlatformSemanticsPreserved: @@ -132,10 +148,11 @@ def test_known_platform_set_stays_in_sync_with_compatibility_map(self): for tuning_type in tuning_types: assert tuning_type.is_compatible_with_platform(platform) is True - def test_duckdb_default_config_still_validates_clean(self): + def test_duckdb_default_config_errors_on_unrendered_constraints(self): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed("duckdb") - assert errors == [] + assert any("requested unique_constraints is not applied at execution" in e for e in errors) + assert any("requested check_constraints is not applied at execution" in e for e in errors) assert warnings == [] def test_layout_incompatibility_on_known_platform_still_errors(self): @@ -158,12 +175,13 @@ def test_platform_specific_opt_on_known_platform_still_errors(self): assert any("'z_ordering'" in e for e in errors) assert config.validate_for_platform("databricks") == [] - def test_constraint_mismatch_on_known_platform_is_warning(self): + def test_requested_but_not_rendered_constraint_is_an_error(self): config = UnifiedTuningConfiguration() errors, warnings = config.validate_for_platform_detailed("clickhouse") - assert errors == [] - assert any("'foreign_keys'" in w for w in warnings) - assert any("'check_constraints'" in w for w in warnings) + assert any("requested unique_constraints is not applied at execution" in e for e in errors) + assert any("requested foreign_keys is not applied at execution" in e for e in errors) + assert any("requested check_constraints is not applied at execution" in e for e in errors) + assert warnings == [] def test_databricks_liquid_layout_validation_still_errors(self): config = UnifiedTuningConfiguration() @@ -220,9 +238,13 @@ def test_registry_constructed_adapter_resolves_aliases(self, tmp_path): def test_validate_tuning_configuration_uses_canonical_key(self): adapter = _StubAdapter(display_name="StarRocks", type="starrocks") - adapter.unified_tuning_configuration = UnifiedTuningConfiguration() + config = UnifiedTuningConfiguration() + config.foreign_keys.enabled = False + config.unique_constraints.enabled = False + config.check_constraints.enabled = False + adapter.unified_tuning_configuration = config assert adapter.validate_tuning_configuration_for_platform() == [] - assert adapter.validate_tuning_configuration(UnifiedTuningConfiguration()) == [] + assert adapter.validate_tuning_configuration(config) == [] class _MetadataStubAdapter: diff --git a/tests/unit/core/tuning/test_tuning_reconciliation.py b/tests/unit/core/tuning/test_tuning_reconciliation.py index d09c1ae5ce..8e02c65bc2 100644 --- a/tests/unit/core/tuning/test_tuning_reconciliation.py +++ b/tests/unit/core/tuning/test_tuning_reconciliation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from unittest.mock import Mock import pytest @@ -15,7 +16,7 @@ STATEMENT_FAILED, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import get_capability +from benchbox.core.tuning.capability_registry import get_capability, not_rendered_reason from benchbox.core.tuning.interface import ( PlatformOptimizationConfiguration, TableTuning, @@ -266,7 +267,8 @@ def test_backtick_quoted_starrocks_clauses_are_evidence(self): assert ledger.dropped == [] assert [(item.intent, item.satisfied_by) for item in ledger.satisfied] == [ - ("distribution:LINEITEM (L_ORDERKEY)", 0) + ("distribution:LINEITEM (L_ORDERKEY)", 0), + ("primary_keys", 1), ] def test_property_style_columns_inside_literals_are_evidence(self): @@ -330,7 +332,7 @@ class TestDropReasons: ("duckdb", "partitioning", TuningType.PARTITIONING), ], ) - def test_registry_none_or_preview_only_drops_with_registry_notes(self, platform, slot, tuning_type): + def test_registry_none_or_preview_only_drops_with_registry_reason(self, platform, slot, tuning_type): ledger = AppliedTuningLedger() _reconcile(_config(**{slot: _columns("L_SHIPDATE")}), ledger, platform) @@ -338,7 +340,8 @@ def test_registry_none_or_preview_only_drops_with_registry_notes(self, platform, [dropped] = ledger.dropped capability = get_capability(platform, tuning_type) assert capability is not None and capability.rendered_via == "none" - assert dropped.reason == capability.notes + assert dropped.reason == f"not rendered at execution: {not_rendered_reason(capability)}" + assert not re.search(r"\.py:\d+", dropped.reason) @pytest.mark.parametrize("platform", ["datafusion", "emr-serverless", "cedardb", "influxdb"]) def test_platform_without_registry_entry_drops_as_not_rendered(self, platform): @@ -609,10 +612,15 @@ def test_declared_keys_that_were_not_rendered_still_drop(self, platform): declared_constraints=frozenset({TuningType.PRIMARY_KEYS, TuningType.FOREIGN_KEYS}), ) - assert {item.intent: item.reason for item in ledger.dropped} == { - "primary_keys": NOT_RENDERED_REASON, - "foreign_keys": NOT_RENDERED_REASON, - } + expected_reasons = {"primary_keys": NOT_RENDERED_REASON, "foreign_keys": NOT_RENDERED_REASON} + if platform == "datafusion": + expected_reasons = { + kind.value: "not rendered at execution: " + "the schema is created without constraints (datafusion_unconstrained_schema)" + for kind in (TuningType.PRIMARY_KEYS, TuningType.FOREIGN_KEYS) + } + + assert {item.intent: item.reason for item in ledger.dropped} == expected_reasons def test_unknown_declarations_reconcile_every_enabled_toggle(self): ledger = AppliedTuningLedger() diff --git a/tests/unit/platforms/test_tuning_reconciliation_adapters.py b/tests/unit/platforms/test_tuning_reconciliation_adapters.py index ac37cc9b3c..20e4f09229 100644 --- a/tests/unit/platforms/test_tuning_reconciliation_adapters.py +++ b/tests/unit/platforms/test_tuning_reconciliation_adapters.py @@ -20,7 +20,7 @@ PHASE_POST_LOAD, AppliedTuningLedger, ) -from benchbox.core.tuning.capability_registry import get_capability +from benchbox.core.tuning.capability_registry import get_capability, not_rendered_reason from benchbox.core.tuning.interface import TableTuning, TuningColumn, TuningType, UnifiedTuningConfiguration from benchbox.core.tuning.reconciliation import NOT_RENDERED_REASON, RECONCILIATION_FAILED_INTENT from benchbox.platforms.base import PlatformAdapter, tuning_trust @@ -89,14 +89,19 @@ def _tuned(adapter: PlatformAdapter) -> PlatformAdapter: return adapter +_DATAFUSION_REASON = ( + "not rendered at execution: the schema is created without constraints (datafusion_unconstrained_schema)" +) + + def _drops(adapter: PlatformAdapter) -> dict[str, str]: return {item.intent: item.reason for item in adapter._applied_tuning_ledger.dropped} -def _registry_notes(platform: str, tuning_type: TuningType) -> str: +def _registry_reason(platform: str, tuning_type: TuningType) -> str: capability = get_capability(platform, tuning_type) assert capability is not None and capability.rendered_via == "none" - return capability.notes + return f"not rendered at execution: {not_rendered_reason(capability)}" def _status(adapter: PlatformAdapter) -> str: @@ -117,7 +122,7 @@ def save_unified_tunings(self, config): class TestFamilies: - def test_postgres_no_op_apply_drops_layout_with_registry_notes(self, monkeypatch, tmp_path): + def test_postgres_no_op_apply_drops_layout_with_registry_reason(self, monkeypatch, tmp_path): monkeypatch.setattr(postgresql_module, "psycopg", Mock(__version__="3.1.0")) monkeypatch.setattr("benchbox.core.tuning.metadata.TuningMetadataManager", FakeMetadataManager) config = _layout_config(partitioning=_columns("L_SHIPDATE"), clustering=_columns("L_ORDERKEY")) @@ -133,8 +138,8 @@ def test_postgres_no_op_apply_drops_layout_with_registry_notes(self, monkeypatch adapter._reconcile_requested_tuning(config) assert _drops(adapter) == { - "partitioning:LINEITEM (L_SHIPDATE)": _registry_notes("postgresql", TuningType.PARTITIONING), - "clustering:LINEITEM (L_ORDERKEY)": _registry_notes("postgresql", TuningType.CLUSTERING), + "partitioning:LINEITEM (L_SHIPDATE)": _registry_reason("postgresql", TuningType.PARTITIONING), + "clustering:LINEITEM (L_ORDERKEY)": _registry_reason("postgresql", TuningType.CLUSTERING), } assert _status(adapter) == NOOP @@ -146,7 +151,12 @@ def test_postgres_family_variants_drop_every_layout_intent(self, platform): [reason] = _drops(adapter).values() capability = get_capability(platform, TuningType.PARTITIONING) - assert reason == (capability.notes if capability is not None else NOT_RENDERED_REASON) + expected = ( + f"not rendered at execution: {not_rendered_reason(capability)}" + if capability is not None + else NOT_RENDERED_REASON + ) + assert reason == expected def test_postgres_constraints_rendered_in_create_table_are_satisfied(self): adapter = _tuned(_Adapter(type="postgresql")) @@ -165,7 +175,7 @@ def test_postgres_constraints_rendered_in_create_table_are_satisfied(self): ] assert _drops(adapter) == {"foreign_keys": NOT_RENDERED_REASON} - def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_notes(self): + def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_reason(self): adapter = _tuned(_Adapter(type="bigquery")) adapter._reconcile_requested_tuning( @@ -173,8 +183,8 @@ def test_bigquery_inspect_only_drops_partitioning_and_clustering_with_registry_n ) assert _drops(adapter) == { - "partitioning:LINEITEM (L_SHIPDATE)": _registry_notes("bigquery", TuningType.PARTITIONING), - "clustering:LINEITEM (L_ORDERKEY)": _registry_notes("bigquery", TuningType.CLUSTERING), + "partitioning:LINEITEM (L_SHIPDATE)": _registry_reason("bigquery", TuningType.PARTITIONING), + "clustering:LINEITEM (L_ORDERKEY)": _registry_reason("bigquery", TuningType.CLUSTERING), } def test_redshift_analyze_only_drops_distribution_and_unrendered_sort(self): @@ -188,7 +198,7 @@ def test_redshift_analyze_only_drops_distribution_and_unrendered_sort(self): ) assert _drops(adapter) == { - "distribution:LINEITEM (L_ORDERKEY)": _registry_notes("redshift", TuningType.DISTRIBUTION), + "distribution:LINEITEM (L_ORDERKEY)": _registry_reason("redshift", TuningType.DISTRIBUTION), "sorting:LINEITEM (L_SHIPDATE)": NOT_RENDERED_REASON, } assert _status(adapter) == APPLIED_UNVERIFIED @@ -221,8 +231,8 @@ def test_datafusion_requested_constraints_are_dropped_not_noop_with_empty_drops( adapter._reconcile_requested_tuning(config) assert _drops(adapter) == { - "primary_keys": NOT_RENDERED_REASON, - "foreign_keys": NOT_RENDERED_REASON, + "primary_keys": _DATAFUSION_REASON, + "foreign_keys": _DATAFUSION_REASON, } assert adapter._applied_tuning_ledger.statements == [] assert _status(adapter) == NOOP @@ -325,7 +335,7 @@ def test_constraint_toggles_follow_what_the_run_benchmark_declares(self, tmp_pat adapter._reconcile_requested_tuning(UnifiedTuningConfiguration()) - assert _drops(adapter) == {"primary_keys": NOT_RENDERED_REASON} + assert _drops(adapter) == {"primary_keys": _DATAFUSION_REASON} def test_reconciliation_error_fails_closed_with_a_blocking_drop(self, monkeypatch): adapter = self._adapter()