diff --git a/vortex-duckdb/cpp/aggregate_fn_pushdown.cpp b/vortex-duckdb/cpp/aggregate_fn_pushdown.cpp index 2caa3463d53..8fdb26dad11 100644 --- a/vortex-duckdb/cpp/aggregate_fn_pushdown.cpp +++ b/vortex-duckdb/cpp/aggregate_fn_pushdown.cpp @@ -53,6 +53,12 @@ LogicalOperatorPtr TryReplaceAggregate(ClientContext &context, return op; } + for (const auto &column_id : get->GetColumnIds()) { + if (column_id.IsVirtualColumn()) { + return op; + } + } + vector> input; const idx_t aggregates_len = agg.expressions.size(); input.reserve(aggregates_len); diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index 2b7703e8921..64d772ada4a 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -38,6 +38,7 @@ use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::expr::Expression; use vortex::expr::stats::Precision; +use vortex::expr::traversal::{TraversalOrder, pre_order_visit_down}; use vortex::file::v2::FileStatsLayoutReader; use vortex::io::kanal_ext::KanalExt as _; use vortex::io::runtime::BlockingRuntime as _; @@ -46,6 +47,7 @@ use vortex::layout::scan::multi::MultiLayoutChild; use vortex::layout::scan::multi::MultiLayoutDataSource; use vortex::metrics::tracing::get_global_labels; use vortex::scalar_fn::fns::binary::Binary; +use vortex::scalar_fn::fns::get_item::GetItem; use vortex::scalar_fn::fns::operators::Operator; use vortex::scalar_fn::fns::pack::Pack; use vortex::scan::DataSource; @@ -377,6 +379,21 @@ impl Stream for ScanDriverStream { } } +fn references_field(expr: &Expression, name: &str) -> VortexResult { + let mut contains = false; + pre_order_visit_down(expr, |node| { + if node + .as_opt::() + .is_some_and(|field| field.as_ref() == name) + { + contains = true; + return Ok(TraversalOrder::Stop); + } + Ok(TraversalOrder::Continue) + })?; + Ok(contains) +} + fn build_partials( aggregates: &[ColumnAggregate], fields: &[DuckdbField], @@ -608,6 +625,11 @@ pub fn pushdown_complex_filter( return Ok(false); }; + // Pushed filter can't resolve virtual columns + if references_field(&expr, "file_row_number")? || references_field(&expr, "file_index")? { + return Ok(false); + } + // Duckdb calls pushdown_complex_filter during planning phase. // If all filters are pushed down, duckdb enables a LEFT_DELIM_JOIN -> // COMPARISON_JOIN (HASH_JOIN) optimization: diff --git a/vortex-sqllogictest/slt/duckdb/agg_corner_virtual.slt b/vortex-sqllogictest/slt/duckdb/agg_corner_virtual.slt new file mode 100644 index 00000000000..c69888548b1 --- /dev/null +++ b/vortex-sqllogictest/slt/duckdb/agg_corner_virtual.slt @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +include ../setup.slt.no + +# aggregates over virtual columns are not pushed + +statement ok +CREATE TABLE tv(i INTEGER); + +statement ok +INSERT INTO tv VALUES (10), (20), (30); + +statement ok +COPY tv TO '${WORK_DIR}/cv.vortex'; + +query II +SELECT count(file_row_number), sum(file_row_number) FROM '${WORK_DIR}/cv.vortex'; +---- +3 3 + +query I +SELECT min(file_index) FROM '${WORK_DIR}/cv.vortex'; +---- +0 + +query II +SELECT count(*), sum(i) FROM '${WORK_DIR}/cv.vortex' WHERE file_row_number < 2; +---- +2 30 + +query I +SELECT i FROM '${WORK_DIR}/cv.vortex' WHERE file_row_number < 2 ORDER BY i; +---- +10 +20