duckdb: forbid aggregations over virtual columns

vortex-duckdb/cpp/aggregate_fn_pushdown.cpp

@@ -53,6 +53,12 @@ LogicalOperatorPtr TryReplaceAggregate(ClientContext &context,
         return op;
     }
 
+    for (const auto &column_id : get->GetColumnIds()) {
+        if (column_id.IsVirtualColumn()) {
+            return op;
+        }
+    }
+
     vector<std::pair<TableColumnScanIndex, const Expression &>> input;
     const idx_t aggregates_len = agg.expressions.size();
     input.reserve(aggregates_len);

vortex-duckdb/src/table_function.rs

@@ -38,6 +38,7 @@ use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
 use vortex::expr::Expression;
 use vortex::expr::stats::Precision;
+use vortex::expr::traversal::{TraversalOrder, pre_order_visit_down};
 use vortex::file::v2::FileStatsLayoutReader;
 use vortex::io::kanal_ext::KanalExt as _;
 use vortex::io::runtime::BlockingRuntime as _;
@@ -46,6 +47,7 @@ use vortex::layout::scan::multi::MultiLayoutChild;
 use vortex::layout::scan::multi::MultiLayoutDataSource;
 use vortex::metrics::tracing::get_global_labels;
 use vortex::scalar_fn::fns::binary::Binary;
+use vortex::scalar_fn::fns::get_item::GetItem;
 use vortex::scalar_fn::fns::operators::Operator;
 use vortex::scalar_fn::fns::pack::Pack;
 use vortex::scan::DataSource;
@@ -377,6 +379,21 @@ impl Stream for ScanDriverStream {
     }
 }
 
+fn references_field(expr: &Expression, name: &str) -> VortexResult<bool> {
+    let mut contains = false;
+    pre_order_visit_down(expr, |node| {
+        if node
+            .as_opt::<GetItem>()
+            .is_some_and(|field| field.as_ref() == name)
+        {
+            contains = true;
+            return Ok(TraversalOrder::Stop);
+        }
+        Ok(TraversalOrder::Continue)
+    })?;
+    Ok(contains)
+}
+
 fn build_partials(
     aggregates: &[ColumnAggregate],
     fields: &[DuckdbField],
@@ -608,6 +625,11 @@ pub fn pushdown_complex_filter(
         return Ok(false);
     };
 
+    // Pushed filter can't resolve virtual columns
+    if references_field(&expr, "file_row_number")? || references_field(&expr, "file_index")? {
+        return Ok(false);
+    }
+
     // Duckdb calls pushdown_complex_filter during planning phase.
     // If all filters are pushed down, duckdb enables a LEFT_DELIM_JOIN ->
     // COMPARISON_JOIN (HASH_JOIN) optimization:

vortex-sqllogictest/slt/duckdb/agg_corner_virtual.slt

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+include ../setup.slt.no
+
+# aggregates over virtual columns are not pushed
+
+statement ok
+CREATE TABLE tv(i INTEGER);
+
+statement ok
+INSERT INTO tv VALUES (10), (20), (30);
+
+statement ok
+COPY tv TO '${WORK_DIR}/cv.vortex';
+
+query II
+SELECT count(file_row_number), sum(file_row_number) FROM '${WORK_DIR}/cv.vortex';
+----
+3 3
+
+query I
+SELECT min(file_index) FROM '${WORK_DIR}/cv.vortex';
+----
+0
+
+query II
+SELECT count(*), sum(i) FROM '${WORK_DIR}/cv.vortex' WHERE file_row_number < 2;
+----
+2 30
+
+query I
+SELECT i FROM '${WORK_DIR}/cv.vortex' WHERE file_row_number < 2 ORDER BY i;
+----
+10
+20