duckdb: forbid pushdown of list_col in (item, NULL)

vortex-duckdb/build.rs

@@ -46,7 +46,7 @@ const SOURCE_FILES: [&str; 11] = [
 
 // Duckdb C API function we use.
 // This lowers codegen'd src/cpp.rs by four times.
-const DUCKDB_C_API_FUNCTIONS: [&str; 133] = [
+const DUCKDB_C_API_FUNCTIONS: [&str; 134] = [
     "duckdb_array_type_array_size",
     "duckdb_array_type_child_type",
     "duckdb_array_vector_get_child",
@@ -79,6 +79,7 @@ const DUCKDB_C_API_FUNCTIONS: [&str; 133] = [
     "duckdb_create_selection_vector",
     "duckdb_create_struct_type",
     "duckdb_create_time",
+    "duckdb_create_time_ns",
     "duckdb_create_timestamp",
     "duckdb_create_timestamp_ms",
     "duckdb_create_timestamp_ns",

vortex-duckdb/src/convert/expr.rs

@@ -78,6 +78,7 @@ use crate::duckdb::ExpressionClass::BoundComparison;
 use crate::duckdb::ExpressionClass::BoundConjunction;
 use crate::duckdb::ExpressionClass::BoundConstant;
 use crate::duckdb::ExpressionClass::BoundRef;
+use crate::duckdb::ExtractedValue;
 use crate::projection::DuckdbField;
 
 fn from_bound_str(value: &duckdb::ExpressionRef) -> VortexResult<String> {
@@ -440,6 +441,21 @@ pub fn can_push_expression(value: &duckdb::ExpressionRef) -> bool {
             ) {
                 return false;
             }
+            // x IN (1, NULL) is NULL when x is not in the list.
+            // list_contains has no way to express this logic
+            if matches!(
+                op.op,
+                DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_COMPARE_IN
+                    | DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_COMPARE_NOT_IN
+            ) && op.children().any(|child| {
+                matches!(
+                    child.as_class(),
+                    Some(BoundConstant(constant))
+                        if matches!(constant.value.extract(), ExtractedValue::Null)
+                )
+            }) {
+                return false;
+            }
             op.children().all(can_push_expression)
         }
         ExpressionClass::BoundAggregate(_) => false,
@@ -722,6 +738,13 @@ fn try_from_compare_in(
     else {
         return Ok(None);
     };
+
+    // x IN (1, NULL) is NULL for x not in the list.
+    // list_contains returns false instead of NULL
+    if list_elements.iter().any(Scalar::is_null) {
+        return Ok(None);
+    }
+
     let list = Scalar::list(
         Arc::new(list_elements[0].dtype().clone()),
         list_elements,

vortex-duckdb/src/convert/scalar.rs

@@ -243,8 +243,9 @@ impl ToDuckDBScalar for ExtScalar<'_> {
                 TimeUnit::Microseconds => Value::new_time(value()?),
                 TimeUnit::Milliseconds => Value::new_time(value()? * 1000),
                 TimeUnit::Seconds => Value::new_time(value()? * 1000 * 1000),
-                TimeUnit::Nanoseconds | TimeUnit::Days => {
-                    vortex_bail!("cannot convert timeunit {unit} to a duckdb MS time")
+                TimeUnit::Nanoseconds => Value::new_time_ns(value()?),
+                TimeUnit::Days => {
+                    vortex_bail!("cannot convert timeunit {unit} to a duckdb time")
                 }
             },
         })
@@ -315,6 +316,13 @@ impl<'a> TryFrom<&'a ValueRef> for Scalar {
                     Some(ScalarValue::from(micros)),
                 )?,
             )),
+            ExtractedValue::TimeNs(nanos) => Ok(Scalar::extension::<Time>(
+                TimeUnit::Nanoseconds,
+                Scalar::try_new(
+                    DType::Primitive(I64, Nullable),
+                    Some(ScalarValue::from(nanos)),
+                )?,
+            )),
             ExtractedValue::TimestampNs(nanos) => Ok(Scalar::extension::<Timestamp>(
                 TimestampOptions {
                     unit: TimeUnit::Nanoseconds,

vortex-duckdb/src/duckdb/value.rs

@@ -117,7 +117,7 @@ impl ValueRef {
                 ExtractedValue::Time(unsafe { cpp::duckdb_get_time(self.as_ptr()).micros })
             }
             DUCKDB_TYPE::DUCKDB_TYPE_TIME_NS => {
-                ExtractedValue::Time(unsafe { cpp::duckdb_get_time_ns(self.as_ptr()).nanos })
+                ExtractedValue::TimeNs(unsafe { cpp::duckdb_get_time_ns(self.as_ptr()).nanos })
             }
             DUCKDB_TYPE::DUCKDB_TYPE_TIMESTAMP_NS => ExtractedValue::TimestampNs(unsafe {
                 cpp::duckdb_get_timestamp_ns(self.as_ptr()).nanos
@@ -249,6 +249,10 @@ impl Value {
         }
     }
 
+    pub fn new_time_ns(nanos: i64) -> Self {
+        unsafe { Self::own(cpp::duckdb_create_time_ns(cpp::duckdb_time_ns { nanos })) }
+    }
+
     pub fn new_time(micros: i64) -> Self {
         unsafe { Self::own(cpp::duckdb_create_time(cpp::duckdb_time { micros })) }
     }
@@ -444,6 +448,7 @@ pub enum ExtractedValue {
     Blob(ByteBuffer),
     Date(i32),
     Time(i64),
+    TimeNs(i64),
     TimestampNs(i64),
     Timestamp(i64),
     TimestampMs(i64),

vortex-sqllogictest/slt/duckdb/filter_literals.slt

@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+# pushed filter literals of less common types must keep duckdb semantics
+
+include ../setup.slt.no
+
+statement ok
+CREATE TABLE tn(t TIME_NS);
+
+statement ok
+INSERT INTO tn VALUES ('10:00:00.123456789'), ('11:00:00');
+
+statement ok
+COPY tn TO '${WORK_DIR}/ftn.vortex';
+
+query I
+SELECT count(*) FROM tn WHERE t = TIME_NS '10:00:00.123456789';
+----
+1
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/ftn.vortex' WHERE t = TIME_NS '10:00:00.123456789';
+----
+1
+
+query I
+SELECT count(*) FROM tn WHERE t > TIME_NS '10:30:00';
+----
+1
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/ftn.vortex' WHERE t > TIME_NS '10:30:00';
+----
+1
+
+statement ok
+CREATE TABLE ts(s VARCHAR);
+
+statement ok
+INSERT INTO ts VALUES ('a%b'), ('axb');
+
+statement ok
+COPY ts TO '${WORK_DIR}/fts.vortex';
+
+query I
+SELECT count(*) FROM ts WHERE s LIKE 'a!%b' ESCAPE '!';
+----
+1
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/fts.vortex' WHERE s LIKE 'a!%b' ESCAPE '!';
+----
+1
+
+statement ok
+CREATE TABLE ti(i INTEGER);
+
+statement ok
+INSERT INTO ti VALUES (1), (2), (NULL);
+
+statement ok
+COPY ti TO '${WORK_DIR}/fti.vortex';
+
+query I
+SELECT count(*) FROM ti WHERE i NOT IN (1, NULL);
+----
+0
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/fti.vortex' WHERE i NOT IN (1, NULL);
+----
+0
+
+query I
+SELECT count(*) FROM ti WHERE i IN (1, NULL);
+----
+1
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/fti.vortex' WHERE i IN (1, NULL);
+----
+1
+
+statement ok
+CREATE TABLE tst(s STRUCT(a INTEGER, b VARCHAR));
+
+statement ok
+INSERT INTO tst VALUES ({'a': 1, 'b': 'x'}), ({'a': 2, 'b': 'y'});
+
+statement ok
+COPY tst TO '${WORK_DIR}/fst.vortex';
+
+query I
+SELECT count(*) FROM tst WHERE s.a = 1;
+----
+1
+
+query I
+SELECT count(*) FROM '${WORK_DIR}/fst.vortex' WHERE s.a = 1;
+----
+1

vortex-sqllogictest/src/duckdb.rs

@@ -288,6 +288,7 @@ impl std::fmt::Display for ValueDisplayAdapter {
             ExtractedValue::Blob(_)
             | ExtractedValue::Date(_)
             | ExtractedValue::Time(_)
+            | ExtractedValue::TimeNs(_)
             | ExtractedValue::TimestampNs(_)
             | ExtractedValue::Timestamp(_)
             | ExtractedValue::TimestampMs(_)