From f40d99ac8b10e03a41374706e9fa07194a922ca9 Mon Sep 17 00:00:00 2001 From: zhengpeng <847850277@qq.com> Date: Fri, 24 Jul 2026 10:33:44 +0800 Subject: [PATCH 001/109] feat: migrate EmptyExec and PlaceholderRowExec to ExecutionPlan proto hooks (#23784) ## Which issue does this PR close? - Closes #23501 . ## Rationale for this change Migrates the `EmptyExec` and `PlaceholderRowExec` leaf plans while preserving the existing protobuf wire format. ## What changes are included in this PR? - Implement `ExecutionPlan::try_to_proto` for `EmptyExec` and `PlaceholderRowExec`. - Add plan-specific `try_from_proto` implementations. - Route protobuf encoding and decoding through the new hooks. - Keep the deprecated helper methods as compatibility shims. ## Are these changes tested? yes ## Are there any user-facing changes? Hi @andygrove , would you be willing to review this PR when you have time? Thanks! --- datafusion/physical-plan/src/empty.rs | 48 ++++++++ .../physical-plan/src/placeholder_row.rs | 50 ++++++++ datafusion/proto/src/physical_plan/mod.rs | 111 ++++++++++-------- 3 files changed, 161 insertions(+), 48 deletions(-) diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs index 44a6f444dc4b5..3bd38bf238dc1 100644 --- a/datafusion/physical-plan/src/empty.rs +++ b/datafusion/physical-plan/src/empty.rs @@ -185,6 +185,54 @@ impl ExecutionPlan for EmptyExec { Ok(Arc::new(stats)) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let schema = self.schema().as_ref().try_into()?; + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Empty( + protobuf::EmptyExecNode { + schema: Some(schema), + partitions: self + .properties() + .output_partitioning() + .partition_count() as u32, + }, + ), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl EmptyExec { + /// Reconstruct an [`EmptyExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + _ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let empty = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::Empty, + "EmptyExec", + ); + let schema = empty.schema.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "EmptyExec is missing required field 'schema'" + ) + })?; + let schema = Arc::new(arrow::datatypes::Schema::try_from(schema)?); + // A zero (absent) partition count comes from a plan encoded before the + // field existed, which always meant a single partition. + let partitions = empty.partitions.max(1) as usize; + Ok(Arc::new(EmptyExec::new(schema).with_partitions(partitions))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index 20d267331b2aa..5d71058269f49 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -186,6 +186,56 @@ impl ExecutionPlan for PlaceholderRowExec { None, ))) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let schema = self.schema().as_ref().try_into()?; + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow( + protobuf::PlaceholderRowExecNode { + schema: Some(schema), + partitions: self + .properties() + .output_partitioning() + .partition_count() as u32, + }, + ), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl PlaceholderRowExec { + /// Reconstruct a [`PlaceholderRowExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + _ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let placeholder = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::PlaceholderRow, + "PlaceholderRowExec", + ); + let schema = placeholder.schema.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "PlaceholderRowExec is missing required field 'schema'" + ) + })?; + let schema = Arc::new(Schema::try_from(schema)?); + // A zero (absent) partition count comes from a plan encoded before the + // field existed, which always meant a single partition. + let partitions = placeholder.partitions.max(1) as usize; + Ok(Arc::new( + PlaceholderRowExec::new(schema).with_partitions(partitions), + )) + } } #[cfg(test)] diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index d62bafa883441..b459368bcb1da 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -814,11 +814,11 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::CrossJoin(_) => { CrossJoinExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::Empty(empty) => { - self.try_into_empty_physical_plan(empty, ctx, proto_converter) + PhysicalPlanType::Empty(_) => { + EmptyExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::PlaceholderRow(placeholder) => { - self.try_into_placeholder_row_physical_plan(placeholder, ctx) + PhysicalPlanType::PlaceholderRow(_) => { + PlaceholderRowExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::Sort(_) => { SortExec::try_from_proto(self.node(), &decode_ctx) @@ -925,16 +925,6 @@ pub trait PhysicalPlanNodeExt: Sized { ); } - if let Some(empty) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_empty_exec(empty, codec); - } - - if let Some(empty) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_placeholder_row_exec( - empty, codec, - ); - } - if let Some(data_source_exec) = plan.downcast_ref::() && let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec( data_source_exec, @@ -1934,31 +1924,48 @@ pub trait PhysicalPlanNodeExt: Sized { CrossJoinExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `EmptyExec` deserializes itself via `EmptyExec::try_from_proto`" + )] fn try_into_empty_physical_plan( &self, empty: &protobuf::EmptyExecNode, - _ctx: &PhysicalPlanDecodeContext<'_>, - _proto_converter: &dyn PhysicalProtoConverterExtension, + ctx: &PhysicalPlanDecodeContext<'_>, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let schema = Arc::new(convert_required!(empty.schema)?); - // A zero (absent) partition count comes from a plan encoded before the - // field existed, which always meant a single partition. - let partitions = empty.partitions.max(1) as usize; - Ok(Arc::new(EmptyExec::new(schema).with_partitions(partitions))) + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::Empty(empty.clone())), + }; + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + EmptyExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `PlaceholderRowExec` deserializes itself via `PlaceholderRowExec::try_from_proto`" + )] fn try_into_placeholder_row_physical_plan( &self, placeholder: &protobuf::PlaceholderRowExecNode, - _ctx: &PhysicalPlanDecodeContext<'_>, + ctx: &PhysicalPlanDecodeContext<'_>, ) -> Result> { - let schema = Arc::new(convert_required!(placeholder.schema)?); - // A zero (absent) partition count comes from a plan encoded before the - // field existed, which always meant a single partition. - let partitions = placeholder.partitions.max(1) as usize; - Ok(Arc::new( - PlaceholderRowExec::new(schema).with_partitions(partitions), - )) + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::PlaceholderRow( + placeholder.clone(), + )), + }; + let proto_converter = DefaultPhysicalProtoConverter {}; + let decoder = ConverterPlanDecoder { + ctx, + proto_converter: &proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + PlaceholderRowExec::try_from_proto(&node, &decode_ctx) } #[deprecated( @@ -2833,33 +2840,41 @@ pub trait PhysicalPlanNodeExt: Sized { }) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `EmptyExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_empty_exec( empty: &EmptyExec, - _codec: &dyn PhysicalExtensionCodec, + codec: &dyn PhysicalExtensionCodec, ) -> Result { - let schema = empty.schema().as_ref().try_into()?; - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Empty(protobuf::EmptyExecNode { - schema: Some(schema), - partitions: empty.properties().output_partitioning().partition_count() - as u32, - })), + let proto_converter = DefaultPhysicalProtoConverter {}; + let encoder = ConverterPlanEncoder { + codec, + proto_converter: &proto_converter, + }; + let ctx = ExecutionPlanEncodeCtx::new(&encoder); + empty.try_to_proto(&ctx)?.ok_or_else(|| { + internal_datafusion_err!("EmptyExec::try_to_proto returned None") }) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `PlaceholderRowExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_placeholder_row_exec( - empty: &PlaceholderRowExec, - _codec: &dyn PhysicalExtensionCodec, + placeholder: &PlaceholderRowExec, + codec: &dyn PhysicalExtensionCodec, ) -> Result { - let schema = empty.schema().as_ref().try_into()?; - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::PlaceholderRow( - protobuf::PlaceholderRowExecNode { - schema: Some(schema), - partitions: empty.properties().output_partitioning().partition_count() - as u32, - }, - )), + let proto_converter = DefaultPhysicalProtoConverter {}; + let encoder = ConverterPlanEncoder { + codec, + proto_converter: &proto_converter, + }; + let ctx = ExecutionPlanEncodeCtx::new(&encoder); + placeholder.try_to_proto(&ctx)?.ok_or_else(|| { + internal_datafusion_err!("PlaceholderRowExec::try_to_proto returned None") }) } From 18b1e359c3c547ec0d649932f85abbe33144ef19 Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:15:20 +0800 Subject: [PATCH 002/109] fix: grouped first_value/last_value FILTER excludes NULL predicate rows (#23707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #22666 ## Rationale for this change Under SQL aggregate `FILTER` semantics, a row passes only when the predicate evaluates to `true`; rows where the predicate is `null` must be excluded. Grouped `first_value` / `last_value` checked only `BooleanArray::value(idx)`, without checking validity, so a NULL predicate row whose underlying value bit is set (as produced by comparison kernels, e.g. `null::int < 1`) was treated as passing: ```sql SELECT g, first_value(a ORDER BY a) FILTER (WHERE b < 1) AS fv FROM (VALUES (0, 10, CAST(NULL AS INT)), (0, 20, 2)) AS t(g, a, b) GROUP BY g; -- returned fv = 10, must return fv = NULL -- (row 1: b < 1 is NULL; row 2: b < 1 is FALSE — no row satisfies `true`) ``` ## What changes are included in this PR? `datafusion/functions-aggregate/src/first_last.rs`, in `FirstLastGroupsAccumulator::get_filtered_extreme_of_each_group` (shared by `first_value` and `last_value`, and by both the `update_batch` and `merge_batch` paths): - `passed_filter` now requires `is_valid(idx) && value(idx)` (the `Some(true)` semantics), matching the convention already used by `variance.rs` / `correlation.rs`. - The `is_set_arr` read gets the same validity check. This is *not* only an internal bitmap: `convert_to_state` stores the user FILTER clause (including its nulls) in the last state column, so on the merge path (e.g. skip-partial-aggregation) NULL predicate rows were likewise treated as set. Verified with a forced skip-partial run (100k unique groups, all-NULL predicates): 83,616 groups were incorrectly assigned non-NULL values before the fix, 0 after. For genuine internal bitmaps (no nulls) the added check is trivially true, so behavior there is unchanged. Regression coverage: - sqllogictest (`aggregate.slt`): the issue reproducer, the `last_value` counterpart, mixed TRUE/FALSE/NULL predicates, all-TRUE and no-FILTER controls, the (already correct) non-grouped path, and a window-function no-regression case. - Unit tests: `test_group_acc_filter_null_predicate` (update path) and `test_group_acc_merge_null_is_set` (merge path via `convert_to_state` → `merge_batch`), both constructing `BooleanArray`s whose null slots carry a set value bit. ## Are these changes tested? Yes — see above. Verified `./dev/rust_lint.sh`, `cargo test -p datafusion-functions-aggregate --lib`, the `aggregate`/`window` sqllogictest files, and `datafusion-cli` end-to-end (grouped first/last_value now return NULL for the issue reproducer; mixed-predicate and non-grouped results unchanged). Also audited the rest of `functions-aggregate` for the same validity-blind pattern: shared helpers (`nulls.rs::filter_to_validity`, `accumulate.rs`, `prim_op.rs`, `count.rs`, `array_agg.rs`, `variance.rs`, `correlation.rs`) already handle validity correctly, and non-grouped paths pre-filter with arrow's `filter` kernel (which drops NULL predicate rows), so no other aggregate needs changes. ## Performance `functions-aggregate/benches/first_last.rs` was run against `main`. The added checks are one validity-bit test per row on the grouped path; measured deltas were within the machine's noise floor (±5% on unchanged `filter=false` cases). A variant hoisting the null check out of the row loop showed no measurable benefit beyond noise, so the simple idiomatic form is kept. ## Are there any user-facing changes? Only the bug fix: grouped `first_value`/`last_value` with a nullable `FILTER` predicate now correctly exclude NULL-predicate rows, matching SQL semantics and the behavior of other aggregates. No API or configuration changes. --------- Co-authored-by: Claude --- .../functions-aggregate/src/first_last.rs | 124 +++++++++++++++++- .../sqllogictest/test_files/aggregate.slt | 63 +++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs index cecb277cb844a..7c2540aa03a2b 100644 --- a/datafusion/functions-aggregate/src/first_last.rs +++ b/datafusion/functions-aggregate/src/first_last.rs @@ -555,8 +555,15 @@ impl FirstLastGroupsAccumulator { for (idx_in_val, group_idx) in group_indices.iter().enumerate() { let group_idx = *group_idx; - let passed_filter = opt_filter.is_none_or(|x| x.value(idx_in_val)); - let is_set = is_set_arr.is_none_or(|x| x.value(idx_in_val)); + // A row passes the FILTER clause only when the predicate is + // `true`; rows whose predicate evaluates to `null` are excluded. + let passed_filter = + opt_filter.is_none_or(|x| x.is_valid(idx_in_val) && x.value(idx_in_val)); + // `is_set_arr` carries the user FILTER clause (including its + // nulls) when the state was produced by `convert_to_state`, so + // the validity check is required here as well (#22666). + let is_set = + is_set_arr.is_none_or(|x| x.is_valid(idx_in_val) && x.value(idx_in_val)); if !passed_filter || !is_set { continue; @@ -1415,6 +1422,7 @@ mod tests { use arrow::{ array::{BooleanArray, Int64Array, ListArray, PrimitiveArray, StringArray}, + buffer::NullBuffer, compute::SortOptions, datatypes::Schema, }; @@ -1773,6 +1781,118 @@ mod tests { Ok(()) } + /// Rows whose FILTER predicate evaluates to `null` must not pass the + /// filter, even when the underlying value bit at the null slot is `true` + /// (#22666). + #[test] + fn test_group_acc_filter_null_predicate() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("c", DataType::Int64, true), + ])); + + let sort_keys = [PhysicalSortExpr { + expr: col("c", &schema).unwrap(), + options: SortOptions::default(), + }]; + + let mut group_acc = FirstLastGroupsAccumulator::try_new( + PrimitiveValueState::::new(DataType::Int64), + sort_keys.into(), + true, + &[DataType::Int64], + true, + )?; + + let val_with_orderings: Vec = vec![ + Arc::new(Int64Array::from(vec![10, 20, 30])), + Arc::new(Int64Array::from(vec![10, 20, 30])), + ]; + + // Row 0: predicate is null (but its value bit is true, as produced by + // kernels such as `b < 1` when the null slot's underlying value is 0) + // Row 1: predicate is false + // Row 2: predicate is true + let filter = BooleanArray::new( + BooleanBuffer::from(vec![false, true, false, true]), + Some(NullBuffer::from(BooleanBuffer::from(vec![ + true, false, true, true, + ]))), + ) + .slice(1, 3); + assert_eq!(filter.offset(), 1); + + group_acc.update_batch(&val_with_orderings, &[0, 0, 1], Some(&filter), 2)?; + + let binding = group_acc.evaluate(EmitTo::All)?; + let eval_result = binding.as_any().downcast_ref::().unwrap(); + + // Group 0 has no row with a `true` predicate, so it must stay unset. + // Group 1 takes the only row with a `true` predicate. + let expect: PrimitiveArray = Int64Array::from(vec![None, Some(30)]); + assert_eq!(eval_result, &expect); + + Ok(()) + } + + /// `convert_to_state` stores the user FILTER clause (including its nulls) + /// in the `is_set` state column, so `merge_batch` must not treat a null + /// `is_set` entry with a set value bit as "is set" (#22666). + #[test] + fn test_group_acc_merge_null_is_set() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, true), + Field::new("c", DataType::Int64, true), + ])); + + let sort_keys = [PhysicalSortExpr { + expr: col("c", &schema).unwrap(), + options: SortOptions::default(), + }]; + + let group_acc = FirstLastGroupsAccumulator::try_new( + PrimitiveValueState::::new(DataType::Int64), + sort_keys.clone().into(), + true, + &[DataType::Int64], + true, + )?; + + let val_with_orderings: Vec = vec![ + Arc::new(Int64Array::from(vec![10, 20])), + Arc::new(Int64Array::from(vec![10, 20])), + ]; + + // Same null-with-set-value-bit filter as above, carried into the state + let filter = BooleanArray::new( + BooleanBuffer::from(vec![true, true]), + Some(NullBuffer::from(BooleanBuffer::from(vec![false, true]))), + ); + + let state = group_acc.convert_to_state(&val_with_orderings, Some(&filter))?; + assert_eq!(state.len(), 3); + + let mut merging_acc = FirstLastGroupsAccumulator::try_new( + PrimitiveValueState::::new(DataType::Int64), + sort_keys.into(), + true, + &[DataType::Int64], + true, + )?; + + merging_acc.merge_batch(&state, &[0, 0], 1)?; + + let binding = merging_acc.evaluate(EmitTo::All)?; + let eval_result = binding.as_any().downcast_ref::().unwrap(); + + // Only the second row is valid and passes; the null-predicate row must + // be skipped even though its value bit is true. + let expect: PrimitiveArray = Int64Array::from(vec![Some(20)]); + assert_eq!(eval_result, &expect); + + Ok(()) + } + #[test] fn test_first_list_acc_size() -> Result<()> { fn size_after_batch(values: &[ArrayRef]) -> Result { diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 9400a09a5d4bf..1515e17e3fdff 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -6689,6 +6689,69 @@ GROUP BY g ---- 0 0 +# first_value_with_group_by_and_nullable_filter +# Rows whose FILTER predicate evaluates to NULL must be excluded (#22666) +query II rowsort +SELECT g, first_value(a ORDER BY a) FILTER (WHERE b < 1) AS fv +FROM (VALUES (0, 10, CAST(NULL AS INT)), (0, 20, 2)) AS t(g, a, b) +GROUP BY g +---- +0 NULL + +# last_value_with_group_by_and_nullable_filter +query II rowsort +SELECT g, last_value(a ORDER BY a) FILTER (WHERE b < 1) AS lv +FROM (VALUES (0, 10, CAST(NULL AS INT)), (0, 20, 2)) AS t(g, a, b) +GROUP BY g +---- +0 NULL + +# first_last_value_with_group_by_and_mixed_filter_results +# Only rows whose FILTER predicate is TRUE participate: a = 10 (b = 1) and +# a = 20 (b = 0) in group 0. The NULL-predicate row (a = 5) and the +# FALSE-predicate row (a = 30) are excluded. No row passes the filter in +# group 1, so the aggregates return NULL there. +query III rowsort +SELECT g, + first_value(a ORDER BY a) FILTER (WHERE b < 2) AS fv, + last_value(a ORDER BY a) FILTER (WHERE b < 2) AS lv +FROM (VALUES (0, 5, CAST(NULL AS INT)), (0, 10, 1), (0, 30, 2), (0, 20, 0), + (1, 100, CAST(NULL AS INT)), (1, 50, 3)) AS t(g, a, b) +GROUP BY g +---- +0 10 20 +1 NULL NULL + +# first_last_value_with_group_by_filter_all_true_and_no_filter +# Behavior is unchanged when every row passes the FILTER or there is no FILTER +query IIIII rowsort +SELECT g, + first_value(a ORDER BY a) FILTER (WHERE a > 0) AS fv, + last_value(a ORDER BY a) FILTER (WHERE a > 0) AS lv, + first_value(a ORDER BY a) AS fv_no_filter, + last_value(a ORDER BY a) AS lv_no_filter +FROM (VALUES (0, 5, CAST(NULL AS INT)), (0, 10, 1), (0, 30, 2), (0, 20, 0)) AS t(g, a, b) +GROUP BY g +---- +0 5 30 5 30 + +# first_value_without_group_by_and_nullable_filter +query I rowsort +SELECT first_value(a ORDER BY a) FILTER (WHERE b < 1) AS fv +FROM (VALUES (10, CAST(NULL AS INT)), (20, 2)) AS t(a, b) +---- +NULL + +# first_value_window_function_no_regression +query II +SELECT a, first_value(a) OVER (ORDER BY a) AS fv +FROM (VALUES (10), (20), (5)) AS t(a) +ORDER BY a +---- +5 5 +10 5 +20 5 + # query_with_untyped_null_filter query I SELECT count(*) FILTER (WHERE NULL) From 5d1c3cdee976895dd80d28e520a0b31bf2bdf21b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 24 Jul 2026 06:34:15 -0500 Subject: [PATCH 003/109] Unwrap widening Date32 -> Date64 casts in comparison predicates (#23729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - N/A. Small, self-contained enhancement to `unwrap_cast_in_comparison`; happy to file a tracking issue if preferred. ## Rationale for this change `unwrap_cast_in_comparison` could not unwrap a cast between the two date types, so a predicate like `CAST(date32_col AS Date64) ` never folded onto the bare column. Folding it lets the literal be compared against the unmodified column, keeping predicate pushdown / pruning effective on date columns. There were two coupled causes in `try_cast_literal_to_type` (`datafusion/expr-common/src/casts.rs`): 1. `try_cast_numeric_literal` scaled both `Date32` and `Date64` literals by the same target multiplier (`mul = 1`). But `Date32` counts **days** since the epoch while `Date64` counts **milliseconds**, so a cross conversion needs a factor of `MILLISECONDS_IN_DAY` (86_400_000). 2. `is_lossy_temporal_cast` classified every `Date <-> temporal` pair as lossy, which swept in `Date32 <-> Date64` and blocked the unwrap outright. The reverse direction is subtle and unsound if handled naively: narrowing a `Date64` **column** down to `Date32` truncates milliseconds to the day (many-to-one), so `CAST(date64 AS Date32) = ` matches any millisecond within that day. arrow-rs does not require `Date64` values to be whole-day (apache/arrow-rs#5288), so the column may carry sub-day values the planner cannot see, and unwrapping would drop those rows. That direction is therefore explicitly blocked. ## What changes are included in this PR? - Relax `is_lossy_temporal_cast` so a date-to-date (and identity) cast is not pre-classified as lossy; per-value exactness is enforced downstream. - Add `scale_date_literal` with exact-only semantics: `Date32 -> Date64` multiplies by `MILLISECONDS_IN_DAY` (overflow-guarded with checked arithmetic); `Date64 -> Date32` divides only on a whole-day boundary and otherwise returns `None`. This mirrors the existing Decimal scaling path in the same function. - Add `is_date_narrowing_cast` and block the narrowing `Date64 -> Date32` column cast in the two logical-optimizer gates (comparison and in-list) **and** in the physical-expr simplifier, mirroring `is_timestamp_precision_narrowing_cast`. The physical-expr guard is required for soundness on the pruning / row-group-filter path (verified by a unit test that fails without it); the widening `Date32 -> Date64` column cast is injective and stays supported. Scope is intentionally limited to `Date32 <-> Date64` scaling, the narrowing gate, and tests. ## Are these changes tested? Yes, at two levels. **End-to-end (`datafusion/sqllogictest/test_files/simplify_expr.slt`)** — the PR is structured as three commits so the behavior change is legible in the diff: 1. `test:` characterizes current behavior (passes on unmodified `main`): neither direction is unwrapped, results are correct. The fixture stores sub-day and pre-epoch `Date64` values on purpose. 2. `feat:` applies only the code change; the recorded widening `EXPLAIN` plans now fail intentionally. 3. `test:` regenerates the expectations. The commit-3 diff is exactly the widening plans flipping from `CAST(d32 AS Date64) Date64(..)` to `d32 Date32(..)`; **every result row and every narrowing plan is byte-identical**, which is the soundness proof. Coverage includes `=`/`<`/`<=`/`>`/`>=`/`IN`, whole-day vs sub-day literals (the latter yields zero rows and is left as-is), the narrowing soundness case (the noon row is still returned), pre-epoch dates (arrow's toward-zero truncation is pinned), and NULL three-valued logic. **Unit** — `scale_date_literal` exactness and `i32::MIN`/`i32::MAX` overflow, `is_date_narrowing_cast`, the relaxed `is_lossy_temporal_cast` date-pair behavior, and the physical-expr narrowing guard. `cargo fmt --check` is clean, `cargo clippy -p datafusion-expr-common -- -D warnings` passes, and the full sqllogictest suite passes. ## Are there any user-facing changes? No public API changes. The optimizer now additionally rewrites widening `Date32 -> Date64` cast comparisons where it previously left them untouched; results are unchanged, plans are simplified. Narrowing `Date64 -> Date32` cast comparisons are deliberately left as-is. ## Note for reviewers The open PR #23727 adds the `if from_type == to_type { return false }` identity guard to this same function. This PR is based independently on `main` and includes that identity line as part of the clean gate shape here, so depending on merge order the two may need a trivial rebase where those lines overlap. --------- Signed-off-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- datafusion/expr-common/src/casts.rs | 248 +++++++++++++++-- .../src/simplify_expressions/unwrap_cast.rs | 11 +- .../src/simplifier/unwrap_cast.rs | 24 +- .../sqllogictest/test_files/simplify_expr.slt | 252 ++++++++++++++++++ 4 files changed, 509 insertions(+), 26 deletions(-) diff --git a/datafusion/expr-common/src/casts.rs b/datafusion/expr-common/src/casts.rs index 8c9616f7b8285..3518c02772672 100644 --- a/datafusion/expr-common/src/casts.rs +++ b/datafusion/expr-common/src/casts.rs @@ -28,7 +28,9 @@ use arrow::datatypes::{ MAX_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION, MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION, TimeUnit, }; -use arrow::temporal_conversions::{MICROSECONDS, MILLISECONDS, NANOSECONDS}; +use arrow::temporal_conversions::{ + MICROSECONDS, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS, +}; use datafusion_common::ScalarValue; /// Convert a literal [`ScalarValue`] to `target_type`, preserving the exact value. @@ -100,17 +102,24 @@ fn is_date_type(data_type: &DataType) -> bool { /// 00:00:00'` matches only midnight. /// /// An identity cast (`from_type == to_type`, e.g. `Date32 -> Date32`) never -/// changes comparison semantics and is therefore not lossy. This has to be -/// handled explicitly because `DataType::is_temporal()` is true for both -/// `Date32` and `Date64`, so `is_date_type(from) && to.is_temporal()` would -/// otherwise report an identity `Date -> Date` cast as lossy and block the -/// rewrite. Note this is deliberately limited to *identical* types: a genuine -/// `Date32 <-> Date64` cast changes units (days vs milliseconds) and must -/// still be treated as lossy here. +/// changes comparison semantics and is therefore not lossy. +/// +/// A cast between the two date types (`Date32` <-> `Date64`) is not pre-filtered +/// as lossy here, because whether it loses information is a per-value question +/// rather than a per-type one. `Date32` -> `Date64` is always exact (a day scaled +/// to midnight in milliseconds). `Date64` -> `Date32` is exact only when the value +/// lands on a day boundary: Arrow nominally defines `Date64` as whole days encoded +/// in milliseconds, but arrow-rs does not enforce that (see arrow-rs#5288), so a +/// `Date64` carrying sub-day milliseconds would lose them. This is not a licence to +/// drop them - [`try_cast_numeric_literal`] returns `None` for a `Date64` value not +/// divisible by 86_400_000, so an inexact `Date64` -> `Date32` fold never happens. fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool { if from_type == to_type { return false; } + if is_date_type(from_type) && is_date_type(to_type) { + return false; + } (is_date_type(from_type) && to_type.is_temporal()) || (is_date_type(to_type) && from_type.is_temporal()) } @@ -135,6 +144,19 @@ pub fn is_timestamp_precision_narrowing_cast( timestamp_unit_scale(from_unit) > timestamp_unit_scale(to_unit) } +/// Returns true when casting a date column from `from_type` to `to_type` narrows +/// `Date64` (milliseconds) to `Date32` (days). +/// +/// Like [`is_timestamp_precision_narrowing_cast`], this guards comparison cast +/// unwrapping against a many-to-one column cast. `CAST(date64 AS Date32) = lit_day` +/// matches any millisecond within that day, but the rewritten `date64 = lit_ms` +/// matches only midnight. Arrow does not require `Date64` values to be whole days +/// (see arrow-rs#5288), so the column may carry sub-day values the planner cannot +/// see; the widening direction (`Date32 -> Date64`) is injective and stays allowed. +pub fn is_date_narrowing_cast(from_type: &DataType, to_type: &DataType) -> bool { + matches!((from_type, to_type), (DataType::Date64, DataType::Date32)) +} + fn timestamp_unit_scale(unit: &TimeUnit) -> i128 { match unit { TimeUnit::Second => 1, @@ -183,6 +205,36 @@ fn is_supported_binary_type(data_type: &DataType) -> bool { matches!(data_type, DataType::Binary | DataType::FixedSizeBinary(_)) } +/// Scale a `Date32`/`Date64` literal value into the units of `target_type`, +/// returning `None` when the conversion is not exact. +/// +/// `Date32` counts **days** since the Unix epoch while `Date64` counts +/// **milliseconds** since the Unix epoch, so a cross conversion scales by +/// [`MILLISECONDS_IN_DAY`]: +/// * `Date32` -> `Date64` is always exact: `days * MILLISECONDS_IN_DAY` +/// (guarded against `i64`/`i128` overflow). +/// * `Date64` -> `Date32` is exact only when the millisecond value lands on a +/// whole-day boundary; otherwise it returns `None` so the cast unwrap is +/// skipped (correct for every operator, including `=`). +/// +/// For a same-type date cast or a date/integer cast the generic `mul` +/// multiplier already applies, so this returns `value * mul`. +fn scale_date_literal( + value: i128, + from_type: &DataType, + target_type: &DataType, + mul: i128, +) -> Option { + const MILLIS_PER_DAY: i128 = MILLISECONDS_IN_DAY as i128; + match (from_type, target_type) { + (DataType::Date32, DataType::Date64) => value.checked_mul(MILLIS_PER_DAY), + (DataType::Date64, DataType::Date32) => { + (value % MILLIS_PER_DAY == 0).then_some(value / MILLIS_PER_DAY) + } + _ => value.checked_mul(mul), + } +} + /// Convert a numeric value from one numeric data type to another fn try_cast_numeric_literal( lit_value: &ScalarValue, @@ -258,8 +310,12 @@ fn try_cast_numeric_literal( ScalarValue::UInt16(Some(v)) => (*v as i128).checked_mul(mul), ScalarValue::UInt32(Some(v)) => (*v as i128).checked_mul(mul), ScalarValue::UInt64(Some(v)) => (*v as i128).checked_mul(mul), - ScalarValue::Date32(Some(v)) => (*v as i128).checked_mul(mul), - ScalarValue::Date64(Some(v)) => (*v as i128).checked_mul(mul), + ScalarValue::Date32(Some(v)) => { + scale_date_literal(*v as i128, &lit_data_type, target_type, mul) + } + ScalarValue::Date64(Some(v)) => { + scale_date_literal(*v as i128, &lit_data_type, target_type, mul) + } ScalarValue::TimestampSecond(Some(v), _) => (*v as i128).checked_mul(mul), ScalarValue::TimestampMillisecond(Some(v), _) => (*v as i128).checked_mul(mul), ScalarValue::TimestampMicrosecond(Some(v), _) => (*v as i128).checked_mul(mul), @@ -855,25 +911,91 @@ mod tests { } #[test] - fn test_try_cast_date32_date64_still_blocked() { - // `Date32` counts days and `Date64` counts milliseconds, but - // try_cast_numeric_literal uses mul = 1 for both, so a cross cast would - // convert units wrongly. The identity short-circuit must NOT open this - // up: Date32 <-> Date64 has to stay blocked. - assert!(is_lossy_temporal_cast(&DataType::Date32, &DataType::Date64)); - assert!(is_lossy_temporal_cast(&DataType::Date64, &DataType::Date32)); - + fn test_try_cast_between_date32_and_date64() { + // 2025-01-01 is day 20089 since the Unix epoch, which is + // 20089 * 86_400_000 = 1_735_689_600_000 milliseconds. + const DAY_2025_01_01: i32 = 20089; + const MS_2025_01_01: i64 = 1_735_689_600_000; + assert_eq!(DAY_2025_01_01 as i64 * MILLISECONDS_IN_DAY, MS_2025_01_01); + + // Date32 -> Date64 is always exact (days scaled up to milliseconds). expect_cast( - ScalarValue::Date32(Some(1)), + ScalarValue::Date32(Some(DAY_2025_01_01)), DataType::Date64, - ExpectedCast::NoValue, + ExpectedCast::Value(ScalarValue::Date64(Some(MS_2025_01_01))), ); + // Date64 -> Date32 is exact only on a whole-day boundary. expect_cast( - ScalarValue::Date64(Some(86_400_000)), + ScalarValue::Date64(Some(MS_2025_01_01)), + DataType::Date32, + ExpectedCast::Value(ScalarValue::Date32(Some(DAY_2025_01_01))), + ); + + // A Date64 value that is not on a day boundary cannot be represented as + // a Date32 exactly, so no rewrite is produced. + expect_cast( + ScalarValue::Date64(Some(MS_2025_01_01 + 1)), DataType::Date32, ExpectedCast::NoValue, ); + expect_cast( + ScalarValue::Date64(Some(MS_2025_01_01 - 1)), + DataType::Date32, + ExpectedCast::NoValue, + ); + + // The epoch and negative (pre-epoch) days round-trip exactly. + expect_cast( + ScalarValue::Date32(Some(0)), + DataType::Date64, + ExpectedCast::Value(ScalarValue::Date64(Some(0))), + ); + expect_cast( + ScalarValue::Date32(Some(-1)), + DataType::Date64, + ExpectedCast::Value(ScalarValue::Date64(Some(-MILLISECONDS_IN_DAY))), + ); + expect_cast( + ScalarValue::Date64(Some(-MILLISECONDS_IN_DAY)), + DataType::Date32, + ExpectedCast::Value(ScalarValue::Date32(Some(-1))), + ); + + // Same-type date casts remain identity conversions. + expect_cast( + ScalarValue::Date32(Some(DAY_2025_01_01)), + DataType::Date32, + ExpectedCast::Value(ScalarValue::Date32(Some(DAY_2025_01_01))), + ); + expect_cast( + ScalarValue::Date64(Some(MS_2025_01_01)), + DataType::Date64, + ExpectedCast::Value(ScalarValue::Date64(Some(MS_2025_01_01))), + ); + } + + #[test] + fn test_is_lossy_temporal_cast_date_pairs() { + // Date <-> Date is let through the pre-filter (per-value exactness is + // enforced downstream in try_cast_numeric_literal, not here). + assert!(!is_lossy_temporal_cast( + &DataType::Date32, + &DataType::Date64 + )); + assert!(!is_lossy_temporal_cast( + &DataType::Date64, + &DataType::Date32 + )); + // Identity is not lossy. + assert!(!is_lossy_temporal_cast( + &DataType::Date32, + &DataType::Date32 + )); + // Date <-> Timestamp remains lossy. + let ts = DataType::Timestamp(TimeUnit::Millisecond, None); + assert!(is_lossy_temporal_cast(&DataType::Date32, &ts)); + assert!(is_lossy_temporal_cast(&ts, &DataType::Date32)); } #[test] @@ -893,6 +1015,90 @@ mod tests { )); } + #[test] + fn test_is_date_narrowing_cast() { + // Only Date64 -> Date32 narrows (ms -> days, many-to-one). + assert!(is_date_narrowing_cast(&DataType::Date64, &DataType::Date32)); + // The widening direction is injective and must not be flagged. + assert!(!is_date_narrowing_cast( + &DataType::Date32, + &DataType::Date64 + )); + // Identity and non-date pairs are not date-narrowing casts. + assert!(!is_date_narrowing_cast( + &DataType::Date32, + &DataType::Date32 + )); + assert!(!is_date_narrowing_cast( + &DataType::Date64, + &DataType::Date64 + )); + assert!(!is_date_narrowing_cast(&DataType::Int64, &DataType::Date32)); + } + + #[test] + fn test_scale_date_literal_exactness_and_overflow() { + const MS_PER_DAY: i128 = MILLISECONDS_IN_DAY as i128; + + // Date32 -> Date64 is always exact: days scaled to midnight milliseconds. + // 2025-01-01 is day 20089 = 1_735_689_600_000 ms. + assert_eq!( + scale_date_literal(20089, &DataType::Date32, &DataType::Date64, 1), + Some(1_735_689_600_000) + ); + assert_eq!( + scale_date_literal(0, &DataType::Date32, &DataType::Date64, 1), + Some(0) + ); + // Negative (pre-epoch) whole day: 1969-12-31 is day -1 = -86_400_000 ms. + assert_eq!( + scale_date_literal(-1, &DataType::Date32, &DataType::Date64, 1), + Some(-86_400_000) + ); + + // Date64 -> Date32 is exact only on a whole-day boundary. + assert_eq!( + scale_date_literal( + 1_735_689_600_000, + &DataType::Date64, + &DataType::Date32, + 1 + ), + Some(20089) + ); + assert_eq!( + scale_date_literal(-86_400_000, &DataType::Date64, &DataType::Date32, 1), + Some(-1) + ); + // Sub-day values are not exactly representable as a Date32, in both the + // positive and the pre-epoch negative direction -> None (no fold). + assert_eq!( + scale_date_literal( + 1_735_732_800_000, + &DataType::Date64, + &DataType::Date32, + 1 + ), + None + ); + assert_eq!( + scale_date_literal(-43_200_000, &DataType::Date64, &DataType::Date32, 1), + None + ); + + // Extremes: a Date32 at i32::MIN / i32::MAX widens with checked i128 + // arithmetic, producing the exact millisecond value without overflow or + // panic. + assert_eq!( + scale_date_literal(i32::MAX as i128, &DataType::Date32, &DataType::Date64, 1), + Some(i32::MAX as i128 * MS_PER_DAY) + ); + assert_eq!( + scale_date_literal(i32::MIN as i128, &DataType::Date32, &DataType::Date64, 1), + Some(i32::MIN as i128 * MS_PER_DAY) + ); + } + #[test] fn test_try_cast_to_type_unsupported() { // int64 to list diff --git a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs index c7f20a6b6f50e..ef0bfa516fe41 100644 --- a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs +++ b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs @@ -60,7 +60,8 @@ use datafusion_common::{internal_err, tree_node::Transformed}; use datafusion_expr::{BinaryExpr, lit}; use datafusion_expr::{Cast, Expr, Operator, TryCast, simplify::SimplifyContext}; use datafusion_expr_common::casts::{ - is_supported_type, is_timestamp_precision_narrowing_cast, try_cast_literal_to_type, + is_date_narrowing_cast, is_supported_type, is_timestamp_precision_narrowing_cast, + try_cast_literal_to_type, }; pub(super) fn unwrap_cast_in_comparison_for_binary( @@ -134,7 +135,9 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary( return false; }; - if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) { + if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) + || is_date_narrowing_cast(&expr_type, field.data_type()) + { return false; } @@ -177,7 +180,9 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist( return false; } - if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) { + if is_timestamp_precision_narrowing_cast(&expr_type, field.data_type()) + || is_date_narrowing_cast(&expr_type, field.data_type()) + { return false; } diff --git a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs b/datafusion/physical-expr/src/simplifier/unwrap_cast.rs index 5caee00962b49..3e67fc8291a4e 100644 --- a/datafusion/physical-expr/src/simplifier/unwrap_cast.rs +++ b/datafusion/physical-expr/src/simplifier/unwrap_cast.rs @@ -37,7 +37,8 @@ use arrow::datatypes::{DataType, Schema}; use datafusion_common::{Result, ScalarValue, tree_node::Transformed}; use datafusion_expr::Operator; use datafusion_expr_common::casts::{ - is_timestamp_precision_narrowing_cast, try_cast_literal_to_type, + is_date_narrowing_cast, is_timestamp_precision_narrowing_cast, + try_cast_literal_to_type, }; use crate::PhysicalExpr; @@ -129,7 +130,9 @@ fn try_unwrap_cast_comparison( // Get the data type of the inner expression let inner_type = inner_expr.data_type(schema)?; - if is_timestamp_precision_narrowing_cast(&inner_type, cast_type) { + if is_timestamp_precision_narrowing_cast(&inner_type, cast_type) + || is_date_narrowing_cast(&inner_type, cast_type) + { return Ok(None); } @@ -231,6 +234,23 @@ mod tests { assert_eq!(*optimized_binary.op(), Operator::Gt); } + #[test] + fn test_no_unwrap_date64_to_date32_narrowing() { + let schema = Schema::new(vec![Field::new("d64", DataType::Date64, false)]); + + // cast(d64 AS Date32) = Date32(20089) must NOT unwrap: narrowing a Date64 + // column to Date32 truncates milliseconds to the day (many-to-one), so the + // rewritten `d64 = ` would drop sub-day rows. + let column_expr = col("d64", &schema).unwrap(); + let cast_expr = Arc::new(CastExpr::new(column_expr, DataType::Date32, None)); + let literal_expr = lit(ScalarValue::Date32(Some(20089))); + let binary_expr = + Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal_expr)); + + let result = unwrap_cast_in_comparison(binary_expr, &schema).unwrap(); + assert!(!result.transformed); + } + #[test] fn test_no_unwrap_when_types_unsupported() { let schema = Schema::new(vec![Field::new("f1", DataType::Float32, false)]); diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt index a291740b914f5..57dc440407dc0 100644 --- a/datafusion/sqllogictest/test_files/simplify_expr.slt +++ b/datafusion/sqllogictest/test_files/simplify_expr.slt @@ -179,3 +179,255 @@ physical_plan statement ok drop table dates; + +# ------------------------------------------------------------------------ +# Unwrapping Date32 <-> Date64 casts in comparison predicates. +# +# `Date32` counts whole days since the epoch; `Date64` counts milliseconds. +# Widening a `Date32` column up to `Date64` (`date32_col -> Date64`) is +# injective, so a comparison against a whole-day `Date64` literal can be +# rewritten onto the bare `Date32` column. Narrowing a `Date64` column down to +# `Date32` truncates the milliseconds to the day (many-to-one) and must NOT be +# rewritten: `CAST(date64 AS Date32) = ` matches any millisecond within +# that day. Arrow does not require `Date64` values to fall on a day boundary +# (arrow-rs#5288), so the table below intentionally stores sub-day `Date64` +# values (ids 2 and 4) to exercise that hazard. +# +# The `Date64` column is built from raw millisecond values with `arrow_cast`; +# `2025-01-01 00:00` = 1735689600000 ms (day 20089), `2025-01-01 12:00` adds +# 43200000 ms. `1969-12-31 00:00` = -86400000 ms (day -1); `1969-12-31 12:00` +# = -43200000 ms (a pre-epoch sub-day value). +statement ok +create table date_unwrap as +select + c.id, + arrow_cast(c.d32, 'Date32') as d32, + arrow_cast(c.d64ms, 'Date64') as d64 +from (values + (1, '2025-01-01', 1735689600000), + (2, '2025-01-01', 1735732800000), + (3, '1969-12-31', -86400000), + (4, '1969-12-31', -43200000), + (5, NULL, NULL) +) as c(id, d32, d64ms); + +query IDD +select id, d32, d64 from date_unwrap order by id; +---- +1 2025-01-01 2025-01-01T00:00:00 +2 2025-01-01 2025-01-01T12:00:00 +3 1969-12-31 1969-12-31T00:00:00 +4 1969-12-31 1969-12-31T12:00:00 +5 NULL NULL + +# --- Widening Date32 -> Date64: folds onto the bare column --------------- +# The plan for these widening queries is what changes when the optimization is +# enabled: the CAST moves off the column and onto the (whole-day) literal. +query TT +explain select id from date_unwrap where arrow_cast(d32, 'Date64') = arrow_cast(1735689600000, 'Date64'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: date_unwrap.d32 = Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: d32@1 = 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') = arrow_cast(1735689600000, 'Date64') order by id; +---- +1 +2 + +# Range operators fold too (Date32 -> Date64 is monotonic). +query TT +explain select id from date_unwrap where arrow_cast(d32, 'Date64') < arrow_cast(1735689600000, 'Date64'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: date_unwrap.d32 < Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: d32@1 < 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +explain select id from date_unwrap where arrow_cast(d32, 'Date64') >= arrow_cast(1735689600000, 'Date64'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: date_unwrap.d32 >= Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: d32@1 >= 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') < arrow_cast(1735689600000, 'Date64') order by id; +---- +3 +4 + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') <= arrow_cast(1735689600000, 'Date64') order by id; +---- +1 +2 +3 +4 + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') > arrow_cast(1735689600000, 'Date64') order by id; +---- + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') >= arrow_cast(1735689600000, 'Date64') order by id; +---- +1 +2 + +# Reversed operands fold too: with the Date64 literal on the LEFT, logical +# simplification moves the bare column to the left and swaps the operator +# (`literal < CAST(col)` becomes `col > literal`). +query TT +explain select id from date_unwrap where arrow_cast(-86400000, 'Date64') < arrow_cast(d32, 'Date64'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: date_unwrap.d32 > Date32("1969-12-31") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: d32@1 > 1969-12-31, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where arrow_cast(-86400000, 'Date64') < arrow_cast(d32, 'Date64') order by id; +---- +1 +2 + +# IN-list widening also folds. +query TT +explain select id from date_unwrap where arrow_cast(d32, 'Date64') in (arrow_cast(1735689600000, 'Date64'), arrow_cast(-86400000, 'Date64')); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: date_unwrap.d32 = Date32("2025-01-01") OR date_unwrap.d32 = Date32("1969-12-31") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: d32@1 = 2025-01-01 OR d32@1 = 1969-12-31, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') in (arrow_cast(1735689600000, 'Date64'), arrow_cast(-86400000, 'Date64')) order by id; +---- +1 +2 +3 +4 + +# A NON-whole-day literal is NOT foldable: a Date32-derived Date64 is always at +# midnight, so it can never equal a sub-day literal. The plan keeps the CAST and +# the query returns zero rows. +query TT +explain select id from date_unwrap where arrow_cast(d32, 'Date64') = arrow_cast(1735732800000, 'Date64'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: CAST(date_unwrap.d32 AS Date64) = Date64("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d32] +physical_plan +01)FilterExec: CAST(d32@1 AS Date64) = 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where arrow_cast(d32, 'Date64') = arrow_cast(1735732800000, 'Date64') order by id; +---- + +# NULL comparison semantics are unchanged by the rewrite (three-valued logic: +# the NULL row yields NULL, not a dropped row). +query IB +select id, arrow_cast(d32, 'Date64') = arrow_cast(1735689600000, 'Date64') as eq from date_unwrap order by id; +---- +1 true +2 true +3 false +4 false +5 NULL + +# --- Narrowing Date64 -> Date32: must NOT fold (soundness) --------------- +# The plan for these queries is invariant: the CAST stays on the column. If it +# were unwrapped, the sub-day rows (ids 2 and 4) would be dropped. +query TT +explain select id from date_unwrap where cast(d64 as date) = DATE '2025-01-01'; +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: CAST(date_unwrap.d64 AS Date32) = Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d64] +physical_plan +01)FilterExec: CAST(d64@1 AS Date32) = 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# id 2 is 2025-01-01 12:00 - it truncates to 2025-01-01 and MUST be returned. +query I +select id from date_unwrap where cast(d64 as date) = DATE '2025-01-01' order by id; +---- +1 +2 + +query TT +explain select id from date_unwrap where cast(d64 as date) < DATE '2025-01-01'; +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: CAST(date_unwrap.d64 AS Date32) < Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d64] +physical_plan +01)FilterExec: CAST(d64@1 AS Date32) < 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +# IN-list narrowing is guarded as well. +query TT +explain select id from date_unwrap where cast(d64 as date) in (DATE '2025-01-01'); +---- +logical_plan +01)Projection: date_unwrap.id +02)--Filter: CAST(date_unwrap.d64 AS Date32) = Date32("2025-01-01") +03)----TableScan: date_unwrap projection=[id, d64] +physical_plan +01)FilterExec: CAST(d64@1 AS Date32) = 2025-01-01, projection=[id@0] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query I +select id from date_unwrap where cast(d64 as date) in (DATE '2025-01-01') order by id; +---- +1 +2 + +# Pre-epoch dates. Arrow's Date64 -> Date32 cast divides by 86_400_000 and +# truncates toward zero, so the pre-epoch sub-day value (id 4, -43200000 ms) +# truncates to day 0 (1970-01-01), not to 1969-12-31. This is arrow's runtime +# behavior; `scale_date_literal` only ever folds on exact whole-day multiples, +# so it can never disagree with the value the cast actually produces. +query ID +select id, cast(d64 as date) as truncated from date_unwrap where d64 is not null order by id; +---- +1 2025-01-01 +2 2025-01-01 +3 1969-12-31 +4 1970-01-01 + +query I +select id from date_unwrap where cast(d64 as date) = DATE '1969-12-31' order by id; +---- +3 + +query I +select id from date_unwrap where cast(d64 as date) = DATE '1970-01-01' order by id; +---- +4 + +statement ok +drop table date_unwrap; From 7dfeeb041211907c487285e2756d438d2af7d28b Mon Sep 17 00:00:00 2001 From: Naman Modi Date: Fri, 24 Jul 2026 17:49:55 +0530 Subject: [PATCH 004/109] test (slt): add memory-limited aggregation sqllogictests (#23838) ## Which issue does this PR close? - Part of #22710. ## Rationale for this change There's almost no slt coverage for grouped aggregation under a memory limit, where the aggregate spills to disk and re-groups the spilled state. #23657 covers the ordered path & this covers the unordered/hash path. ## What changes are included in this PR? - Adds `aggregate_memory_limit.slt` in which the high-cardinality "GROUP BY" is run under a 1M limit, so the hash aggregate spills. The group key is scrambled with `(v * 7) % 100000` (a bijection, so still 100000 groups) to keep the input unsorted; otherwise it takes the streaming path and never spills. - Scoped to a single partition (`target_partitions = 1`), so the spill happens in one aggregate operator with no repartition. - Cases cover different accumulator states: single-column, multi-column, count(DISTINCT), sum/min/max, avg (widening), array_agg (growable). ## Are these changes tested? This PR is tests. All pass locally. ## Are there any user-facing changes? No. --- .../test_files/aggregate_memory_spill.slt | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 datafusion/sqllogictest/test_files/aggregate_memory_spill.slt diff --git a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt new file mode 100644 index 0000000000000..7615209255394 --- /dev/null +++ b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt @@ -0,0 +1,228 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Memory-limited (spilling) grouped hash aggregation. +# +# High-cardinality GROUP BY under a tight memory limit: the aggregate spills to +# disk, re-groups the spilled state, and must still return the right answer. +# +# The group key is scrambled with `(v * 7) % 100000` because generate_series is +# sorted, which would take the streaming path that never spills. gcd(7, 100000) +# = 1, so it's a bijection over 1..100000. Still 100000 groups, just unsorted, +# so the hash table grows and spills. +# +# Each query aggregates over the grouped result, so the expected output is one +# row. sum(1..100000) = 5000050000, and every v lands in one group, so the +# per-group sums always add back to that total. + +# Single partition keeps the aggregation in one operator (no repartition). +statement ok +SET datafusion.execution.target_partitions = 1 + +statement ok +SET datafusion.runtime.memory_limit = '1M' + +# --- Case A: single-column high-cardinality GROUP BY --- +query II +SELECT count(*), sum(total) +FROM ( + SELECT (v * 7) % 100000 AS k, sum(v) AS total + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +100000 5000050000 + +# Prove the inner aggregate actually spills (else these tests would silently stop covering the spill path). +# Only `spill_count` is pinned; the other metrics vary per run. +query TT +EXPLAIN ANALYZE +SELECT count(*), sum(total) +FROM ( + SELECT (v * 7) % 100000 AS k, sum(v) AS total + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(total)@1 as sum(total)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(total)], metrics=[] +03)----ProjectionExec: expr=[sum(t.v)@1 as total], metrics=[] +04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[spill_count=9,] +05)--------ProjectionExec: expr=[value@0 as v], metrics=[] +06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# --- Case B: multi-column GROUP BY (is_single() = false) --- +# Both keys are bijections of v, so each (a, b) pair is unique: 100000 groups. +query II +SELECT count(*), sum(total) +FROM ( + SELECT (v * 7) % 100000 AS a, (v * 13) % 100000 AS b, sum(v) AS total + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000, (v * 13) % 100000 +) +---- +100000 5000050000 + +# Assert this case spills too. +query TT +EXPLAIN ANALYZE +SELECT count(*), sum(total) +FROM ( + SELECT (v * 7) % 100000 AS a, (v * 13) % 100000 AS b, sum(v) AS total + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000, (v * 13) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(total)@1 as sum(total)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(total)], metrics=[] +03)----ProjectionExec: expr=[sum(t.v)@2 as total], metrics=[] +04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000), v@0 * 13 % 100000 as t.v * Int64(13) % Int64(100000)], aggr=[sum(t.v)], metrics=[spill_count=11,] +05)--------ProjectionExec: expr=[value@0 as v], metrics=[] +06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# --- Case C: DISTINCT aggregate under memory limit --- +# One distinct value per group, so each count(DISTINCT v) = 1. +query II +SELECT count(*), sum(d) +FROM ( + SELECT (v * 7) % 100000 AS k, count(DISTINCT v) AS d + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +100000 100000 + +# Assert this case spills too. +query TT +EXPLAIN ANALYZE +SELECT count(*), sum(d) +FROM ( + SELECT (v * 7) % 100000 AS k, count(DISTINCT v) AS d + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(d)@1 as sum(d)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(d)], metrics=[] +03)----ProjectionExec: expr=[count(alias1)@1 as d], metrics=[] +04)------AggregateExec: mode=Single, gby=[group_alias_0@0 as group_alias_0], aggr=[count(alias1)], metrics=[spill_count=18,] +05)--------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as group_alias_0, v@0 as alias1], aggr=[], ordering_mode=Sorted, metrics=[] +06)----------ProjectionExec: expr=[value@0 as v], metrics=[] +07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# --- Case D: multiple aggregates (sum/min/max) under memory limit --- +# Each group holds a single v, so min(v) = max(v) = v within the group. +query IIII +SELECT count(*), sum(s), min(mn), max(mx) +FROM ( + SELECT (v * 7) % 100000 AS k, sum(v) AS s, min(v) AS mn, max(v) AS mx + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +100000 5000050000 1 100000 + +# Assert this case spills too. +query TT +EXPLAIN ANALYZE +SELECT count(*), sum(s), min(mn), max(mx) +FROM ( + SELECT (v * 7) % 100000 AS k, sum(v) AS s, min(v) AS mn, max(v) AS mx + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(s)@1 as sum(s), min(mn)@2 as min(mn), max(mx)@3 as max(mx)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(s), min(mn), max(mx)], metrics=[] +03)----ProjectionExec: expr=[sum(t.v)@1 as s, min(t.v)@2 as mn, max(t.v)@3 as mx], metrics=[] +04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v), min(t.v), max(t.v)], metrics=[spill_count=27,] +05)--------ProjectionExec: expr=[value@0 as v], metrics=[] +06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# --- Case E: avg() aggregate (Float64 output) under memory limit --- +# Each group holds a single v, so avg(v) = v within the group. +query IRR +SELECT count(*), min(a), max(a) +FROM ( + SELECT (v * 7) % 100000 AS k, avg(v) AS a + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +100000 1 100000 + +# Assert this case spills too. +query TT +EXPLAIN ANALYZE +SELECT count(*), min(a), max(a) +FROM ( + SELECT (v * 7) % 100000 AS k, avg(v) AS a + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), min(a)@1 as min(a), max(a)@2 as max(a)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), min(a), max(a)], metrics=[] +03)----ProjectionExec: expr=[avg(t.v)@1 as a], metrics=[] +04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[avg(t.v)], metrics=[spill_count=11,] +05)--------ProjectionExec: expr=[value@0 as v], metrics=[] +06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# --- Case F: array_agg() aggregate (growable state) under memory limit --- +# Each group holds a single v, so array_length(array_agg(v)) = 1. +query II +SELECT count(*), sum(l) +FROM ( + SELECT (v * 7) % 100000 AS k, array_length(array_agg(v)) AS l + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +100000 100000 + +# Assert this case spills too. +query TT +EXPLAIN ANALYZE +SELECT count(*), sum(l) +FROM ( + SELECT (v * 7) % 100000 AS k, array_length(array_agg(v)) AS l + FROM generate_series(1, 100000) AS t(v) + GROUP BY (v * 7) % 100000 +) +---- +Plan with Metrics +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*), sum(l)@1 as sum(l)], metrics=[] +02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1)), sum(l)], metrics=[] +03)----ProjectionExec: expr=[array_length(array_agg(t.v)@1) as l], metrics=[] +04)------AggregateExec: mode=Single, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[array_agg(t.v)], metrics=[spill_count=10,] +05)--------ProjectionExec: expr=[value@0 as v], metrics=[] +06)----------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192], metrics=[] + +# Restore settings to slt runner defaults +statement ok +RESET datafusion.runtime.memory_limit + +statement ok +SET datafusion.execution.target_partitions = 4 + +statement ok +RESET datafusion.catalog.create_default_catalog_and_schema From e9a75bf47a962cd7ee11e563bc948f3f0d0a51c8 Mon Sep 17 00:00:00 2001 From: Kumar Ujjawal Date: Fri, 24 Jul 2026 17:50:56 +0530 Subject: [PATCH 005/109] feat: add OR pre-selection short-circuit (#22979) ## Which issue does this PR close? - Closes #22342. ## Rationale for this change `BinaryExpr` already uses pre-selection for `AND` when only a small set of LHS rows can affect the final result. This adds the matching optimization for `OR` when most LHS rows are already true. ## What changes are included in this PR? This PR extends pre-selection short-circuiting to `OR`. For `OR`, the RHS is evaluated only for rows where the LHS is false. Rows where the LHS is true are filled directly as true. The existing `AND` path is kept and the scatter logic is shared. ## Are these changes tested? Yes ## Are there any user-facing changes? No Public API Change Co-authored-by: Andrew Lamb --- .../physical-expr/src/expressions/binary.rs | 368 ++++++++++++------ 1 file changed, 259 insertions(+), 109 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index a39691674d18b..89828620a5930 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -482,41 +482,50 @@ impl PhysicalExpr for BinaryExpr { let rhs = self.right.evaluate(batch)?; return Ok(rhs); } - ShortCircuitStrategy::PreSelection(selection) => { - // The function `evaluate_selection` was not called for filtering and calculation, - // as it takes into account cases where the selection contains null values. - let batch = filter_record_batch(batch, selection)?; - let right_ret = self.right.evaluate(&batch)?; + ShortCircuitStrategy::PreSelection { mask, fill_value } => { + // `mask` selects the rows whose result depends on the RHS; the + // unselected rows are all `fill_value` (see `ShortCircuitStrategy`). + // + // Use `filter_record_batch` directly because `evaluate_selection` + // scatters the RHS back to the original batch length. + let selection_batch = filter_record_batch(batch, &mask)?; + let right_ret = self.right.evaluate(&selection_batch)?; match &right_ret { ColumnarValue::Array(array) => { - // When the array on the right is all true or all false, skip the scatter process let boolean_array = array.as_boolean(); - if boolean_array.null_count() == 0 && !boolean_array.has_false() { - return Ok(lhs); - } else if boolean_array.null_count() == 0 - && !boolean_array.has_true() - { - // If the right-hand array is returned at this point,the lengths will be inconsistent; - // returning a scalar can avoid this issue - return Ok(ColumnarValue::Scalar(ScalarValue::Boolean( - Some(false), - ))); + // If the RHS is uniform on the selected rows, the whole + // expression collapses and no scatter is needed. + if boolean_array.null_count() == 0 { + let rhs_value = if !boolean_array.has_false() { + Some(true) + } else if !boolean_array.has_true() { + Some(false) + } else { + None + }; + if let Some(rhs_value) = rhs_value { + return Ok(uniform_pre_selection_result( + rhs_value, fill_value, lhs, + )); + } } - return pre_selection_scatter(selection, Some(boolean_array)); + return pre_selection_scatter( + &mask, + Some(boolean_array), + fill_value, + ); } ColumnarValue::Scalar(scalar) => { if let ScalarValue::Boolean(v) = scalar { - // When the scalar is true or false, skip the scatter process + // A scalar RHS applies uniformly to all selected rows. if let Some(v) = v { - if *v { - return Ok(lhs); - } else { - return Ok(right_ret); - } + return Ok(uniform_pre_selection_result( + *v, fill_value, lhs, + )); } else { - return pre_selection_scatter(selection, None); + return pre_selection_scatter(&mask, None, fill_value); } } else { return internal_err!( @@ -1038,16 +1047,28 @@ impl BinaryExpr { } } -enum ShortCircuitStrategy<'a> { +enum ShortCircuitStrategy { None, ReturnLeft, ReturnRight, - PreSelection(&'a BooleanArray), + /// Evaluate the right-hand side only on the rows selected by `mask`, then + /// scatter the results back, filling the unselected rows with `fill_value`. + /// + /// - For `AND`, `mask` selects the rows where the LHS is `true` and + /// `fill_value` is `false` (rows where the LHS is `false` are `false`). + /// - For `OR`, `mask` selects the rows where the LHS is `false` and + /// `fill_value` is `true` (rows where the LHS is `true` are `true`). + PreSelection { + mask: BooleanArray, + fill_value: bool, + }, } /// Based on the results calculated from the left side of the short-circuit operation, -/// if the proportion of `true` is less than 0.2 and the current operation is an `and`, -/// the `RecordBatch` will be filtered in advance. +/// pre-selection filters the `RecordBatch` before evaluating the right-hand side when +/// the side that cannot short-circuit the operator is rare: +/// - for `AND`, when the proportion of `true` is less than or equal to 0.2 +/// - for `OR`, when the proportion of `false` is less than or equal to 0.2 const PRE_SELECTION_THRESHOLD: f32 = 0.2; /// Checks if a logical operator (`AND`/`OR`) can short-circuit evaluation based on the left-hand side (lhs) result. @@ -1056,24 +1077,21 @@ const PRE_SELECTION_THRESHOLD: f32 = 0.2; /// - For `AND`: /// - if LHS is all false => short-circuit → return LHS /// - if LHS is all true => short-circuit → return RHS -/// - if LHS is mixed and true_count/sum_count <= [`PRE_SELECTION_THRESHOLD`] -> pre-selection +/// - if LHS is mixed and true_count / len <= [`PRE_SELECTION_THRESHOLD`] -> pre-selection /// - For `OR`: /// - if LHS is all true => short-circuit → return LHS /// - if LHS is all false => short-circuit → return RHS +/// - if LHS is mixed and false_count / len <= [`PRE_SELECTION_THRESHOLD`] -> pre-selection /// # Arguments /// * `lhs` - The left-hand side (lhs) columnar value (array or scalar) -/// * `lhs` - The left-hand side (lhs) columnar value (array or scalar) /// * `op` - The logical operator (`AND` or `OR`) /// /// # Implementation Notes /// 1. Only works with Boolean-typed arguments (other types automatically return `false`) /// 2. Handles both scalar values and array values /// 3. For arrays, uses optimized bit counting techniques for boolean arrays -fn check_short_circuit<'a>( - lhs: &'a ColumnarValue, - op: &Operator, -) -> ShortCircuitStrategy<'a> { - // Quick reject for non-logical operators,and quick judgment when op is and +fn check_short_circuit(lhs: &ColumnarValue, op: &Operator) -> ShortCircuitStrategy { + // Only logical operators can use this path. let is_and = match op { Operator::And => true, Operator::Or => false, @@ -1101,36 +1119,42 @@ fn check_short_circuit<'a>( let true_count = bool_array.values().count_set_bits(); if is_and { - // For AND, prioritize checking for all-false (short circuit case) - // Uses optimized false_count() method provided by Arrow - - // Short circuit if all values are false if true_count == 0 { return ShortCircuitStrategy::ReturnLeft; } - // If no false values, then all must be true if true_count == len { return ShortCircuitStrategy::ReturnRight; } - // determine if we can pre-selection if true_count as f32 / len as f32 <= PRE_SELECTION_THRESHOLD { - return ShortCircuitStrategy::PreSelection(bool_array); + // Select rows where the LHS is true; rows where the LHS + // is false are false regardless of the RHS. + return ShortCircuitStrategy::PreSelection { + mask: bool_array.clone(), + fill_value: false, + }; } } else { - // For OR, prioritize checking for all-true (short circuit case) - // Uses optimized true_count() method provided by Arrow - - // Short circuit if all values are true if true_count == len { return ShortCircuitStrategy::ReturnLeft; } - // If no true values, then all must be false if true_count == 0 { return ShortCircuitStrategy::ReturnRight; } + + let false_count = len - true_count; + if false_count as f32 / len as f32 <= PRE_SELECTION_THRESHOLD { + // Select rows where the LHS is false; rows where the LHS + // is true are true regardless of the RHS. The LHS has no + // nulls here, so negating its bits is infallible. + let mask = BooleanArray::new(!bool_array.values(), None); + return ShortCircuitStrategy::PreSelection { + mask, + fill_value: true, + }; + } } } } @@ -1153,62 +1177,54 @@ fn check_short_circuit<'a>( ShortCircuitStrategy::None } -/// Creates a new boolean array based on the evaluation of the right expression, -/// but only for positions where the left_result is true. +/// Collapses a pre-selected expression whose RHS is uniformly `rhs_value` across +/// every selected row, avoiding a scatter: +/// - when it equals `fill_value`, every row is `fill_value` (a scalar); +/// - otherwise the selected rows already equal the RHS, which matches the LHS +/// there, and the unselected rows are the LHS value too, so the result is `lhs`. +fn uniform_pre_selection_result( + rhs_value: bool, + fill_value: bool, + lhs: ColumnarValue, +) -> ColumnarValue { + if rhs_value == fill_value { + ColumnarValue::Scalar(ScalarValue::Boolean(Some(fill_value))) + } else { + lhs + } +} + +/// Creates a boolean array by scattering compact RHS results into the positions +/// selected by `mask`. /// -/// This function is used for short-circuit evaluation optimization of logical AND operations: -/// - When left_result has few true values, we only evaluate the right expression for those positions -/// - Values are copied from right_array where left_result is true -/// - All other positions are filled with false values +/// This function is used for short-circuit evaluation optimization of logical AND/OR operations: +/// - Only selected rows are evaluated on the RHS +/// - Values are copied from `right_result` where `mask` is true +/// - All other positions are filled with `fill_value` (`false` for AND, `true` for OR) /// /// # Parameters -/// - `left_result` Boolean array with selection mask (typically from left side of AND) +/// - `mask` Boolean array with the rows whose result depends on the RHS /// - `right_result` Result of evaluating right side of expression (only for selected positions) +/// - `fill_value` The value for the unselected positions (`false` for AND, `true` for OR) /// /// # Returns -/// A combined ColumnarValue with values from right_result where left_result is true -/// -/// # Example -/// Initial Data: { 1, 2, 3, 4, 5 } -/// Left Evaluation -/// (Condition: Equal to 2 or 3) -/// ↓ -/// Filtered Data: {2, 3} -/// Left Bitmap: { 0, 1, 1, 0, 0 } -/// ↓ -/// Right Evaluation -/// (Condition: Even numbers) -/// ↓ -/// Right Data: { 2 } -/// Right Bitmap: { 1, 0 } -/// ↓ -/// Combine Results -/// Final Bitmap: { 0, 1, 0, 0, 0 } -/// -/// # Note -/// Perhaps it would be better to modify `left_result` directly without creating a copy? -/// In practice, `left_result` should have only one owner, so making changes should be safe. -/// However, this is difficult to achieve under the immutable constraints of [`Arc`] and [`BooleanArray`]. +/// A combined `ColumnarValue` with the same length as `mask`. fn pre_selection_scatter( - left_result: &BooleanArray, + mask: &BooleanArray, right_result: Option<&BooleanArray>, + fill_value: bool, ) -> Result { - let result_len = left_result.len(); + let result_len = mask.len(); let mut result_array_builder = BooleanArray::builder(result_len); - // keep track of current position we have in right boolean array let mut right_array_pos = 0; - - // keep track of how much is filled let mut last_end = 0; - // reduce if condition in for_each match right_result { Some(right_result) => { - SlicesIterator::new(left_result).for_each(|(start, end)| { - // the gap needs to be filled with false + SlicesIterator::new(mask).for_each(|(start, end)| { if start > last_end { - result_array_builder.append_n(start - last_end, false); + result_array_builder.append_n(start - last_end, fill_value); } // copy values from right array for this slice @@ -1222,13 +1238,11 @@ fn pre_selection_scatter( last_end = end; }); } - None => SlicesIterator::new(left_result).for_each(|(start, end)| { - // the gap needs to be filled with false + None => SlicesIterator::new(mask).for_each(|(start, end)| { if start > last_end { - result_array_builder.append_n(start - last_end, false); + result_array_builder.append_n(start - last_end, fill_value); } - // append nulls for this slice derictly let len = end - start; result_array_builder.append_nulls(len); @@ -1236,9 +1250,9 @@ fn pre_selection_scatter( }), } - // Fill any remaining positions with false + // Fill any remaining positions with `fill_value` if last_end < result_len { - result_array_builder.append_n(result_len - last_end, false); + result_array_builder.append_n(result_len - last_end, fill_value); } let boolean_result = result_array_builder.finish(); @@ -5400,14 +5414,17 @@ mod tests { let ColumnarValue::Array(array) = &left_value else { panic!("Expected ColumnarValue::Array"); }; - let ShortCircuitStrategy::PreSelection(value) = + let ShortCircuitStrategy::PreSelection { mask, fill_value } = check_short_circuit(&left_value, &Operator::And) else { panic!("Expected ShortCircuitStrategy::PreSelection"); }; + // For AND, the mask selects the rows where the LHS is true and the + // unselected rows are filled with `false`. + assert!(!fill_value); let expected_boolean_arr: Vec<_> = as_boolean_array(array).unwrap().iter().collect(); - let boolean_arr: Vec<_> = value.iter().collect(); + let boolean_arr: Vec<_> = mask.iter().collect(); assert_eq!(expected_boolean_arr, boolean_arr); // op: OR left: all true @@ -5418,10 +5435,33 @@ mod tests { ShortCircuitStrategy::ReturnLeft )); - // op: OR left: not all true + // 20% false: OR can pre-select the false rows. let left_expr: Arc = logical2physical(&logical_col("a").gt(expr_lit(2)), &schema); let left_value = left_expr.evaluate(&batch).unwrap(); + let ColumnarValue::Array(array) = &left_value else { + panic!("Expected ColumnarValue::Array"); + }; + let ShortCircuitStrategy::PreSelection { mask, fill_value } = + check_short_circuit(&left_value, &Operator::Or) + else { + panic!("Expected ShortCircuitStrategy::PreSelection"); + }; + // For OR, the mask selects the rows where the LHS is false (the negation + // of the LHS) and the unselected rows are filled with `true`. + assert!(fill_value); + let negated_lhs: Vec<_> = as_boolean_array(array) + .unwrap() + .iter() + .map(|v| v.map(|b| !b)) + .collect(); + let boolean_arr: Vec<_> = mask.iter().collect(); + assert_eq!(negated_lhs, boolean_arr); + + // 60% false: OR falls back to normal evaluation. + let left_expr: Arc = + logical2physical(&logical_col("a").gt(expr_lit(4)), &schema); + let left_value = left_expr.evaluate(&batch).unwrap(); assert!(matches!( check_short_circuit(&left_value, &Operator::Or), ShortCircuitStrategy::None @@ -5525,15 +5565,10 @@ mod tests { )); } - /// Test for [pre_selection_scatter] - /// Since [check_short_circuit] ensures that the left side does not contain null and is neither all_true nor all_false, as well as not being empty, - /// the following tests have been designed: - /// 1. Test sparse left with interleaved true/false - /// 2. Test multiple consecutive true blocks - /// 3. Test multiple consecutive true blocks - /// 4. Test single true at first position - /// 5. Test single true at last position - /// 6. Test nulls in right array + /// Test for [pre_selection_scatter]. + /// + /// `check_short_circuit` only calls this helper with a non-empty, + /// non-null mask that is neither all true nor all false. #[test] fn test_pre_selection_scatter() { fn create_bool_array(bools: Vec) -> BooleanArray { @@ -5546,7 +5581,7 @@ mod tests { let left = create_bool_array(vec![true, false, true, false, true]); let right = create_bool_array(vec![false, true, false]); - let result = pre_selection_scatter(&left, Some(&right)).unwrap(); + let result = pre_selection_scatter(&left, Some(&right), false).unwrap(); let result_arr = result.into_array(left.len()).unwrap(); let expected = create_bool_array(vec![false, false, true, false, false]); @@ -5560,7 +5595,7 @@ mod tests { create_bool_array(vec![false, true, true, false, true, true, true]); let right = create_bool_array(vec![true, false, false, true, false]); - let result = pre_selection_scatter(&left, Some(&right)).unwrap(); + let result = pre_selection_scatter(&left, Some(&right), false).unwrap(); let result_arr = result.into_array(left.len()).unwrap(); let expected = @@ -5574,7 +5609,7 @@ mod tests { let left = create_bool_array(vec![true, false, false]); let right = create_bool_array(vec![false]); - let result = pre_selection_scatter(&left, Some(&right)).unwrap(); + let result = pre_selection_scatter(&left, Some(&right), false).unwrap(); let result_arr = result.into_array(left.len()).unwrap(); let expected = create_bool_array(vec![false, false, false]); @@ -5587,7 +5622,7 @@ mod tests { let left = create_bool_array(vec![false, false, true]); let right = create_bool_array(vec![false]); - let result = pre_selection_scatter(&left, Some(&right)).unwrap(); + let result = pre_selection_scatter(&left, Some(&right), false).unwrap(); let result_arr = result.into_array(left.len()).unwrap(); let expected = create_bool_array(vec![false, false, false]); @@ -5600,7 +5635,7 @@ mod tests { let left = create_bool_array(vec![false, true, false, true]); let right = BooleanArray::from(vec![None, Some(false)]); - let result = pre_selection_scatter(&left, Some(&right)).unwrap(); + let result = pre_selection_scatter(&left, Some(&right), false).unwrap(); let result_arr = result.into_array(left.len()).unwrap(); let expected = BooleanArray::from(vec![ @@ -5611,6 +5646,38 @@ mod tests { ]); assert_eq!(&expected, result_arr.as_boolean()); } + // OR semantics: selected rows take the RHS, unselected rows become true. + { + // Selection (LHS false rows): [T, F, T, F, T] + // Right (RHS on those rows): [F, T, F] + let left = create_bool_array(vec![true, false, true, false, true]); + let right = create_bool_array(vec![false, true, false]); + + let result = pre_selection_scatter(&left, Some(&right), true).unwrap(); + let result_arr = result.into_array(left.len()).unwrap(); + + // selected rows take the RHS value; unselected rows are `true` + let expected = create_bool_array(vec![false, true, true, true, false]); + assert_eq!(&expected, result_arr.as_boolean()); + } + // OR semantics with nulls in the right array. + { + // Selection (LHS false rows): [F, T, F, T] + // Right: [None, Some(false)] + let left = create_bool_array(vec![false, true, false, true]); + let right = BooleanArray::from(vec![None, Some(false)]); + + let result = pre_selection_scatter(&left, Some(&right), true).unwrap(); + let result_arr = result.into_array(left.len()).unwrap(); + + let expected = BooleanArray::from(vec![ + Some(true), // unselected => true + None, // null from right + Some(true), // unselected => true + Some(false), + ]); + assert_eq!(&expected, result_arr.as_boolean()); + } } #[test] @@ -5637,6 +5704,89 @@ mod tests { ); } + #[test] + fn test_or_false_preselection_returns_lhs() { + // `c OR false` over a mostly-true `c` triggers OR pre-selection; the + // result must equal `c`. + let schema = + Arc::new(Schema::new(vec![Field::new("c", DataType::Boolean, false)])); + let c_array = + Arc::new(BooleanArray::from(vec![true, false, true, true, true])) as ArrayRef; + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::clone(&c_array)]) + .unwrap(); + + let expr = logical2physical(&logical_col("c").or(expr_lit(false)), &schema); + + let result = expr.evaluate(&batch).unwrap(); + let ColumnarValue::Array(result_arr) = result else { + panic!("Expected ColumnarValue::Array"); + }; + + let expected: Vec<_> = c_array.as_boolean().iter().collect(); + let actual: Vec<_> = result_arr.as_boolean().iter().collect(); + assert_eq!( + expected, actual, + "OR with FALSE must equal LHS even with PreSelection" + ); + } + + #[test] + fn test_or_preselection_matches_kleene() { + // The OR pre-selection path must match full-batch Kleene OR. + use arrow::compute::kernels::boolean::or_kleene; + + let schema = Arc::new(Schema::new(vec![ + Field::new("c", DataType::Boolean, true), + Field::new("d", DataType::Boolean, true), + ])); + + // `c` is mostly true (2/10 false => 20% <= threshold) so OR pre-selects. + let c = BooleanArray::from(vec![ + true, true, false, true, true, true, true, false, true, true, + ]); + + let d_cases = vec![ + // Mixed RHS with nulls exercises scatter and null copy. + BooleanArray::from(vec![ + Some(false), + Some(true), + Some(true), + Some(false), + Some(false), + Some(true), + Some(false), + None, + Some(true), + None, + ]), + // RHS true on selected rows exercises the uniform-fill path. + BooleanArray::from(vec![Some(true); 10]), + // RHS false on selected rows exercises the return-LHS path. + BooleanArray::from(vec![Some(false); 10]), + ]; + + for d in d_cases { + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(c.clone()) as ArrayRef, + Arc::new(d.clone()) as ArrayRef, + ], + ) + .unwrap(); + + let expr = logical2physical(&logical_col("c").or(logical_col("d")), &schema); + let result = expr.evaluate(&batch).unwrap().into_array(c.len()).unwrap(); + + let expected = or_kleene(&c, &d).unwrap(); + assert_eq!( + expected, + *result.as_boolean(), + "OR pre-selection must match Kleene OR for d = {d:?}" + ); + } + } + #[test] fn test_evaluate_bounds_int32() { let schema = Schema::new(vec![ From b8d3b0b525afaf7049320c426a2be3884600dd77 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Jul 2026 06:21:28 -0600 Subject: [PATCH 006/109] perf: optimize `find_in_set` (up to 24x faster) (#23460) ## Which issue does this PR close? N/A ## Rationale for this change Improve performance of existing expression. ## What changes are included in this PR? Replace per-row O(set_len) linear scan in find_in_set's constant-list path with a one-time HashMap lookup (threshold-guarded so short lists keep the linear scan), giving O(1) per-row probing for large sets. ## Are these changes tested? Existing tests + new tests Benchmark (criterion): - long_list_256: 95.827% faster (base 1246412ns -> cand 52009ns) - ~24x faster - short_list_4: 2.13% faster (base 64343ns -> cand 62972ns) - long_list_64: 88.562% faster (base 422083ns -> cand 48276ns) ## Are there any user-facing changes? No Co-authored-by: Jeffrey Vo --- datafusion/functions/Cargo.toml | 5 + .../functions/benches/find_in_set_literal.rs | 98 +++++++++++++++++++ .../functions/src/unicode/find_in_set.rs | 75 +++++++++++++- 3 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 datafusion/functions/benches/find_in_set_literal.rs diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index d0ce0d0be3b15..83f8c0f2a3299 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -330,6 +330,11 @@ harness = false name = "find_in_set" required-features = ["unicode_expressions"] +[[bench]] +harness = false +name = "find_in_set_literal" +required-features = ["unicode_expressions"] + [[bench]] harness = false name = "contains" diff --git a/datafusion/functions/benches/find_in_set_literal.rs b/datafusion/functions/benches/find_in_set_literal.rs new file mode 100644 index 0000000000000..013c7c2081668 --- /dev/null +++ b/datafusion/functions/benches/find_in_set_literal.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks the `find_in_set(column, constant_list)` path where the set is a +//! scalar literal. A long list exercises the pre-built lookup; a short list +//! stays on the per-row linear scan. + +use arrow::array::StringArray; +use arrow::datatypes::{DataType, Field}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion_common::ScalarValue; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ScalarFunctionArgs}; +use rand::prelude::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const N_ROWS: usize = 8192; + +/// Builds a string column whose values are drawn from `entries` plus a small +/// fraction of misses, so both hits and misses are exercised. +fn build_column(entries: &[String]) -> StringArray { + let mut rng = StdRng::seed_from_u64(42); + let values: Vec> = (0..N_ROWS) + .map(|_| { + let r = rng.random::(); + if r < 0.1 { + None + } else if r < 0.4 { + Some("__miss__".to_string()) + } else { + let idx = rng.random_range(0..entries.len()); + Some(entries[idx].clone()) + } + }) + .collect(); + StringArray::from(values) +} + +fn bench_case(c: &mut Criterion, label: &str, num_entries: usize) { + let find_in_set = datafusion_functions::unicode::find_in_set(); + let entries: Vec = (0..num_entries).map(|i| format!("item{i}")).collect(); + let list = entries.join(","); + + let column = build_column(&entries); + let args = vec![ + ColumnarValue::Array(Arc::new(column)), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(list))), + ]; + let arg_fields = args + .iter() + .map(|arg| Field::new("a", arg.data_type().clone(), true).into()) + .collect::>(); + let return_field = Arc::new(Field::new("f", DataType::Int32, true)); + let config_options = Arc::new(ConfigOptions::default()); + + c.bench_with_input( + BenchmarkId::new("find_in_set_literal", label), + &num_entries, + |b, _| { + b.iter(|| { + black_box(find_in_set.invoke_with_args(ScalarFunctionArgs { + args: args.clone(), + arg_fields: arg_fields.clone(), + number_rows: N_ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config_options), + })) + }) + }, + ); +} + +fn criterion_benchmark(c: &mut Criterion) { + // Short list stays on the linear scan (below the lookup threshold). + bench_case(c, "short_list_4", 4); + // Long lists exercise the pre-built lookup. + bench_case(c, "long_list_64", 64); + bench_case(c, "long_list_256", 256); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/datafusion/functions/src/unicode/find_in_set.rs b/datafusion/functions/src/unicode/find_in_set.rs index 0a83eb3ed61ef..fa23532406ce1 100644 --- a/datafusion/functions/src/unicode/find_in_set.rs +++ b/datafusion/functions/src/unicode/find_in_set.rs @@ -25,7 +25,7 @@ use arrow_buffer::NullBuffer; use crate::utils::utf8_to_int_type; use datafusion_common::{ - Result, ScalarValue, exec_err, internal_err, utils::take_function_args, + HashMap, Result, ScalarValue, exec_err, internal_err, utils::take_function_args, }; use datafusion_expr::TypeSignature::Exact; use datafusion_expr::{ @@ -316,6 +316,11 @@ where Ok(Arc::new(PrimitiveArray::::new(values.into(), nulls)) as ArrayRef) } +/// Minimum set length at which a pre-built lookup beats a per-row linear scan. +/// Below this, the linear scan's small constant factor wins, so short sets are +/// left untouched to avoid regressing them. +const FIND_IN_SET_LOOKUP_THRESHOLD: usize = 16; + fn find_in_set_right_literal<'a, T, V>( string_array: V, str_list: &[&str], @@ -329,16 +334,34 @@ where let nulls = string_array.nulls().cloned(); let zero = T::Native::from_usize(0).unwrap(); + // The set (`str_list`) is constant across all rows. For a large set, the + // per-row `position` linear scan is O(set_len). Building a lookup from each + // distinct entry to its 1-based position once turns each row into an O(1) + // probe (first occurrence wins, exactly matching `position`). Below the + // threshold the linear scan's small constant factor is faster, so the map is + // built at most once here rather than per row. + let map: Option> = + (str_list.len() >= FIND_IN_SET_LOOKUP_THRESHOLD).then(|| { + let mut map = HashMap::with_capacity(str_list.len()); + for (idx, entry) in str_list.iter().enumerate() { + map.entry(*entry).or_insert(idx + 1); + } + map + }); + let values: Vec = (0..len) .map(|i| { if nulls.as_ref().is_some_and(|n| n.is_null(i)) { return zero; } let string = string_array.value(i); - let position = str_list - .iter() - .position(|s| *s == string) - .map_or(0, |idx| idx + 1); + let position = match &map { + Some(map) => map.get(string).copied().unwrap_or(0), + None => str_list + .iter() + .position(|s| *s == string) + .map_or(0, |idx| idx + 1), + }; T::Native::from_usize(position).unwrap() }) .collect(); @@ -545,4 +568,46 @@ mod tests { ], Int32Array::from(vec![None::; 3]) ); + + // Exercises both the lookup-map path (list length >= threshold) and the + // linear-scan path (short list), including a duplicate entry to confirm the + // first occurrence wins in both. + #[test] + fn test_right_literal_lookup_matches_linear() { + use super::find_in_set_right_literal; + use arrow::datatypes::Int32Type; + + // 40 unique entries plus a duplicate of "item5" appended at index 40, so + // the length is well over FIND_IN_SET_LOOKUP_THRESHOLD. + let mut long_list: Vec = (0..40).map(|i| format!("item{i}")).collect(); + long_list.push("item5".to_string()); + let long_refs: Vec<&str> = long_list.iter().map(|s| s.as_str()).collect(); + let short_refs = ["a", "b", "c"]; + + let strings = StringArray::from(vec![ + Some("item0"), + Some("item39"), + Some("item5"), + Some("missing"), + None, + Some("b"), + ]); + + let long = + find_in_set_right_literal::(&strings, &long_refs).unwrap(); + let long = long.as_any().downcast_ref::().unwrap(); + assert_eq!(long.value(0), 1); + assert_eq!(long.value(1), 40); + assert_eq!(long.value(2), 6); // first occurrence of "item5" + assert_eq!(long.value(3), 0); + assert!(long.is_null(4)); + assert_eq!(long.value(5), 0); + + let short = + find_in_set_right_literal::(&strings, &short_refs).unwrap(); + let short = short.as_any().downcast_ref::().unwrap(); + assert_eq!(short.value(0), 0); + assert!(short.is_null(4)); + assert_eq!(short.value(5), 2); // "b" at position 2 + } } From 82f1b3646d8cb2c580e007d9db5713e77a7fc31f Mon Sep 17 00:00:00 2001 From: Nathan <56370526+nathanb9@users.noreply.github.com> Date: Fri, 24 Jul 2026 08:21:56 -0400 Subject: [PATCH 007/109] fix: NOT IN with NULL subquery returns wrong results under SortMergeJoin (#22810) ## Problem `NOT IN (subquery)` is a null-aware anti join: when the subquery yields a NULL the predicate is never TRUE, so the query must return zero rows. With `prefer_hash_join = false` and multiple partitions, the planner routed the null-aware anti join to `SortMergeJoinExec`, which is not null-aware, so it returned wrong results. HashJoin (the default) was already correct. ## Proof ```sql set datafusion.optimizer.prefer_hash_join = false; create table t1(x int) as values (1); create table t2(y int) as values (NULL); select x from t1 where x not in (select y from t2); ``` Expected 0 rows (the subquery contains a NULL). Before this change it returned `1`. With `prefer_hash_join = true` it correctly returned 0 rows. `EXPLAIN` showed the wrong config selecting `SortMergeJoinExec: join_type=LeftAnti`. ## Solution The planner already requires null-aware joins to use the CollectLeft HashJoin, and the HashJoin branch guards on `!null_aware`. The SortMergeJoin branch was missing the same guard, so this adds `&& !*null_aware` to it. Null-aware anti joins now fall through to the CollectLeft HashJoin regardless of `prefer_hash_join`. `SortMergeJoinExec` has no `null_aware` parameter and cannot honor these semantics. Added a regression test in `subquery.slt` (under `prefer_hash_join = false`) covering both a null-containing subquery (zero rows) and a null-free subquery (normal anti join). All 61 SortMergeJoin unit tests pass. --- datafusion/core/src/physical_planner.rs | 5 ++ .../sqllogictest/test_files/subquery.slt | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index aef8036c749a8..4e914556b4cc0 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -1803,6 +1803,11 @@ impl DefaultPhysicalPlanner { } else if session_state.config().target_partitions() > 1 && session_state.config().repartition_joins() && !prefer_hash_join + && !*null_aware + // Null-aware joins (e.g. `NOT IN` with a nullable subquery) must + // use the CollectLeft HashJoin below: SortMergeJoinExec does not + // implement null-aware anti-join semantics and would return wrong + // results when the right side contains a null join key. { // Use SortMergeJoin if hash join is not preferred let join_on_len = join_on.len(); diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt index 325cff62d3986..dcca13c4164c5 100644 --- a/datafusion/sqllogictest/test_files/subquery.slt +++ b/datafusion/sqllogictest/test_files/subquery.slt @@ -2599,3 +2599,61 @@ DROP TABLE sq_count_customer; statement ok DROP TABLE sq_count_orders; + +# Regression test: `NOT IN` is a null-aware anti join. When the subquery yields a +# NULL the predicate is never TRUE, so the query must return zero rows. This must +# hold regardless of the chosen physical join operator. Previously, with +# prefer_hash_join = false and multiple partitions, the planner routed the +# null-aware anti join to SortMergeJoin (which is not null-aware) and returned +# wrong results; null-aware anti joins must use the CollectLeft HashJoin. + +statement ok +set datafusion.optimizer.prefer_hash_join = false; + +statement ok +CREATE TABLE nia_left(x INT) AS VALUES (1), (2), (3), (4); + +statement ok +CREATE TABLE nia_right_with_null(y INT) AS VALUES (2), (NULL); + +statement ok +CREATE TABLE nia_right_no_null(y INT) AS VALUES (2), (4); + +# Subquery contains a NULL -> NOT IN must return no rows. +query I +SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null) ORDER BY x; +---- + +# The null-aware anti join must be planned as a CollectLeft HashJoinExec even with +# prefer_hash_join = false: SortMergeJoinExec is not null-aware and must not be used. +query TT +EXPLAIN SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null); +---- +logical_plan +01)LeftAnti Join: nia_left.x = __correlated_sq_1.y null_aware +02)--TableScan: nia_left projection=[x] +03)--SubqueryAlias: __correlated_sq_1 +04)----TableScan: nia_right_with_null projection=[y] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(x@0, y@0)], null_aware +02)--DataSourceExec: partitions=1, partition_sizes=[1] +03)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Subquery has no NULL -> NOT IN behaves like a normal anti join. +query I +SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_no_null) ORDER BY x; +---- +1 +3 + +statement ok +DROP TABLE nia_left; + +statement ok +DROP TABLE nia_right_with_null; + +statement ok +DROP TABLE nia_right_no_null; + +statement ok +reset datafusion.optimizer.prefer_hash_join; From 3f816137306112f75b412e2806abd24fed0b916f Mon Sep 17 00:00:00 2001 From: linfeng <33561138+lyne7-sc@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:22:36 +0800 Subject: [PATCH 008/109] Remove `GroupsAccumulator::supports_convert_to_state` and require `convert_to_state` (#23489) ## Which issue does this PR close? - Closes #23081. ## Rationale for this change Following #23275, all `GroupsAccumulator` implementations now provide `convert_to_state`. The `supports_convert_to_state` capability flag is therefore no longer needed. ## What changes are included in this PR? - Make `GroupsAccumulator::convert_to_state` a required trait method. - Remove `GroupsAccumulator::supports_convert_to_state` and its implementations. - Remove the corresponding capability checks from hash aggregation. - Simplify skip-partial aggregation to use the required `convert_to_state` implementation directly. - Add a regression test covering the partial hash aggregation skip path. - Document the breaking trait change in the 55.0.0 upgrading guide. - Remove `FFI_GroupsAccumulator::supports_convert_to_state`. This changes the FFI ABI layout, so providers and consumers must be rebuilt against DataFusion 55. ## Are these changes tested? Yes. Added a regression test verifying that skip-partial aggregation uses the required `convert_to_state` implementation without a capability flag. Existing physical-plan and FFI tests continue to pass. ## Are there any user-facing changes? Yes. This is a breaking Rust API change for external `GroupsAccumulator` implementations: - `convert_to_state` must now be implemented. - `supports_convert_to_state` should be removed. The migration is documented in the 55.0.0 upgrading guide. The `FFI_GroupsAccumulator` layout has changed. FFI providers and consumers must be rebuilt against DataFusion 55 and must not exchange this struct with older major versions. --- .../examples/udf/advanced_udaf.rs | 5 --- .../user_defined/user_defined_aggregates.rs | 5 --- .../expr-common/src/groups_accumulator.rs | 16 +++------ datafusion/ffi/src/udaf/groups_accumulator.rs | 8 ----- .../src/aggregate/count_distinct/groups.rs | 5 --- .../src/aggregate/groups_accumulator.rs | 4 --- .../aggregate/groups_accumulator/bool_op.rs | 4 --- .../aggregate/groups_accumulator/prim_op.rs | 5 --- .../src/approx_distinct.rs | 5 --- .../functions-aggregate/src/array_agg.rs | 5 --- datafusion/functions-aggregate/src/average.rs | 5 --- .../functions-aggregate/src/correlation.rs | 5 --- datafusion/functions-aggregate/src/count.rs | 5 --- .../functions-aggregate/src/first_last.rs | 5 --- datafusion/functions-aggregate/src/median.rs | 5 --- .../src/min_max/min_max_bytes.rs | 5 --- .../src/min_max/min_max_struct.rs | 5 --- .../src/percentile_cont.rs | 5 --- datafusion/functions-aggregate/src/stddev.rs | 5 --- .../functions-aggregate/src/string_agg.rs | 5 --- .../functions-aggregate/src/variance.rs | 5 --- .../aggregates/aggregate_hash_table/common.rs | 4 --- .../aggregate_hash_table/partial_table.rs | 8 ----- .../src/aggregates/grouped_hash_stream.rs | 8 +---- .../src/aggregates/hash_stream.rs | 4 +-- .../physical-plan/src/aggregates/mod.rs | 16 +++++++++ .../spark/src/function/aggregate/avg.rs | 11 ------ .../library-user-guide/upgrading/55.0.0.md | 34 +++++++++++++++++++ 28 files changed, 56 insertions(+), 146 deletions(-) diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rs index 096753d2b5d7b..bca4c7edab2c5 100644 --- a/datafusion-examples/examples/udf/advanced_udaf.rs +++ b/datafusion-examples/examples/udf/advanced_udaf.rs @@ -393,11 +393,6 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator { Arc::new(counts) as ArrayRef, ]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.counts.capacity() * size_of::() + self.prods.capacity() * size_of::() diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs index 1d4b22230147f..323925bcfaf82 100644 --- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs +++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs @@ -888,11 +888,6 @@ impl GroupsAccumulator for TestGroupsAccumulator { as ArrayRef, ]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { size_of::() } diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rs index 13b2f853c95dc..5c01418e04ce7 100644 --- a/datafusion/expr-common/src/groups_accumulator.rs +++ b/datafusion/expr-common/src/groups_accumulator.rs @@ -18,7 +18,7 @@ //! Vectorized [`GroupsAccumulator`] use arrow::array::{ArrayRef, BooleanArray}; -use datafusion_common::{Result, not_impl_err, utils::split_vec_min_alloc}; +use datafusion_common::{Result, utils::split_vec_min_alloc}; /// Describes how many rows should be emitted during grouping. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -231,17 +231,9 @@ pub trait GroupsAccumulator: Send + std::any::Any { /// [`Accumulator::state`]: crate::accumulator::Accumulator::state fn convert_to_state( &self, - _values: &[ArrayRef], - _opt_filter: Option<&BooleanArray>, - ) -> Result> { - not_impl_err!("Input batch conversion to state not implemented") - } - - /// Returns `true` if [`Self::convert_to_state`] is implemented to support - /// intermediate aggregate state conversion. - fn supports_convert_to_state(&self) -> bool { - false - } + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result>; /// Amount of memory used to store the state of this accumulator, /// in bytes. diff --git a/datafusion/ffi/src/udaf/groups_accumulator.rs b/datafusion/ffi/src/udaf/groups_accumulator.rs index 272afdb6abfb1..4d1b0b4be0a2b 100644 --- a/datafusion/ffi/src/udaf/groups_accumulator.rs +++ b/datafusion/ffi/src/udaf/groups_accumulator.rs @@ -73,8 +73,6 @@ pub struct FFI_GroupsAccumulator { opt_filter: FFI_Option, ) -> FFI_Result>, - pub supports_convert_to_state: bool, - /// Release the memory of the private data when it is no longer being used. pub release: unsafe extern "C" fn(accumulator: &mut Self), @@ -247,7 +245,6 @@ impl From> for FFI_GroupsAccumulator { return accumulator.accumulator; } - let supports_convert_to_state = accumulator.supports_convert_to_state(); let private_data = GroupsAccumulatorPrivateData { accumulator }; Self { @@ -257,7 +254,6 @@ impl From> for FFI_GroupsAccumulator { state: state_fn_wrapper, merge_batch: merge_batch_fn_wrapper, convert_to_state: convert_to_state_fn_wrapper, - supports_convert_to_state, release: release_fn_wrapper, private_data: Box::into_raw(Box::new(private_data)) as *mut c_void, @@ -421,10 +417,6 @@ impl GroupsAccumulator for ForeignGroupsAccumulator { .collect() } } - - fn supports_convert_to_state(&self) -> bool { - self.accumulator.supports_convert_to_state - } } #[repr(C)] diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs index 986d4ec0d71ae..10aa21c3acad2 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs @@ -207,11 +207,6 @@ where Ok(vec![Arc::new(builder.finish())]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { size_of::() + self.seen.capacity() * (size_of::<(usize, T::Native)>() + size_of::()) diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs index b412b4ffe09f2..b5610419166df 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs @@ -441,10 +441,6 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter { Ok(arrays) } - - fn supports_convert_to_state(&self) -> bool { - true - } } /// Extension trait for [`Vec`] to account for allocations. diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs index afb1dec24a484..77bb7598e2747 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/bool_op.rs @@ -156,8 +156,4 @@ where Ok(vec![Arc::new(values_filtered)]) } - - fn supports_convert_to_state(&self) -> bool { - true - } } diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs index 474899d8f3c6a..c5d74978664c9 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs @@ -189,11 +189,6 @@ where Ok(vec![Arc::new(state_values)]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.values.capacity() * size_of::() + self.null_state.size() } diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs index f36c658e7f385..1746edd8239f2 100644 --- a/datafusion/functions-aggregate/src/approx_distinct.rs +++ b/datafusion/functions-aggregate/src/approx_distinct.rs @@ -615,11 +615,6 @@ impl GroupsAccumulator for HllGroupsAccumulator { Ok(vec![Arc::new(builder.finish())]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.groups.capacity() * size_of::() + self.allocated_bytes diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index 1937f17973950..b563a6389ec7e 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -793,11 +793,6 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { Ok(vec![Arc::new(list_array)]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.batches .iter() diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs index f1159f22b2de0..e5030bf39e409 100644 --- a/datafusion/functions-aggregate/src/average.rs +++ b/datafusion/functions-aggregate/src/average.rs @@ -1133,11 +1133,6 @@ where Ok(vec![Arc::new(counts) as ArrayRef, Arc::new(sums)]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { // Heap buffers self.counts.capacity() * size_of::() diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rs index 2e90cac6d9298..b9bc57dfa989c 100644 --- a/datafusion/functions-aggregate/src/correlation.rs +++ b/datafusion/functions-aggregate/src/correlation.rs @@ -539,11 +539,6 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator { Arc::new(Float64Array::from(sum_yy)), ]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn merge_batch( &mut self, values: &[ArrayRef], diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs index 983828ea90b7c..f0de9d9848627 100644 --- a/datafusion/functions-aggregate/src/count.rs +++ b/datafusion/functions-aggregate/src/count.rs @@ -773,11 +773,6 @@ impl GroupsAccumulator for CountGroupsAccumulator { Ok(vec![state_array]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.counts.capacity() * size_of::() } diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs index 7c2540aa03a2b..e2da7ec753aa5 100644 --- a/datafusion/functions-aggregate/src/first_last.rs +++ b/datafusion/functions-aggregate/src/first_last.rs @@ -719,11 +719,6 @@ impl GroupsAccumulator for FirstLastGroupsAccumulator() + self.extreme_of_each_group_buf.1.capacity() / 8 } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn convert_to_state( &self, values: &[ArrayRef], diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs index 9a6ef3e7e5fc5..7a399f73ec8e2 100644 --- a/datafusion/functions-aggregate/src/median.rs +++ b/datafusion/functions-aggregate/src/median.rs @@ -535,11 +535,6 @@ impl GroupsAccumulator for MedianGroupsAccumulator bool { - true - } - fn size(&self) -> usize { self.group_values .iter() diff --git a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs index 7a3c605d82e4d..efeaea314c4f5 100644 --- a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs +++ b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs @@ -325,11 +325,6 @@ impl GroupsAccumulator for MinMaxBytesAccumulator { let output = apply_filter_as_nulls(&values[0], opt_filter)?; Ok(vec![output]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.inner.size() } diff --git a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs b/datafusion/functions-aggregate/src/min_max/min_max_struct.rs index 15df0f1d44eff..d1bac4e2f90db 100644 --- a/datafusion/functions-aggregate/src/min_max/min_max_struct.rs +++ b/datafusion/functions-aggregate/src/min_max/min_max_struct.rs @@ -150,11 +150,6 @@ impl GroupsAccumulator for MinMaxStructAccumulator { let output = apply_filter_as_nulls(&values[0], opt_filter)?; Ok(vec![output]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.inner.size() } diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs index e8e6fd127e65d..cfab1303028ad 100644 --- a/datafusion/functions-aggregate/src/percentile_cont.rs +++ b/datafusion/functions-aggregate/src/percentile_cont.rs @@ -652,11 +652,6 @@ where Ok(vec![Arc::new(converted_list_array)]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.group_values .iter() diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rs index a31517b93e003..15511bf4a565f 100644 --- a/datafusion/functions-aggregate/src/stddev.rs +++ b/datafusion/functions-aggregate/src/stddev.rs @@ -352,11 +352,6 @@ impl GroupsAccumulator for StddevGroupsAccumulator { ) -> Result> { self.variance.convert_to_state(values, opt_filter) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.variance.size() } diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs index 6b0665f479d78..3fe2b0a186ae3 100644 --- a/datafusion/functions-aggregate/src/string_agg.rs +++ b/datafusion/functions-aggregate/src/string_agg.rs @@ -432,11 +432,6 @@ impl GroupsAccumulator for StringAggGroupsAccumulator { }; Ok(vec![result]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.total_data_bytes + self.values.capacity() * size_of::>() diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs index 0278ce2c233e4..df652731ff4f4 100644 --- a/datafusion/functions-aggregate/src/variance.rs +++ b/datafusion/functions-aggregate/src/variance.rs @@ -613,11 +613,6 @@ impl GroupsAccumulator for VarianceGroupsAccumulator { Arc::new(Float64Array::new(m2s.into(), None)), ]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.m2s.capacity() * size_of::() + self.means.capacity() * size_of::() diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs index eaf39929ced62..42014f336f3d8 100644 --- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs +++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs @@ -569,10 +569,6 @@ impl HashAggregateAccumulator { self.accumulator.state(emit_to) } - pub(super) fn supports_convert_to_state(&self) -> bool { - self.accumulator.supports_convert_to_state() - } - pub(super) fn convert_to_state( &mut self, values: &EvaluatedAccumulatorArgs, diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs index ffac42feaa3b3..4bcacb49afb04 100644 --- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs +++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs @@ -68,14 +68,6 @@ impl AggregateHashTable { self.next_output_batch_inner(HashAggregateAccumulator::state) } - pub(in crate::aggregates) fn can_skip_aggregation(&self) -> bool { - self.state - .building() - .accumulators - .iter() - .all(|acc| acc.supports_convert_to_state()) - } - /// In skip-partial-aggregation optimization, when a decision has been made to skip /// partial stage, build a typed hash table only for aggregation state conversion /// row-by-row. diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs index 0d00e5c4d0d86..99c101199459f 100644 --- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs +++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs @@ -217,8 +217,7 @@ enum OutOfMemoryMode { /// aggregator must store the intermediate state for each group. /// /// If the ratio of the number of groups to the number of input rows exceeds a -/// threshold, and [`GroupsAccumulator::supports_convert_to_state`] is -/// supported, this operator will stop applying Partial aggregation and directly +/// threshold, this operator will stop applying Partial aggregation and directly /// pass the input rows to the next aggregation phase. /// /// [`Accumulator::state`]: datafusion_expr::Accumulator::state @@ -545,14 +544,9 @@ impl GroupedHashAggregateStream { // - aggregation mode is Partial // - input is not ordered by GROUP BY expressions, // since Final mode expects unique group values as its input - // - all accumulators support input batch to intermediate - // aggregate state conversion // - there is only one GROUP BY expressions set let skip_aggregation_probe = if agg.mode == AggregateMode::Partial && matches!(group_ordering, GroupOrdering::None) - && accumulators - .iter() - .all(|acc| acc.supports_convert_to_state()) && agg_group_by.is_single() { let options = &context.session_config().options().execution; diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rs index 62b92965030ae..e7f0f075b33a5 100644 --- a/datafusion/physical-plan/src/aggregates/hash_stream.rs +++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs @@ -293,9 +293,7 @@ impl PartialHashAggregateStream { Arc::clone(&schema), batch_size, )?; - let can_skip_aggregation = - agg.group_by.is_single() && hash_table.can_skip_aggregation(); - let skip_aggregation_probe = if can_skip_aggregation { + let skip_aggregation_probe = if agg.group_by.is_single() { let options = &context.session_config().options().execution; let probe_ratio_threshold = options.skip_partial_aggregation_probe_ratio_threshold; diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 7d93cd739815d..e3cf1c4568009 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -7132,6 +7132,22 @@ mod tests { Ok(vec![self.emit_counts(emit_to)?]) } + fn convert_to_state( + &self, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, + ) -> Result> { + assert_eq!(values.len(), 1, "one argument to convert_to_state"); + let counts = match opt_filter { + Some(filter) => filter + .iter() + .map(|value| i64::from(value.unwrap_or(false))) + .collect::>(), + None => vec![1; values[0].len()], + }; + Ok(vec![Arc::new(Int64Array::from(counts))]) + } + fn merge_batch( &mut self, _values: &[ArrayRef], diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rs index 6ca3c59309e70..46e63013dbafb 100644 --- a/datafusion/spark/src/function/aggregate/avg.rs +++ b/datafusion/spark/src/function/aggregate/avg.rs @@ -367,11 +367,6 @@ where Arc::new(counts) as ArrayRef, ]) } - - fn supports_convert_to_state(&self) -> bool { - true - } - fn size(&self) -> usize { self.counts.capacity() * size_of::() + self.sums.capacity() * size_of::() } @@ -387,12 +382,6 @@ mod tests { Ok(sum / count as f64) }) } - - #[test] - fn supports_convert_to_state() { - assert!(make_acc().supports_convert_to_state()); - } - #[test] fn convert_to_state_basic() { let acc = make_acc(); diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index 57da7b7dac248..5f1609af30fc6 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -276,6 +276,40 @@ it was `None`), that code can simply be deleted. See [issue #22775](https://github.com/apache/datafusion/issues/22775) for details. +### `GroupsAccumulator::convert_to_state` is now required + +`datafusion_expr_common::groups_accumulator::GroupsAccumulator::convert_to_state` +no longer provides a default implementation, and the +`GroupsAccumulator::supports_convert_to_state` capability method has been +removed. All `GroupsAccumulator` implementations must now support converting +input batches directly to intermediate aggregate state. + +**Who is affected:** + +- Users with custom `GroupsAccumulator` implementations. +- FFI providers and consumers that use `FFI_GroupsAccumulator`. + +**Migration guide:** + +Custom `GroupsAccumulator` implementations must now provide their own +`convert_to_state` implementation. + +Delete `supports_convert_to_state` implementations because `convert_to_state` +is now required: + +```diff +- fn supports_convert_to_state(&self) -> bool { +- true +- } +``` + +The `supports_convert_to_state` field has also been removed from +`datafusion_ffi::udaf::groups_accumulator::FFI_GroupsAccumulator`, changing its +ABI layout. Rebuild both FFI providers and consumers against DataFusion 55, and +do not exchange this struct with libraries built against older major versions. + +See [issue #23081](https://github.com/apache/datafusion/issues/23081) for details. + ### `is_dynamic_physical_expr` is deprecated `datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr` is From 7d0ca3e8f81d7263d382279bf56c37d398340e42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 07:58:35 -0700 Subject: [PATCH 009/109] chore(deps-dev): bump ws from 8.18.2 to 8.21.1 in /datafusion/wasmtest/datafusion-wasm-app (#23866) Bumps [ws](https://github.com/websockets/ws) from 8.18.2 to 8.21.1.
Release notes

Sourced from ws's releases.

8.21.1

Bug fixes

  • Empty fragments are now counted toward the limit (a2f4e7c0).
  • The default values of the maxBufferedChunks and maxFragments options have been reduced (f197ac65).

8.21.0

Features

  • Introduced the maxBufferedChunks and maxFragments options (2b2abd45).

Bug fixes

  • Fixed a remote memory exhaustion DoS vulnerability (2b2abd45).

A high volume of tiny fragments and data chunks could be sent by a peer, using modest network traffic, to crash a ws server or client due to OOM.

import { WebSocket, WebSocketServer } from 'ws';

const wss = new WebSocketServer({ port: 0 }, function () {
const data = Buffer.alloc(1);
const options = { fin: false };
const { port } = wss.address();
const ws = new WebSocket(ws://localhost:${port});


ws.on('open', function () {
(function send() {
ws.send(data, options, function (err) {
if (err) return;
send();
});
})();
});


ws.on('error', console.error);
ws.on('close', function (code, reason) {
console.log(client close - code: ${code} reason:
${reason.toString()});
});
});


wss.on('connection', function (ws) {
ws.on('error', console.error);
ws.on('close', function (code, reason) {
console.log(server close - code: ${code} reason:
${reason.toString()});
});
});

... (truncated)

Commits
  • ae1de54 [dist] 8.21.1
  • 8e9511b [ci] Trust Coveralls Homebrew tap
  • f197ac6 [fix] Lower default values of maxBufferedChunks and maxFragments
  • 8df8265 [ci] Update actions/checkout action to v7
  • a2f4e7c [fix] Count empty fragments toward the limit (#2329)
  • e79f912 [pkg] Approve install scripts for bufferutil and utf-8-validate
  • 4ea355d [doc] Document 32-bit signed integer coercion for option values
  • 2120f4c [example] Remove uuid dependency
  • 4c534a6 [security] Add latest vulnerability to SECURITY.md
  • bca91ad [dist] 8.21.0
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ws&package-manager=npm_and_yarn&previous-version=8.18.2&new-version=8.21.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/datafusion/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../wasmtest/datafusion-wasm-app/package-lock.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json index b2a72228e8115..7f51bac7a1c59 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json @@ -4158,11 +4158,10 @@ "dev": true }, "node_modules/ws": { - "version": "8.18.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", - "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", "dev": true, - "license": "MIT", "engines": { "node": ">=10.0.0" }, @@ -7225,9 +7224,9 @@ "dev": true }, "ws": { - "version": "8.18.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", - "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", "dev": true, "requires": {} } From 0a8eacf1fb93523aea71707ae77bb290801d8040 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 07:58:54 -0700 Subject: [PATCH 010/109] chore(deps-dev): bump http-proxy-middleware from 2.0.9 to 2.0.10 in /datafusion/wasmtest/datafusion-wasm-app (#23865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [http-proxy-middleware](https://github.com/chimurai/http-proxy-middleware) from 2.0.9 to 2.0.10.
Release notes

Sourced from http-proxy-middleware's releases.

v2.0.10-beta.0

What's Changed

New Contributors

Full Changelog: https://github.com/chimurai/http-proxy-middleware/compare/v2.0.9...v2.0.10-beta.0

Changelog

Sourced from http-proxy-middleware's changelog.

v2.0.10

  • fix(router): harden proxy-table matching (exact host for host+path keys, prefix-only path matching) to prevent routing bypass
Commits
Maintainer changes

This version was pushed to npm by GitHub Actions, a new releaser for http-proxy-middleware since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=http-proxy-middleware&package-manager=npm_and_yarn&previous-version=2.0.9&new-version=2.0.10)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/datafusion/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../wasmtest/datafusion-wasm-app/package-lock.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json index 7f51bac7a1c59..e863fe5e8da15 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json @@ -2142,11 +2142,10 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", - "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", + "integrity": "sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", @@ -5839,9 +5838,9 @@ } }, "http-proxy-middleware": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", - "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", + "integrity": "sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ==", "dev": true, "requires": { "@types/http-proxy": "^1.17.8", From 592eeab9cca76fc84fc12a4c03e46967eaa895e2 Mon Sep 17 00:00:00 2001 From: Jeffrey Vo Date: Fri, 24 Jul 2026 23:59:51 +0900 Subject: [PATCH 011/109] Add codecov badge to README (#23860) add badge; easier to see at a glance what the coverage is, and also easier to access our codecov page image --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index dfffcbddfaae7..73c4409ef9b54 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ [![Discord chat][discord-badge]][discord-url] [![Linkedin][linkedin-badge]][linkedin-url] ![Crates.io MSRV][msrv-badge] +[![Codecov][codecov-badge]][codecov-url] [crates-badge]: https://img.shields.io/crates/v/datafusion.svg [crates-url]: https://crates.io/crates/datafusion @@ -45,6 +46,8 @@ [linkedin-badge]: https://img.shields.io/badge/Follow-Linkedin-blue [linkedin-url]: https://www.linkedin.com/company/apache-datafusion/ [msrv-badge]: https://img.shields.io/crates/msrv/datafusion?label=Min%20Rust%20Version +[codecov-badge]: https://codecov.io/github/apache/datafusion/graph/badge.svg +[codecov-url]: https://app.codecov.io/github/apache/datafusion/tree/main [Website](https://datafusion.apache.org/) | [API Docs](https://docs.rs/datafusion/latest/datafusion/) | From 582453b680e7a9b927d659753edb8f33a444a3c8 Mon Sep 17 00:00:00 2001 From: Matthew Kim <38759997+friendlymatthew@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:15:05 -0400 Subject: [PATCH 012/109] fix: align physical CASE nullability through casts (#23844) ## Rationale for this change Logical `CASE` nullability unwraps null preserving casts before analyzing guarded branches, but physical `CASE` nullability did not. Type coercion could therefore produce conflicting schemas and cause valid aggregation queries to fail during planning --------- Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- .../physical-expr/src/expressions/case.rs | 81 ++++++++++++++++++- datafusion/sqllogictest/test_files/case.slt | 13 +++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs index 8a0f15467c47b..17288a9737699 100644 --- a/datafusion/physical-expr/src/expressions/case.rs +++ b/datafusion/physical-expr/src/expressions/case.rs @@ -19,7 +19,9 @@ mod literal_lookup_table; use super::{Column, Literal}; use crate::PhysicalExpr; -use crate::expressions::{LambdaVariable, lit, try_cast}; +use crate::expressions::{ + CastExpr, LambdaVariable, NegativeExpr, NotExpr, lit, try_cast, +}; use arrow::array::*; use arrow::compute::kernels::zip::zip; use arrow::compute::{ @@ -1278,7 +1280,11 @@ impl PhysicalExpr for CaseExpr { // it would evaluate to null. // Replace the `then` expression with `NULL` in the `when` expression - let with_null = match replace_with_null(w, t.as_ref(), input_schema) { + let with_null = match replace_with_null( + w, + unwrap_certainly_null_expr(t.as_ref()), + input_schema, + ) { Err(e) => return Some(Err(e)), Ok(e) => e, }; @@ -1537,6 +1543,25 @@ fn replace_with_null( Ok(with_null) } +/// Returns the innermost [`PhysicalExpr`] that is provably null if `expr` is null. +/// +/// Keep this in sync with the logical-plan equivalent, `unwrap_certainly_null_expr` +/// in `datafusion/expr/src/expr_schema.rs`. If the two disagree on which wrappers +/// are null-preserving, `CASE` nullability computed by the logical and physical +/// planners can diverge and cause a schema mismatch during planning. +/// See for rationale. +fn unwrap_certainly_null_expr(expr: &dyn PhysicalExpr) -> &dyn PhysicalExpr { + if let Some(expr) = expr.downcast_ref::() { + unwrap_certainly_null_expr(expr.arg().as_ref()) + } else if let Some(expr) = expr.downcast_ref::() { + unwrap_certainly_null_expr(expr.arg().as_ref()) + } else if let Some(expr) = expr.downcast_ref::() { + unwrap_certainly_null_expr(expr.expr.as_ref()) + } else { + expr + } +} + /// Create a CASE expression pub fn case( expr: Option>, @@ -2577,10 +2602,45 @@ mod tests { let zero = lit(0); let foo_eq_zero = binary(Arc::clone(&foo), Operator::Eq, Arc::clone(&zero), &schema)?; + let cast_foo = cast(Arc::clone(&foo), &schema, DataType::Int64)?; + let negative_foo = expressions::negative(Arc::clone(&foo), &schema)?; assert_not_nullable(when_then_else(&foo_is_not_null, &foo, &zero)?, &schema); assert_not_nullable(when_then_else(¬_foo_is_null, &foo, &zero)?, &schema); assert_not_nullable(when_then_else(&foo_eq_zero, &foo, &zero)?, &schema); + assert_not_nullable( + when_then_else(&foo_is_not_null, &cast_foo, &lit(0i64))?, + &schema, + ); + assert_not_nullable( + when_then_else(&foo_is_not_null, &negative_foo, &zero)?, + &schema, + ); + + // Nested null-preserving wrappers must be unwrapped recursively. `CAST(-foo)` + // still collapses `foo IS NOT NULL` to `false`, so the branch is + // unreachable-as-null and the `CASE` is not nullable. + let cast_negative_foo = cast( + expressions::negative(Arc::clone(&foo), &schema)?, + &schema, + DataType::Int64, + )?; + assert_not_nullable( + when_then_else(&foo_is_not_null, &cast_negative_foo, &lit(0i64))?, + &schema, + ); + + // `TRY_CAST` is intentionally NOT treated as null-preserving: it yields + // NULL on a failed cast even for a non-null input, so a guarded `TRY_CAST` + // branch is still reachable-as-null and the `CASE` stays nullable. This must + // stay consistent with the logical planner (`unwrap_certainly_null_expr` in + // `datafusion/expr/src/expr_schema.rs`); unwrapping it on only one side would + // reintroduce a logical/physical schema mismatch. + let try_cast_foo = try_cast(Arc::clone(&foo), &schema, DataType::Int64)?; + assert_nullable( + when_then_else(&foo_is_not_null, &try_cast_foo, &lit(0i64))?, + &schema, + ); assert_not_nullable( when_then_else( @@ -2702,6 +2762,23 @@ mod tests { &schema, ); + let boolean_schema = + Schema::new(vec![Field::new("predicate", DataType::Boolean, true)]); + let predicate = col("predicate", &boolean_schema)?; + let predicate_is_not_null = is_not_null(Arc::clone(&predicate))?; + let not_predicate = expressions::not(Arc::clone(&predicate))?; + assert_not_nullable( + when_then_else(&predicate_is_not_null, ¬_predicate, &lit(false))?, + &boolean_schema, + ); + + // Nested `NOT` is likewise unwrapped recursively. + let not_not_predicate = expressions::not(Arc::clone(¬_predicate))?; + assert_not_nullable( + when_then_else(&predicate_is_not_null, ¬_not_predicate, &lit(false))?, + &boolean_schema, + ); + Ok(()) } diff --git a/datafusion/sqllogictest/test_files/case.slt b/datafusion/sqllogictest/test_files/case.slt index 3953878ceb666..f7ae380242942 100644 --- a/datafusion/sqllogictest/test_files/case.slt +++ b/datafusion/sqllogictest/test_files/case.slt @@ -41,6 +41,19 @@ NULL 6 7 +# CASE nullability remains consistent through type coercion +query I +SELECT count(endpoint) +FROM ( + SELECT CASE + WHEN a IS NOT NULL THEN CAST(a AS BIGINT) + ELSE CAST(0 AS BIGINT) + END AS endpoint + FROM foo +) +---- +6 + # column or explicit null query I SELECT CASE WHEN a > 2 THEN b ELSE null END FROM foo From efa84e8309ced6614e62d1607899c47ed0b6b08a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 24 Jul 2026 12:22:52 -0400 Subject: [PATCH 013/109] test: add functional_dependencies.slt covering functional dependency driven optimizations (#23821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? This PR adds test coverage rather than closing an issue. It documents the current behavior of these bugs so that fixing them shows up as a test change: - https://github.com/apache/datafusion/issues/23634 - https://github.com/apache/datafusion/issues/23818 - https://github.com/apache/datafusion/issues/23819 - https://github.com/apache/datafusion/issues/23820 ## Rationale for this change While reviewing #23636 @neilconway and I kept coming up with more examples of bad plans, and it was not obvious which of the surrounding wrong answers were pre-existing and which the PR introduced. ## What changes are included in this PR? A new `datafusion/sqllogictest/test_files/functional_dependencies.slt` with one section per consumer of functional dependencies: 1. `ReplaceDistinctWithAggregate` — removing `DISTINCT` 2. `eliminate_duplicated_expr` — dropping trailing `ORDER BY` keys 3. `optimize_projections` — dropping `GROUP BY` expressions 4. `add_group_by_exprs_from_dependencies` — selecting non-grouped columns 5. `GROUP BY` derived keys on the NULL-padded side of an outer join Cases that currently return wrong answers are labelled `BUG` with the expected result and a link to the issue: | Case | Symptom | Issue | | --- | --- | --- | | 1.2 | `DISTINCT` over a nullable `UNIQUE` column returns both `NULL` rows | #23634 | | 2.2 | `ORDER BY x, y` drops the `y` key, so the `NULL` rows come back unordered | #23818 | | 3.2 | `GROUP BY x, y` drops `y`, merging the two `NULL` groups and losing a row | #23819 | | 4.2 | `SELECT x, y ... GROUP BY x` returns two rows for the `x = NULL` group | #23820 | ## Are these changes tested? CI ## Are there any user-facing changes? No. --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../test_files/functional_dependencies.slt | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 datafusion/sqllogictest/test_files/functional_dependencies.slt diff --git a/datafusion/sqllogictest/test_files/functional_dependencies.slt b/datafusion/sqllogictest/test_files/functional_dependencies.slt new file mode 100644 index 0000000000000..92aedf66e69e1 --- /dev/null +++ b/datafusion/sqllogictest/test_files/functional_dependencies.slt @@ -0,0 +1,314 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +########## +# Tests for functional dependencies +# (`datafusion/common/src/functional_dependencies.rs`) +# +# A functional dependency records that one set of columns (the *determinant*) +# determines the values of the others. DataFusion derives them from PRIMARY +# KEY / UNIQUE constraints and from GROUP BY keys, and four optimizer rules +# consume them to remove redundant work, each tested here in a different section. +# +# NULL handling is (as always) important: +# +# * A PRIMARY KEY is unique AND not nullable. +# * A `UNIQUE` constraint permits *multiple NULL rows*, because NULLs +# compare distinct. +# +# It is important not to mix `UNIQUE` columns with `DISTINCT` or `GROUP BY`, +# which treat NULLs as equal and can produce wrong answers. +########## + +# These rules all run during logical optimization, so show only logical plans. +statement ok +set datafusion.explain.logical_plan_only = true; + +# Set target_partitions explicitly so query results are stable. +statement ok +set datafusion.execution.target_partitions = 4; + +########## +## Test tables +########## + +statement ok +CREATE TABLE t_pk (x INT, y INT, PRIMARY KEY (x)) AS VALUES (1, 10), (2, 20); + +statement ok +CREATE TABLE t_uniq (x INT UNIQUE, y INT) AS VALUES (NULL, 2), (NULL, 1), (1, 3); + +query II rowsort +SELECT x, y FROM t_uniq; +---- +1 3 +NULL 1 +NULL 2 + + +# 1.1 PRIMARY KEY: rows are unique; the DISTINCT is removed and no +# Aggregate appears in the plan. +query TT +EXPLAIN SELECT DISTINCT x FROM t_pk; +---- +logical_plan TableScan: t_pk projection=[x] + +# 1.2 Nullable UNIQUE: the DISTINCT must be KEPT. UNIQUE allows several NULL +# rows, but DISTINCT treats NULLs as equal and has to collapse them into one. +# +# BUG: the DISTINCT is removed and both NULL rows are returned. +# Expected: `1`, `NULL`. +# Issue: https://github.com/apache/datafusion/issues/23634 +query I +SELECT DISTINCT x FROM t_uniq ORDER BY x NULLS LAST; +---- +1 +NULL +NULL + +query TT +EXPLAIN SELECT DISTINCT x FROM t_uniq; +---- +logical_plan TableScan: t_uniq projection=[x] + +# 1.3 A PRIMARY KEY downgraded to a non-unique dependency by a LEFT JOIN +# so the DISTINCT must be KEPT. +# Fixed by: https://github.com/apache/datafusion/pull/23548 +statement ok +CREATE TABLE t_orders (x INT, amount INT) AS VALUES (1, 10), (1, 20), (2, 30); + +query I +SELECT DISTINCT p.x FROM t_pk p LEFT JOIN t_orders o ON p.x = o.x ORDER BY p.x; +---- +1 +2 + +query TT +EXPLAIN SELECT DISTINCT p.x FROM t_pk p LEFT JOIN t_orders o ON p.x = o.x; +---- +logical_plan +01)Aggregate: groupBy=[[p.x]], aggr=[[]] +02)--Projection: p.x +03)----Left Join: p.x = o.x +04)------SubqueryAlias: p +05)--------TableScan: t_pk projection=[x] +06)------SubqueryAlias: o +07)--------TableScan: t_orders projection=[x] + +statement ok +drop table t_orders; + +# 1.4 DISTINCT over a GROUP BY output. Grouping collapses the multiple NULL +# rows, (NULL included) and the DISTINCT can be removed. +query I +SELECT DISTINCT x FROM (SELECT x FROM t_uniq GROUP BY x) ORDER BY x NULLS LAST; +---- +1 +NULL + +query TT +EXPLAIN SELECT DISTINCT x FROM (SELECT x FROM t_uniq GROUP BY x); +---- +logical_plan +01)Aggregate: groupBy=[[t_uniq.x]], aggr=[[]] +02)--TableScan: t_uniq projection=[x] + + +# 2.1 PRIMARY KEY: `x` determines `y`, so `ORDER BY x, y` is equivalent to +# `ORDER BY x` and the `y` key is dropped from the plan. +query TT +EXPLAIN SELECT x, y FROM t_pk ORDER BY x, y; +---- +logical_plan +01)Sort: t_pk.x ASC NULLS LAST +02)--TableScan: t_pk projection=[x, y] + +# 2.2 Nullable UNIQUE: `x` does NOT determine `y` across the two NULL rows, +# so the `y` sort key must be kept. +# +# BUG: +# Expected: `1 3`, `NULL 1`, `NULL 2`. +# Issue: https://github.com/apache/datafusion/issues/23818 +query II +SELECT x, y FROM t_uniq ORDER BY x NULLS LAST, y; +---- +1 3 +NULL 2 +NULL 1 + +query TT +EXPLAIN SELECT x, y FROM t_uniq ORDER BY x NULLS LAST, y; +---- +logical_plan +01)Sort: t_uniq.x ASC NULLS LAST +02)--TableScan: t_uniq projection=[x, y] + +# 2.3 After `GROUP BY x` the `x` does determine `cnt`, so can drop `cnt` from sort +query TT +EXPLAIN SELECT x, cnt FROM (SELECT x, count(*) AS cnt FROM t_uniq GROUP BY x) ORDER BY x, cnt; +---- +logical_plan +01)Sort: t_uniq.x ASC NULLS LAST +02)--Projection: t_uniq.x, count(Int64(1)) AS cnt +03)----Aggregate: groupBy=[[t_uniq.x]], aggr=[[count(Int64(1))]] +04)------TableScan: t_uniq projection=[x] + + +# 3.1 PRIMARY KEY: `x` determines `y`, and `y` is not selected, so grouping +# by `x, y` is the same as grouping by `x`. +query TT +EXPLAIN SELECT x FROM t_pk GROUP BY x, y; +---- +logical_plan +01)Aggregate: groupBy=[[t_pk.x]], aggr=[[]] +02)--TableScan: t_pk projection=[x] + +# 3.2 Nullable UNIQUE: grouping by `x, y` is NOT the same as grouping by +# `x` -- two NULL rows differ in `y` and belong in separate groups. +# +# BUG: `y` is dropped from the GROUP BY and the two NULL groups are merged, +# so one row goes missing. +# Expected: `1`, `NULL`, `NULL` (three rows). +# Issue: https://github.com/apache/datafusion/issues/23819 +query I rowsort +SELECT x FROM t_uniq GROUP BY x, y; +---- +1 +NULL + +query TT +EXPLAIN SELECT x FROM t_uniq GROUP BY x, y; +---- +logical_plan +01)Aggregate: groupBy=[[t_uniq.x]], aggr=[[]] +02)--TableScan: t_uniq projection=[x] + +# 3.3 The same grouping, but with `y` selected so the parent needs it: no +# column can be dropped and the answer is right. +query II rowsort +SELECT x, y FROM t_uniq GROUP BY x, y; +---- +1 3 +NULL 1 +NULL 2 + +# 4.1 PRIMARY KEY: `x` determines `y`, so `y` has a single well-defined +# value per group and one row is returned per `x`. +query II rowsort +SELECT x, y FROM t_pk GROUP BY x; +---- +1 10 +2 20 + +query TT +EXPLAIN SELECT x, y FROM t_pk GROUP BY x; +---- +logical_plan +01)Aggregate: groupBy=[[t_pk.x, t_pk.y]], aggr=[[]] +02)--TableScan: t_pk projection=[x, y] + +# 4.2 Nullable UNIQUE: `x` does NOT determine `y`, so there is no +# well-defined `y` for the `x = NULL` group. +# +# BUG: `y` is appended to the GROUP BY anyway, so `GROUP BY x` returns TWO +# rows for `x = NULL`. +# Expected: one row per distinct `x` (or a planning error -- postgres +# rejects this query, and accepts the 4.1 PRIMARY KEY form). +# Issue: https://github.com/apache/datafusion/issues/23820 +query II rowsort +SELECT x, y FROM t_uniq GROUP BY x; +---- +1 3 +NULL 1 +NULL 2 + +query TT +EXPLAIN SELECT x, y FROM t_uniq GROUP BY x; +---- +logical_plan +01)Aggregate: groupBy=[[t_uniq.x, t_uniq.y]], aggr=[[]] +02)--TableScan: t_uniq projection=[x, y] + + +statement ok +CREATE TABLE t_null (x INT) AS VALUES (NULL), (NULL); + +statement ok +CREATE TABLE t_probe (z INT) AS VALUES (0), (2); + +# 5.1 Grouping by `g.x, g.cnt` must keep both columns: `g.x` alone does not +# determine `g.cnt` after NULL padding. +query II +SELECT g.x, count(*) AS c + FROM t_probe a + LEFT JOIN (SELECT x, count(*) AS cnt FROM t_null GROUP BY x) g + ON a.z = g.cnt + GROUP BY g.x, g.cnt + ORDER BY c; +---- +NULL 1 +NULL 1 + +query TT +EXPLAIN SELECT g.x, count(*) AS c + FROM t_probe a + LEFT JOIN (SELECT x, count(*) AS cnt FROM t_null GROUP BY x) g + ON a.z = g.cnt + GROUP BY g.x, g.cnt; +---- +logical_plan +01)Projection: g.x, count(Int64(1)) AS count(*) AS c +02)--Aggregate: groupBy=[[g.x, g.cnt]], aggr=[[count(Int64(1))]] +03)----Projection: g.x, g.cnt +04)------Left Join: CAST(a.z AS Int64) = g.cnt +05)--------SubqueryAlias: a +06)----------TableScan: t_probe projection=[z] +07)--------SubqueryAlias: g +08)----------Projection: t_null.x, count(Int64(1)) AS count(*) AS cnt +09)------------Aggregate: groupBy=[[t_null.x]], aggr=[[count(Int64(1))]] +10)--------------TableScan: t_null projection=[x] + +# 5.2 The ORDER BY variant: `g.x` is NULL for both rows, so the `g.cnt` +# tie-breaker is what orders them. +query II +SELECT g.x, g.cnt + FROM t_probe a + LEFT JOIN (SELECT x, count(*) AS cnt FROM t_null GROUP BY x) g + ON a.z = g.cnt + ORDER BY g.x, g.cnt; +---- +NULL 2 +NULL NULL + +statement ok +drop table t_null; + +statement ok +drop table t_probe; + +########## +## Cleanup +########## + +statement ok +drop table t_pk; + +statement ok +drop table t_uniq; + +statement ok +RESET datafusion.explain.logical_plan_only; From 7269d13e63aa6b67d5db08655134d9c6f9c71ce8 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Fri, 24 Jul 2026 14:48:43 -0400 Subject: [PATCH 014/109] chore: Enable `unused_async` lint, make some functions sync (#23679) ## Which issue does this PR close? - Closes: N/A ## Rationale for this change There are a reasonable number of places where we have functions marked as `async` that don't need to be so. Fix this by enabling the `unused_async` lint, and fixing up the resulting breakage. There are a handful of spots that need an `expect(clippy::unused_async)` -- mostly mocks of `async` methods and example code. ## What changes are included in this PR? * Enable `unused_async` lint * Remove unnecessary `async` annotations in a bunch of places * Add `expect(clippy::unused_async)` where necessary ## Are these changes tested? Yes, covered by existing tests (no behavioral change expected). ## Are there any user-facing changes? Yes: this PR updates a few public APIs: - `datafusion_cli::command::OutputFormat::execute` - `datafusion::test_util::parquet::TestParquetFile::create_scan` - `datafusion_datasource_csv::file_format::CsvFormat::read_to_delimited_chunks_from_stream` - `datafusion_substrait::serializer::deserialize_bytes` Migration is mostly straightforward (e.g., removing `await` from calling code). --- Cargo.toml | 1 + benchmarks/src/imdb/run.rs | 4 +- datafusion-cli/src/command.rs | 2 +- datafusion-cli/src/exec.rs | 2 +- datafusion-cli/src/object_storage.rs | 12 ++-- .../custom_data_source/custom_datasource.rs | 4 +- .../examples/data_io/remote_catalog.rs | 3 + .../proto/composed_extension_codec.rs | 10 ++-- .../proto/expression_deduplication.rs | 2 +- datafusion-examples/examples/proto/main.rs | 4 +- .../examples/query_planning/expr_api.rs | 2 +- .../examples/query_planning/main.rs | 4 +- .../examples/query_planning/pruning.rs | 2 +- datafusion/catalog/src/information_schema.rs | 4 +- datafusion/core/benches/filter_query_sql.rs | 13 ++--- datafusion/core/benches/struct_query_sql.rs | 5 +- datafusion/core/benches/topk_aggregate.rs | 42 ++++---------- .../core/src/datasource/file_format/csv.rs | 3 +- datafusion/core/src/execution/context/mod.rs | 34 +++++------- datafusion/core/src/test_util/parquet.rs | 2 +- datafusion/core/tests/fuzz_cases/pruning.rs | 19 +++---- datafusion/core/tests/memory_limit/mod.rs | 10 ++-- .../core/tests/parquet/filter_pushdown.rs | 1 - datafusion/core/tests/parquet/mod.rs | 8 +-- .../core/tests/parquet/schema_coercion.rs | 6 +- .../physical_optimizer/enforce_sorting.rs | 14 ++--- .../physical_optimizer/join_selection.rs | 26 ++++----- .../core/tests/sql/aggregates/dict_nulls.rs | 8 +-- datafusion/core/tests/sql/aggregates/mod.rs | 45 +++++++-------- .../user_defined_async_scalar_functions.rs | 1 + datafusion/datasource-csv/src/file_format.rs | 3 +- datafusion/datasource-parquet/src/metadata.rs | 6 +- datafusion/datasource-parquet/src/sink.rs | 16 +++--- datafusion/execution/src/async_stream.rs | 4 ++ datafusion/physical-plan/src/limit.rs | 55 ++++++++----------- datafusion/physical-plan/src/sorts/sort.rs | 12 ++-- datafusion/sqllogictest/bin/sqllogictests.rs | 12 ++++ datafusion/sqllogictest/src/test_context.rs | 18 +++--- .../consumer/expr/scalar_function.rs | 40 +++++++------- datafusion/substrait/src/serializer.rs | 6 +- .../tests/cases/roundtrip_logical_plan.rs | 11 ++-- .../library-user-guide/upgrading/55.0.0.md | 29 ++++++++++ 42 files changed, 251 insertions(+), 254 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6f4c10f8e7552..87c23cc456651 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -222,6 +222,7 @@ needless_pass_by_value = "warn" # https://github.com/apache/datafusion/issues/18881 allow_attributes = "warn" assigning_clones = "warn" +unused_async = "warn" [workspace.lints.rust] unexpected_cfgs = { level = "warn", check-cfg = [ diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs index e0e302e466840..a8e202888794f 100644 --- a/benchmarks/src/imdb/run.rs +++ b/benchmarks/src/imdb/run.rs @@ -355,7 +355,7 @@ impl RunOpt { async fn register_tables(&self, ctx: &SessionContext) -> Result<()> { for table in IMDB_TABLES { - let table_provider = { self.get_table(ctx, table).await? }; + let table_provider = { self.get_table(ctx, table)? }; if self.mem_table { println!("Loading table '{table}' into memory"); @@ -416,7 +416,7 @@ impl RunOpt { Ok(result) } - async fn get_table( + fn get_table( &self, ctx: &SessionContext, table: &str, diff --git a/datafusion-cli/src/command.rs b/datafusion-cli/src/command.rs index 8aaa8025d1c3a..e847f7fdb501b 100644 --- a/datafusion-cli/src/command.rs +++ b/datafusion-cli/src/command.rs @@ -259,7 +259,7 @@ impl FromStr for OutputFormat { } impl OutputFormat { - pub async fn execute(&self, print_options: &mut PrintOptions) -> Result<()> { + pub fn execute(&self, print_options: &mut PrintOptions) -> Result<()> { match self { Self::ChangeFormat(format) => { if let Ok(format) = format.parse::() { diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index bc2c15f48debb..fc230d5362346 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -148,7 +148,7 @@ pub async fn exec_from_repl( Command::OutputFormat(subcommand) => { if let Some(subcommand) = subcommand { if let Ok(command) = subcommand.parse::() { - if let Err(e) = command.execute(print_options).await { + if let Err(e) = command.execute(print_options) { eprintln!("{e}") } } else { diff --git a/datafusion-cli/src/object_storage.rs b/datafusion-cli/src/object_storage.rs index 4293788e0c03a..e2ba992961c40 100644 --- a/datafusion-cli/src/object_storage.rs +++ b/datafusion-cli/src/object_storage.rs @@ -56,6 +56,10 @@ use object_store::aws::resolve_bucket_region; // Provide a local mock when running tests so we don't make network calls #[cfg(test)] +#[expect( + clippy::unused_async, + reason = "matches object_store::aws::resolve_bucket_region" +)] async fn resolve_bucket_region( _bucket: &str, _client_options: &ClientOptions, @@ -600,7 +604,7 @@ mod tests { #[tokio::test] async fn s3_object_store_builder_default() -> Result<()> { - if let Err(DataFusionError::Execution(e)) = check_aws_envs().await { + if let Err(DataFusionError::Execution(e)) = check_aws_envs() { // Skip test if AWS envs are not set eprintln!("{e}"); return Ok(()); @@ -765,7 +769,7 @@ mod tests { #[tokio::test] async fn s3_object_store_builder_resolves_region_when_none_provided() -> Result<()> { - if let Err(DataFusionError::Execution(e)) = check_aws_envs().await { + if let Err(DataFusionError::Execution(e)) = check_aws_envs() { // Skip test if AWS envs are not set eprintln!("{e}"); return Ok(()); @@ -798,7 +802,7 @@ mod tests { #[tokio::test] async fn s3_object_store_builder_overrides_region_when_resolve_region_enabled() -> Result<()> { - if let Err(DataFusionError::Execution(e)) = check_aws_envs().await { + if let Err(DataFusionError::Execution(e)) = check_aws_envs() { // Skip test if AWS envs are not set eprintln!("{e}"); return Ok(()); @@ -909,7 +913,7 @@ mod tests { table_options } - async fn check_aws_envs() -> Result<()> { + fn check_aws_envs() -> Result<()> { let aws_envs = [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs index a67738520b010..a2d7d7699927f 100644 --- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs +++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs @@ -145,7 +145,7 @@ impl Debug for CustomDataSource { } impl CustomDataSource { - pub(crate) async fn create_physical_plan( + pub(crate) fn create_physical_plan( &self, projections: Option<&Vec>, schema: SchemaRef, @@ -207,7 +207,7 @@ impl TableProvider for CustomDataSource { _filters: &[Expr], _limit: Option, ) -> Result> { - return self.create_physical_plan(projection, self.schema()).await; + self.create_physical_plan(projection, self.schema()) } } diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rs index 16814752b3ec2..a24ca2238181d 100644 --- a/datafusion-examples/examples/data_io/remote_catalog.rs +++ b/datafusion-examples/examples/data_io/remote_catalog.rs @@ -130,6 +130,7 @@ struct RemoteCatalogInterface {} impl RemoteCatalogInterface { /// Establish a connection to the remote catalog + #[expect(clippy::unused_async)] pub async fn connect() -> Result { // In a real implementation this method might connect to a remote // catalog, validate credentials, cache basic information, etc @@ -137,6 +138,7 @@ impl RemoteCatalogInterface { } /// Fetches information for a specific table + #[expect(clippy::unused_async)] pub async fn table_info(&self, name: &str) -> Result> { if name != "remote_table" { return Ok(None); @@ -155,6 +157,7 @@ impl RemoteCatalogInterface { } /// Fetches data for a table from a remote data source + #[expect(clippy::unused_async)] pub async fn read_data(&self, name: &str) -> Result { if name != "remote_table" { return plan_err!("Remote table not found: {}", name); diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rs index 2581f4a2ce247..6077a982c320d 100644 --- a/datafusion-examples/examples/proto/composed_extension_codec.rs +++ b/datafusion-examples/examples/proto/composed_extension_codec.rs @@ -47,8 +47,8 @@ use datafusion_proto::physical_plan::{ use datafusion_proto::protobuf; /// Example of using multiple extension codecs for serialization / deserialization -pub async fn composed_extension_codec() -> Result<()> { - // build execution plan that has both types of nodes +pub fn composed_extension_codec() -> Result<()> { + // Build execution plan that has both types of nodes // // Note each node requires a different `PhysicalExtensionCodec` to decode let exec_plan = Arc::new(ParentExec { @@ -63,18 +63,18 @@ pub async fn composed_extension_codec() -> Result<()> { Arc::new(ChildPhysicalExtensionCodec {}), ]); - // serialize execution plan to proto + // Serialize execution plan to proto let proto: protobuf::PhysicalPlanNode = protobuf::PhysicalPlanNode::try_from_physical_plan( exec_plan.clone(), &composed_codec, )?; - // deserialize proto back to execution plan + // Deserialize proto back to execution plan let result_exec_plan: Arc = proto.try_into_physical_plan(&ctx.task_ctx(), &composed_codec)?; - // assert that the original and deserialized execution plans are equal + // Assert that the original and deserialized execution plans are equal assert_eq!(format!("{exec_plan:?}"), format!("{result_exec_plan:?}")); Ok(()) diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rs index 31bb234e287f5..8ee59fa14d9cd 100644 --- a/datafusion-examples/examples/proto/expression_deduplication.rs +++ b/datafusion-examples/examples/proto/expression_deduplication.rs @@ -72,7 +72,7 @@ use prost::Message; /// In real scenarios, expressions can be much more complex, e.g. a large InList /// expression could be megabytes in size, so deduplication can save significant memory /// in addition to more correctly representing the original plan structure. -pub async fn expression_deduplication() -> Result<()> { +pub fn expression_deduplication() -> Result<()> { println!("=== Expression Deduplication Example ===\n"); // Create a schema for our test expressions diff --git a/datafusion-examples/examples/proto/main.rs b/datafusion-examples/examples/proto/main.rs index 3f525b5d46afa..d534eda24ba64 100644 --- a/datafusion-examples/examples/proto/main.rs +++ b/datafusion-examples/examples/proto/main.rs @@ -64,10 +64,10 @@ impl ExampleKind { } } ExampleKind::ComposedExtensionCodec => { - composed_extension_codec::composed_extension_codec().await? + composed_extension_codec::composed_extension_codec()? } ExampleKind::ExpressionDeduplication => { - expression_deduplication::expression_deduplication().await? + expression_deduplication::expression_deduplication()? } } Ok(()) diff --git a/datafusion-examples/examples/query_planning/expr_api.rs b/datafusion-examples/examples/query_planning/expr_api.rs index dd5145def3cfe..08efff7777691 100644 --- a/datafusion-examples/examples/query_planning/expr_api.rs +++ b/datafusion-examples/examples/query_planning/expr_api.rs @@ -58,7 +58,7 @@ use datafusion::prelude::*; /// 5. Analyze predicates for boundary ranges: [`range_analysis_demo`] /// 6. Get the types of the expressions: [`expression_type_demo`] /// 7. Apply type coercion to expressions: [`type_coercion_demo`] -pub async fn expr_api() -> Result<()> { +pub fn expr_api() -> Result<()> { // The easiest way to do create expressions is to use the // "fluent"-style API: let expr = col("a") + lit(5); diff --git a/datafusion-examples/examples/query_planning/main.rs b/datafusion-examples/examples/query_planning/main.rs index d3f99aedceb3d..2e4310082c9dd 100644 --- a/datafusion-examples/examples/query_planning/main.rs +++ b/datafusion-examples/examples/query_planning/main.rs @@ -94,12 +94,12 @@ impl ExampleKind { } } ExampleKind::AnalyzerRule => analyzer_rule::analyzer_rule().await?, - ExampleKind::ExprApi => expr_api::expr_api().await?, + ExampleKind::ExprApi => expr_api::expr_api()?, ExampleKind::OptimizerRule => optimizer_rule::optimizer_rule().await?, ExampleKind::ParseSqlExpr => parse_sql_expr::parse_sql_expr().await?, ExampleKind::PlanToSql => plan_to_sql::plan_to_sql_examples().await?, ExampleKind::PlannerApi => planner_api::planner_api().await?, - ExampleKind::Pruning => pruning::pruning().await?, + ExampleKind::Pruning => pruning::pruning()?, ExampleKind::ThreadPools => thread_pools::thread_pools().await?, } Ok(()) diff --git a/datafusion-examples/examples/query_planning/pruning.rs b/datafusion-examples/examples/query_planning/pruning.rs index df26aa57b6bc1..dad57cd261600 100644 --- a/datafusion-examples/examples/query_planning/pruning.rs +++ b/datafusion-examples/examples/query_planning/pruning.rs @@ -44,7 +44,7 @@ use datafusion::prelude::*; /// one might do as part of a higher level storage engine. See /// `parquet_index.rs` for an example that uses pruning in the context of an /// individual query. -pub async fn pruning() -> Result<()> { +pub fn pruning() -> Result<()> { // In this example, we'll use the PruningPredicate to determine if // the expression `x = 5 AND y = 10` can never be true based on statistics diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index ca5060896f787..d9ad7791af67c 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -151,7 +151,7 @@ impl InformationSchemaConfig { Ok(()) } - async fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { + fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { for catalog_name in self.catalog_list.catalog_names() { let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); @@ -1152,7 +1152,7 @@ impl PartitionStream for InformationSchemata { Arc::clone(&self.schema), // TODO: Stream this futures::stream::once(async move { - config.make_schemata(&mut builder).await; + config.make_schemata(&mut builder); builder.finish() }), )) diff --git a/datafusion/core/benches/filter_query_sql.rs b/datafusion/core/benches/filter_query_sql.rs index 3b80518d32dcd..6ddf6fa31820a 100644 --- a/datafusion/core/benches/filter_query_sql.rs +++ b/datafusion/core/benches/filter_query_sql.rs @@ -23,12 +23,11 @@ use arrow::{ use criterion::{Criterion, criterion_group, criterion_main}; use datafusion::prelude::SessionContext; use datafusion::{datasource::MemTable, error::Result}; -use futures::executor::block_on; use std::hint::black_box; use std::sync::Arc; use tokio::runtime::Runtime; -async fn query(ctx: &SessionContext, rt: &Runtime, sql: &str) { +fn query(ctx: &SessionContext, rt: &Runtime, sql: &str) { // execute the query let df = rt.block_on(ctx.sql(sql)).unwrap(); black_box(rt.block_on(df.collect()).unwrap()); @@ -71,28 +70,28 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("filter_array", |b| { let ctx = create_context(array_len, batch_size).unwrap(); - b.iter(|| block_on(query(&ctx, &rt, "select f32, f64 from t where f32 >= f64"))) + b.iter(|| query(&ctx, &rt, "select f32, f64 from t where f32 >= f64")) }); c.bench_function("filter_scalar", |b| { let ctx = create_context(array_len, batch_size).unwrap(); b.iter(|| { - block_on(query( + query( &ctx, &rt, "select f32, f64 from t where f32 >= 250 and f64 > 250", - )) + ) }) }); c.bench_function("filter_scalar in list", |b| { let ctx = create_context(array_len, batch_size).unwrap(); b.iter(|| { - block_on(query( + query( &ctx, &rt, "select f32, f64 from t where f32 in (10, 20, 30, 40)", - )) + ) }) }); } diff --git a/datafusion/core/benches/struct_query_sql.rs b/datafusion/core/benches/struct_query_sql.rs index 96434fc379ea6..848d5a3c3e5de 100644 --- a/datafusion/core/benches/struct_query_sql.rs +++ b/datafusion/core/benches/struct_query_sql.rs @@ -23,12 +23,11 @@ use arrow::{ use criterion::{Criterion, criterion_group, criterion_main}; use datafusion::prelude::SessionContext; use datafusion::{datasource::MemTable, error::Result}; -use futures::executor::block_on; use std::hint::black_box; use std::sync::Arc; use tokio::runtime::Runtime; -async fn query(ctx: &SessionContext, rt: &Runtime, sql: &str) { +fn query(ctx: &SessionContext, rt: &Runtime, sql: &str) { // execute the query let df = rt.block_on(ctx.sql(sql)).unwrap(); black_box(rt.block_on(df.collect()).unwrap()); @@ -71,7 +70,7 @@ fn criterion_benchmark(c: &mut Criterion) { let rt = Runtime::new().unwrap(); c.bench_function("struct", |b| { - b.iter(|| block_on(query(&ctx, &rt, "select struct(f32, f64) from t"))) + b.iter(|| query(&ctx, &rt, "select struct(f32, f64) from t")) }); } diff --git a/datafusion/core/benches/topk_aggregate.rs b/datafusion/core/benches/topk_aggregate.rs index c78b1ea494407..d8ca0d58b8d21 100644 --- a/datafusion/core/benches/topk_aggregate.rs +++ b/datafusion/core/benches/topk_aggregate.rs @@ -74,7 +74,7 @@ fn test_distinct_schema() -> SchemaRef { Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) } -async fn create_context( +fn create_context( partition_cnt: i32, sample_cnt: i32, asc: bool, @@ -94,7 +94,7 @@ async fn create_context( Ok(ctx) } -async fn create_context_distinct( +fn create_context_distinct( partition_cnt: i32, sample_cnt: i32, use_topk: bool, @@ -306,12 +306,8 @@ fn assert_utf8_utf8view_match( asc: bool, use_topk: bool, ) { - let ctx_utf8 = rt - .block_on(create_context(partitions, samples, asc, use_topk, false)) - .unwrap(); - let ctx_view = rt - .block_on(create_context(partitions, samples, asc, use_topk, true)) - .unwrap(); + let ctx_utf8 = create_context(partitions, samples, asc, use_topk, false).unwrap(); + let ctx_view = create_context(partitions, samples, asc, use_topk, true).unwrap(); let batches_utf8 = rt .block_on(aggregate_string(ctx_utf8, limit, use_topk)) .unwrap(); @@ -390,15 +386,9 @@ fn criterion_benchmark(c: &mut Criterion) { .name_tpl .replace("{rows}", &total_rows.to_string()) .replace("{limit}", &limit.to_string()); - let ctx = rt - .block_on(create_context( - partitions, - samples, - case.asc, - case.use_topk, - case.use_view, - )) - .unwrap(); + let ctx = + create_context(partitions, samples, case.asc, case.use_topk, case.use_view) + .unwrap(); c.bench_function(&name, |b| { b.iter(|| run(&rt, ctx.clone(), limit, case.use_topk, case.asc)) }); @@ -462,15 +452,9 @@ fn criterion_benchmark(c: &mut Criterion) { } else { format!("string aggregate {total_rows} {scenario} rows [{type_label}]") }; - let ctx = rt - .block_on(create_context( - partitions, - samples, - case.asc, - case.use_topk, - case.use_view, - )) - .unwrap(); + let ctx = + create_context(partitions, samples, case.asc, case.use_topk, case.use_view) + .unwrap(); c.bench_function(&name, |b| { b.iter(|| run_string(&rt, ctx.clone(), limit, case.use_topk)) }); @@ -478,11 +462,7 @@ fn criterion_benchmark(c: &mut Criterion) { // DISTINCT benchmarks for use_topk in [false, true] { - let ctx = rt.block_on(async { - create_context_distinct(partitions, samples, use_topk) - .await - .unwrap() - }); + let ctx = create_context_distinct(partitions, samples, use_topk).unwrap(); let topk_label = if use_topk { "TopK" } else { "no TopK" }; for asc in [false, true] { let dir = if asc { "asc" } else { "desc" }; diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 651a15d776e4d..90d7eb3b41388 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -591,8 +591,7 @@ mod tests { //convert compressed_stream to decoded_stream let decoded_stream = compressed_csv - .read_to_delimited_chunks_from_stream(compressed_stream.unwrap()) - .await; + .read_to_delimited_chunks_from_stream(compressed_stream.unwrap()); let (schema, records_read) = compressed_csv .infer_schema_from_stream(&session_state, records_to_read, decoded_stream) .await?; diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 281cb4dd79d4d..cd30193e307e3 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -687,8 +687,8 @@ impl SessionContext { pub async fn execute_logical_plan(&self, plan: LogicalPlan) -> Result { match plan { LogicalPlan::Ddl(ddl) => { - // Box::pin avoids allocating the stack space within this function's frame - // for every one of these individual async functions, decreasing the risk of + // Box async DDL handlers to avoid reserving space for all of their + // futures in this function's state machine, decreasing the risk of // stack overflows. match ddl { DdlStatement::CreateExternalTable(cmd) => { @@ -703,32 +703,26 @@ impl SessionContext { Box::pin(self.create_view(cmd)).await } DdlStatement::CreateCatalogSchema(cmd) => { - Box::pin(self.create_catalog_schema(cmd)).await - } - DdlStatement::CreateCatalog(cmd) => { - Box::pin(self.create_catalog(cmd)).await + self.create_catalog_schema(cmd) } + DdlStatement::CreateCatalog(cmd) => self.create_catalog(cmd), DdlStatement::DropTable(cmd) => Box::pin(self.drop_table(cmd)).await, DdlStatement::DropView(cmd) => Box::pin(self.drop_view(cmd)).await, - DdlStatement::DropCatalogSchema(cmd) => { - Box::pin(self.drop_schema(cmd)).await - } + DdlStatement::DropCatalogSchema(cmd) => self.drop_schema(cmd), DdlStatement::CreateFunction(cmd) => { Box::pin(self.create_function(*cmd)).await } - DdlStatement::DropFunction(cmd) => { - Box::pin(self.drop_function(cmd)).await - } + DdlStatement::DropFunction(cmd) => self.drop_function(&cmd), ddl => Ok(DataFrame::new(self.state(), LogicalPlan::Ddl(ddl))), } } // TODO what about the other statements (like TransactionStart and TransactionEnd) LogicalPlan::Statement(Statement::SetVariable(stmt)) => { - self.set_variable(stmt).await?; + self.set_variable(stmt)?; self.return_empty_dataframe() } LogicalPlan::Statement(Statement::ResetVariable(stmt)) => { - self.reset_variable(stmt).await?; + self.reset_variable(stmt)?; self.return_empty_dataframe() } LogicalPlan::Statement(Statement::Prepare(Prepare { @@ -987,7 +981,7 @@ impl SessionContext { Ok(()) } - async fn create_catalog_schema(&self, cmd: CreateCatalogSchema) -> Result { + fn create_catalog_schema(&self, cmd: CreateCatalogSchema) -> Result { let CreateCatalogSchema { schema_name, if_not_exists, @@ -1028,7 +1022,7 @@ impl SessionContext { } } - async fn create_catalog(&self, cmd: CreateCatalog) -> Result { + fn create_catalog(&self, cmd: CreateCatalog) -> Result { let CreateCatalog { catalog_name, if_not_exists, @@ -1078,7 +1072,7 @@ impl SessionContext { } } - async fn drop_schema(&self, cmd: DropCatalogSchema) -> Result { + fn drop_schema(&self, cmd: DropCatalogSchema) -> Result { let DropCatalogSchema { name, if_exists: allow_missing, @@ -1113,7 +1107,7 @@ impl SessionContext { exec_err!("Schema '{schema_ref}' doesn't exist.") } - async fn set_variable(&self, stmt: SetVariable) -> Result<()> { + fn set_variable(&self, stmt: SetVariable) -> Result<()> { let SetVariable { variable, value, .. } = stmt; @@ -1148,7 +1142,7 @@ impl SessionContext { Ok(()) } - async fn reset_variable(&self, stmt: ResetVariable) -> Result<()> { + fn reset_variable(&self, stmt: ResetVariable) -> Result<()> { let variable = stmt.variable; if variable.starts_with("datafusion.runtime.") { return self.reset_runtime_variable(&variable); @@ -1531,7 +1525,7 @@ impl SessionContext { self.return_empty_dataframe() } - async fn drop_function(&self, stmt: DropFunction) -> Result { + fn drop_function(&self, stmt: &DropFunction) -> Result { // we don't know function type at this point // decision has been made to drop all functions let mut dropped = false; diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rs index d1018f3fb0f04..e25fe746695cf 100644 --- a/datafusion/core/src/test_util/parquet.rs +++ b/datafusion/core/src/test_util/parquet.rs @@ -150,7 +150,7 @@ impl TestParquetFile { /// ``` /// /// Otherwise if `maybe_filter` is None, return just a `DataSourceExec` - pub async fn create_scan( + pub fn create_scan( &self, ctx: &SessionContext, maybe_filter: Option, diff --git a/datafusion/core/tests/fuzz_cases/pruning.rs b/datafusion/core/tests/fuzz_cases/pruning.rs index 8ce5207f91190..7624c97cf47f7 100644 --- a/datafusion/core/tests/fuzz_cases/pruning.rs +++ b/datafusion/core/tests/fuzz_cases/pruning.rs @@ -249,12 +249,7 @@ impl Utf8Test { for (idx, truncation_length) in [Some(1), Some(2), None].iter().enumerate() { // parquet files only support 32767 row groups per file, so chunk up into multiple files so we don't error if running on a large number of row groups for (rg_idx, row_groups) in row_groups.chunks(32766).enumerate() { - let buf = write_parquet_file( - *truncation_length, - Arc::clone(&schema), - row_groups.to_vec(), - ) - .await; + let buf = write_parquet_file(*truncation_length, &schema, row_groups); let filename = format!("test_fuzz_utf8_{idx}_{rg_idx}.parquet"); let size = buf.len(); let path = Path::from(filename); @@ -314,10 +309,10 @@ async fn execute_with_predicate( values } -async fn write_parquet_file( +fn write_parquet_file( truncation_length: Option, - schema: Arc, - row_groups: Vec>, + schema: &Arc, + row_groups: &[Vec], ) -> Bytes { let mut buf = BytesMut::new().writer(); let props = WriterProperties::builder() @@ -326,11 +321,11 @@ async fn write_parquet_file( let props = props.build(); { let mut writer = - ArrowWriter::try_new(&mut buf, schema.clone(), Some(props)).unwrap(); - for rg_values in row_groups.iter() { + ArrowWriter::try_new(&mut buf, Arc::clone(schema), Some(props)).unwrap(); + for rg_values in row_groups { let arr = StringArray::from_iter_values(rg_values.iter()); let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(arr)]).unwrap(); + RecordBatch::try_new(Arc::clone(schema), vec![Arc::new(arr)]).unwrap(); writer.write(&batch).unwrap(); writer.flush().unwrap(); // finishes the current row group and starts a new one } diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs index ebbe4312b1e1a..d6e38b5d01995 100644 --- a/datafusion/core/tests/memory_limit/mod.rs +++ b/datafusion/core/tests/memory_limit/mod.rs @@ -614,7 +614,7 @@ async fn test_sort_skewed_batches_spill() { // ------------------------------------------------------------------ // Create a new `SessionContext` with specified disk limit, memory pool limit, and spill compression codec -async fn setup_context( +fn setup_context( disk_limit: u64, memory_pool_limit: usize, spill_compression: SpillCompression, @@ -655,7 +655,7 @@ async fn setup_context( #[tokio::test] async fn test_disk_spill_limit_reached() -> Result<()> { let spill_compression = SpillCompression::Uncompressed; - let ctx = setup_context(1024 * 1024, 1024 * 1024, spill_compression).await?; // 1MB disk limit, 1MB memory limit + let ctx = setup_context(1024 * 1024, 1024 * 1024, spill_compression)?; // 1MB disk limit, 1MB memory limit let df = ctx .sql("select * from generate_series(1, 1000000000000) as t1(v1) order by v1 desc") @@ -683,7 +683,7 @@ async fn test_disk_spill_limit_reached() -> Result<()> { async fn test_disk_spill_limit_not_reached() -> Result<()> { let disk_spill_limit = 1024 * 1024; // 1MB let spill_compression = SpillCompression::Uncompressed; - let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression).await?; // 1MB disk limit, 128KB memory limit + let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression)?; // 1MB disk limit, 128KB memory limit let df = ctx .sql("select * from generate_series(1, 10000) as t1(v1) order by v1 desc") @@ -719,7 +719,7 @@ async fn test_disk_spill_limit_not_reached() -> Result<()> { async fn test_spill_file_compressed_with_zstd() -> Result<()> { let disk_spill_limit = 1024 * 1024; // 1MB let spill_compression = SpillCompression::Zstd; - let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression).await?; // 1MB disk limit, 128KB memory limit, zstd + let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression)?; // 1MB disk limit, 128KB memory limit, zstd let df = ctx .sql("select * from generate_series(1, 100000) as t1(v1) order by v1 desc") @@ -755,7 +755,7 @@ async fn test_spill_file_compressed_with_zstd() -> Result<()> { async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> { let disk_spill_limit = 1024 * 1024; // 1MB let spill_compression = SpillCompression::Lz4Frame; - let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression).await?; // 1MB disk limit, 128KB memory limit, lz4_frame + let ctx = setup_context(disk_spill_limit, 128 * 1024, spill_compression)?; // 1MB disk limit, 128KB memory limit, lz4_frame let df = ctx .sql("select * from generate_series(1, 100000) as t1(v1) order by v1 desc") diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rs index 5dfcd50c014c9..dabb2f35b24b1 100644 --- a/datafusion/core/tests/parquet/filter_pushdown.rs +++ b/datafusion/core/tests/parquet/filter_pushdown.rs @@ -515,7 +515,6 @@ impl<'a> TestCase<'a> { let exec = self .test_parquet_file .create_scan(&ctx, Some(filter.clone())) - .await .unwrap(); let result = collect(exec.clone(), ctx.task_ctx()).await.unwrap(); diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index 1cc4bb32d9eba..7066a4147c017 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -330,11 +330,10 @@ impl ContextWithParquet { custom_schema, custom_batches, ) - .await } Unit::Page(row_per_page) => { config = config.with_parquet_page_index_pruning(true); - make_test_file_page(scenario, row_per_page).await + make_test_file_page(scenario, row_per_page) } Unit::RowGroupAndPage(row_per_group, row_per_page) => { config = config.with_parquet_bloom_filter_pruning(true); @@ -347,7 +346,6 @@ impl ContextWithParquet { custom_schema, custom_batches, ) - .await } }; let parquet_path = file.path().to_string_lossy(); @@ -1173,7 +1171,7 @@ fn create_data_batch(scenario: Scenario) -> Vec { } /// Create a test parquet file with various data types -async fn make_test_file_rg( +fn make_test_file_rg( scenario: Scenario, row_per_group: usize, row_per_page: Option, @@ -1219,7 +1217,7 @@ async fn make_test_file_rg( output_file } -async fn make_test_file_page(scenario: Scenario, row_per_page: usize) -> NamedTempFile { +fn make_test_file_page(scenario: Scenario, row_per_page: usize) -> NamedTempFile { let mut output_file = tempfile::Builder::new() .prefix("parquet_page_pruning") .suffix(".parquet") diff --git a/datafusion/core/tests/parquet/schema_coercion.rs b/datafusion/core/tests/parquet/schema_coercion.rs index 6f7e2e328d0c3..be45ab38dabad 100644 --- a/datafusion/core/tests/parquet/schema_coercion.rs +++ b/datafusion/core/tests/parquet/schema_coercion.rs @@ -53,7 +53,7 @@ async fn multi_parquet_coercion() { // batch2: c2(int64), c3(float32) let batch2 = RecordBatch::try_from_iter(vec![("c2", c2), ("c3", c3)]).unwrap(); - let (meta, _files) = store_parquet(vec![batch1, batch2]).await.unwrap(); + let (meta, _files) = store_parquet(vec![batch1, batch2]).unwrap(); let file_group = meta.into_iter().map(Into::into).collect(); // cast c1 to utf8, c2 to int32, c3 to float64 @@ -107,7 +107,7 @@ async fn multi_parquet_coercion_projection() { let batch2 = RecordBatch::try_from_iter(vec![("c2", c2), ("c1", c1s), ("c3", c3)]).unwrap(); - let (meta, _files) = store_parquet(vec![batch1, batch2]).await.unwrap(); + let (meta, _files) = store_parquet(vec![batch1, batch2]).unwrap(); let file_group = meta.into_iter().map(Into::into).collect(); // cast c1 to utf8, c2 to int32, c3 to float64 @@ -146,7 +146,7 @@ async fn multi_parquet_coercion_projection() { } /// Writes `batches` to a temporary parquet file -pub async fn store_parquet( +pub fn store_parquet( batches: Vec, ) -> Result<(Vec, Vec)> { // Each batch writes to their own file diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs index e9ad978b2e0cb..9338fcc0bce35 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs @@ -425,12 +425,12 @@ async fn test_union_inputs_different_sorted2() -> Result<()> { Ok(()) } -#[tokio::test] +#[test] // Test with `repartition_sorts` enabled to preserve pre-sorted partitions and avoid resorting -async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true() +fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true() -> Result<()> { assert_snapshot!( - union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(true).await?, + union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(true)?, @r" Input Plan: OutputRequirementExec: order_by=[(nullable_col@0, asc)], dist_by=SinglePartition @@ -451,12 +451,12 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti Ok(()) } -#[tokio::test] +#[test] // Test with `repartition_sorts` disabled, causing a full resort of the data -async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false() +fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_false() -> Result<()> { assert_snapshot!( - union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(false).await?, + union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(false)?, @r" Input Plan: OutputRequirementExec: order_by=[(nullable_col@0, asc)], dist_by=SinglePartition @@ -477,7 +477,7 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti Ok(()) } -async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl( +fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl( repartition_sorts: bool, ) -> Result { let schema = create_test_schema()?; diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs index cca54909a1375..3827e6e98b5e6 100644 --- a/datafusion/core/tests/physical_optimizer/join_selection.rs +++ b/datafusion/core/tests/physical_optimizer/join_selection.rs @@ -1271,8 +1271,8 @@ struct TestCase { expecting_swap: bool, } -#[tokio::test] -async fn test_join_with_swap_full() -> Result<()> { +#[test] +fn test_join_with_swap_full() -> Result<()> { // NOTE: Currently, some initial conditions are not viable after join order selection. // For example, full join always comes in partitioned mode. See the warning in // function "swap". If this changes in the future, we should update these tests. @@ -1319,13 +1319,13 @@ async fn test_join_with_swap_full() -> Result<()> { }, ]; for case in cases.into_iter() { - test_join_with_maybe_swap_unbounded_case(case).await? + test_join_with_maybe_swap_unbounded_case(case)? } Ok(()) } -#[tokio::test] -async fn test_cases_without_collect_left_check() -> Result<()> { +#[test] +fn test_cases_without_collect_left_check() -> Result<()> { let mut cases = vec![]; let join_types = vec![JoinType::LeftSemi, JoinType::Inner]; for join_type in join_types { @@ -1412,13 +1412,13 @@ async fn test_cases_without_collect_left_check() -> Result<()> { } for case in cases.into_iter() { - test_join_with_maybe_swap_unbounded_case(case).await? + test_join_with_maybe_swap_unbounded_case(case)? } Ok(()) } -#[tokio::test] -async fn test_not_support_collect_left() -> Result<()> { +#[test] +fn test_not_support_collect_left() -> Result<()> { let mut cases = vec![]; // After [JoinSelection] optimization, these join types cannot run in CollectLeft mode except // [JoinType::LeftSemi] @@ -1467,13 +1467,13 @@ async fn test_not_support_collect_left() -> Result<()> { } for case in cases.into_iter() { - test_join_with_maybe_swap_unbounded_case(case).await? + test_join_with_maybe_swap_unbounded_case(case)? } Ok(()) } -#[tokio::test] -async fn test_not_supporting_swaps_possible_collect_left() -> Result<()> { +#[test] +fn test_not_supporting_swaps_possible_collect_left() -> Result<()> { let mut cases = vec![]; let the_ones_not_support_collect_left = vec![JoinType::Right, JoinType::RightAnti, JoinType::RightSemi]; @@ -1567,12 +1567,12 @@ async fn test_not_supporting_swaps_possible_collect_left() -> Result<()> { } for case in cases.into_iter() { - test_join_with_maybe_swap_unbounded_case(case).await? + test_join_with_maybe_swap_unbounded_case(case)? } Ok(()) } -async fn test_join_with_maybe_swap_unbounded_case(t: TestCase) -> Result<()> { +fn test_join_with_maybe_swap_unbounded_case(t: TestCase) -> Result<()> { let left_unbounded = t.initial_sources_unbounded.0 == SourceType::Unbounded; let right_unbounded = t.initial_sources_unbounded.1 == SourceType::Unbounded; let left_exec = Arc::new(UnboundedExec::new( diff --git a/datafusion/core/tests/sql/aggregates/dict_nulls.rs b/datafusion/core/tests/sql/aggregates/dict_nulls.rs index 8733b9e87b57a..c6c3f02829c43 100644 --- a/datafusion/core/tests/sql/aggregates/dict_nulls.rs +++ b/datafusion/core/tests/sql/aggregates/dict_nulls.rs @@ -292,7 +292,7 @@ async fn test_first_last_value_group_by_dict_nulls() -> Result<()> { /// Test MAX with dictionary columns containing null keys and values as specified in the SQL query #[tokio::test] async fn test_max_with_fuzz_table_dict_nulls() -> Result<()> { - let (ctx_single, ctx_multi) = setup_fuzz_test_contexts().await?; + let (ctx_single, ctx_multi) = setup_fuzz_test_contexts()?; // Execute the SQL query with MAX aggregations let sql = "SELECT @@ -333,7 +333,7 @@ async fn test_max_with_fuzz_table_dict_nulls() -> Result<()> { /// Test MIN with fuzz table containing dictionary columns with null keys and values and timestamp data (single and multiple partitions) #[tokio::test] async fn test_min_timestamp_with_fuzz_table_dict_nulls() -> Result<()> { - let (ctx_single, ctx_multi) = setup_fuzz_timestamp_test_contexts().await?; + let (ctx_single, ctx_multi) = setup_fuzz_timestamp_test_contexts()?; // Execute the SQL query with MIN aggregation on timestamp let sql = "SELECT @@ -373,7 +373,7 @@ async fn test_min_timestamp_with_fuzz_table_dict_nulls() -> Result<()> { /// Test COUNT and COUNT DISTINCT with fuzz table containing dictionary columns with null keys and values (single and multiple partitions) #[tokio::test] async fn test_count_distinct_with_fuzz_table_dict_nulls() -> Result<()> { - let (ctx_single, ctx_multi) = setup_fuzz_count_test_contexts().await?; + let (ctx_single, ctx_multi) = setup_fuzz_count_test_contexts()?; // Execute the SQL query with COUNT and COUNT DISTINCT aggregations let sql = "SELECT @@ -414,7 +414,7 @@ async fn test_count_distinct_with_fuzz_table_dict_nulls() -> Result<()> { /// Test MEDIAN and MEDIAN DISTINCT with fuzz table containing various numeric types and dictionary columns with null keys and values (single and multiple partitions) #[tokio::test] async fn test_median_distinct_with_fuzz_table_dict_nulls() -> Result<()> { - let (ctx_single, ctx_multi) = setup_fuzz_median_test_contexts().await?; + let (ctx_single, ctx_multi) = setup_fuzz_median_test_contexts()?; // Execute the SQL query with MEDIAN and MEDIAN DISTINCT aggregations let sql = "SELECT diff --git a/datafusion/core/tests/sql/aggregates/mod.rs b/datafusion/core/tests/sql/aggregates/mod.rs index ede40d5c4ceca..b209e91cc81e7 100644 --- a/datafusion/core/tests/sql/aggregates/mod.rs +++ b/datafusion/core/tests/sql/aggregates/mod.rs @@ -259,20 +259,20 @@ impl TestData { } /// Sets up test contexts for TestData with both single and multiple partitions -pub async fn setup_test_contexts( +pub fn setup_test_contexts( test_data: &TestData, ) -> Result<(SessionContext, SessionContext)> { // Single partition context - let ctx_single = create_context_with_partitions(test_data, 1).await?; + let ctx_single = create_context_with_partitions(test_data, 1)?; // Multiple partition context - let ctx_multi = create_context_with_partitions(test_data, 3).await?; + let ctx_multi = create_context_with_partitions(test_data, 3)?; Ok((ctx_single, ctx_multi)) } /// Creates a session context with the specified number of partitions and registers test data -pub async fn create_context_with_partitions( +pub fn create_context_with_partitions( test_data: &TestData, num_partitions: usize, ) -> Result { @@ -348,7 +348,7 @@ pub async fn run_snapshot_test( test_data: &TestData, sql: &str, ) -> Result> { - let (ctx_single, ctx_multi) = setup_test_contexts(test_data).await?; + let (ctx_single, ctx_multi) = setup_test_contexts(test_data)?; let results = test_query_consistency(&ctx_single, &ctx_multi, sql).await?; Ok(results) } @@ -430,20 +430,20 @@ impl FuzzTestData { } /// Sets up test contexts for fuzz table with both single and multiple partitions -pub async fn setup_fuzz_test_contexts() -> Result<(SessionContext, SessionContext)> { +pub fn setup_fuzz_test_contexts() -> Result<(SessionContext, SessionContext)> { let test_data = FuzzTestData::new(); // Single partition context - let ctx_single = create_fuzz_context_with_partitions(&test_data, 1).await?; + let ctx_single = create_fuzz_context_with_partitions(&test_data, 1)?; // Multiple partition context - let ctx_multi = create_fuzz_context_with_partitions(&test_data, 3).await?; + let ctx_multi = create_fuzz_context_with_partitions(&test_data, 3)?; Ok((ctx_single, ctx_multi)) } /// Creates a session context with fuzz table partitioned into specified number of partitions -pub async fn create_fuzz_context_with_partitions( +pub fn create_fuzz_context_with_partitions( test_data: &FuzzTestData, num_partitions: usize, ) -> Result { @@ -604,21 +604,20 @@ impl FuzzCountTestData { } /// Sets up test contexts for fuzz table with duration/binary columns and both single and multiple partitions -pub async fn setup_fuzz_count_test_contexts() -> Result<(SessionContext, SessionContext)> -{ +pub fn setup_fuzz_count_test_contexts() -> Result<(SessionContext, SessionContext)> { let test_data = FuzzCountTestData::new(); // Single partition context - let ctx_single = create_fuzz_count_context_with_partitions(&test_data, 1).await?; + let ctx_single = create_fuzz_count_context_with_partitions(&test_data, 1)?; // Multiple partition context - let ctx_multi = create_fuzz_count_context_with_partitions(&test_data, 3).await?; + let ctx_multi = create_fuzz_count_context_with_partitions(&test_data, 3)?; Ok((ctx_single, ctx_multi)) } /// Creates a session context with fuzz count table partitioned into specified number of partitions -pub async fn create_fuzz_count_context_with_partitions( +pub fn create_fuzz_count_context_with_partitions( test_data: &FuzzCountTestData, num_partitions: usize, ) -> Result { @@ -808,21 +807,20 @@ impl FuzzMedianTestData { } /// Sets up test contexts for fuzz table with numeric types for median testing and both single and multiple partitions -pub async fn setup_fuzz_median_test_contexts() -> Result<(SessionContext, SessionContext)> -{ +pub fn setup_fuzz_median_test_contexts() -> Result<(SessionContext, SessionContext)> { let test_data = FuzzMedianTestData::new(); // Single partition context - let ctx_single = create_fuzz_median_context_with_partitions(&test_data, 1).await?; + let ctx_single = create_fuzz_median_context_with_partitions(&test_data, 1)?; // Multiple partition context - let ctx_multi = create_fuzz_median_context_with_partitions(&test_data, 3).await?; + let ctx_multi = create_fuzz_median_context_with_partitions(&test_data, 3)?; Ok((ctx_single, ctx_multi)) } /// Creates a session context with fuzz median table partitioned into specified number of partitions -pub async fn create_fuzz_median_context_with_partitions( +pub fn create_fuzz_median_context_with_partitions( test_data: &FuzzMedianTestData, num_partitions: usize, ) -> Result { @@ -959,21 +957,20 @@ impl FuzzTimestampTestData { } /// Sets up test contexts for fuzz table with timestamps and both single and multiple partitions -pub async fn setup_fuzz_timestamp_test_contexts() --> Result<(SessionContext, SessionContext)> { +pub fn setup_fuzz_timestamp_test_contexts() -> Result<(SessionContext, SessionContext)> { let test_data = FuzzTimestampTestData::new(); // Single partition context - let ctx_single = create_fuzz_timestamp_context_with_partitions(&test_data, 1).await?; + let ctx_single = create_fuzz_timestamp_context_with_partitions(&test_data, 1)?; // Multiple partition context - let ctx_multi = create_fuzz_timestamp_context_with_partitions(&test_data, 3).await?; + let ctx_multi = create_fuzz_timestamp_context_with_partitions(&test_data, 3)?; Ok((ctx_single, ctx_multi)) } /// Creates a session context with fuzz timestamp table partitioned into specified number of partitions -pub async fn create_fuzz_timestamp_context_with_partitions( +pub fn create_fuzz_timestamp_context_with_partitions( test_data: &FuzzTimestampTestData, num_partitions: usize, ) -> Result { diff --git a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs index dd91267d583fe..5b552e5369ef7 100644 --- a/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_async_scalar_functions.rs @@ -267,6 +267,7 @@ impl AsyncScalarUDFImpl for TestAsyncUDFImpl { } /// Simulates calling an async external service +#[expect(clippy::unused_async)] async fn call_external_service(arg1: ColumnarValue) -> Result { Ok(arg1) } diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index 89c3d374e68fc..a7f01f6ffec13 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -158,7 +158,6 @@ impl CsvFormat { .map_err(|e| DataFusionError::ObjectStore(Box::new(e))) .boxed(), ) - .await .map_err(DataFusionError::from) .left_stream(), Err(e) => { @@ -170,7 +169,7 @@ impl CsvFormat { /// Convert a stream of bytes into a stream of [`Bytes`] containing newline /// delimited CSV records, while accounting for `\` and `"`. - pub async fn read_to_delimited_chunks_from_stream<'a>( + pub fn read_to_delimited_chunks_from_stream<'a>( &self, stream: BoxStream<'a, Result>, ) -> BoxStream<'a, Result> { diff --git a/datafusion/datasource-parquet/src/metadata.rs b/datafusion/datasource-parquet/src/metadata.rs index ad1caa59b8d32..56abf52144028 100644 --- a/datafusion/datasource-parquet/src/metadata.rs +++ b/datafusion/datasource-parquet/src/metadata.rs @@ -181,14 +181,14 @@ impl<'a> DFParquetMetadata<'a> { Self::load_page_index(self.store, self.object_meta, cached_metadata) .await?; if cache_metadata { - self.cache_metadata(Arc::clone(&metadata)).await?; + self.cache_metadata(Arc::clone(&metadata))?; } return Ok(metadata); } let metadata = self.fetch_metadata_from_store(page_index_policy).await?; if cache_metadata { - self.cache_metadata(Arc::clone(&metadata)).await?; + self.cache_metadata(Arc::clone(&metadata))?; } Ok(metadata) } @@ -207,7 +207,7 @@ impl<'a> DFParquetMetadata<'a> { metadata.column_index().is_some() && metadata.offset_index().is_some() } - async fn cache_metadata(&self, metadata: Arc) -> Result<()> { + fn cache_metadata(&self, metadata: Arc) -> Result<()> { if let Some(file_metadata_cache) = &self.file_metadata_cache { file_metadata_cache.put( &self.object_meta.location, diff --git a/datafusion/datasource-parquet/src/sink.rs b/datafusion/datasource-parquet/src/sink.rs index f15f67aab0a87..df2f17c6be22d 100644 --- a/datafusion/datasource-parquet/src/sink.rs +++ b/datafusion/datasource-parquet/src/sink.rs @@ -171,7 +171,7 @@ impl ParquetSink { /// Creates an AsyncArrowWriter which serializes a parquet file to an ObjectStore /// AsyncArrowWriters are used when individual parquet file serialization is not parallelized - async fn create_async_arrow_writer( + fn create_async_arrow_writer( &self, location: &Path, object_store: Arc, @@ -296,14 +296,12 @@ impl FileSink for ParquetSink { if !parquet_opts.global.allow_single_file_parallelism || parquet_opts.global.content_defined_chunking.enabled { - let mut writer = self - .create_async_arrow_writer( - &path, - Arc::clone(&object_store), - context, - parquet_props.clone(), - ) - .await?; + let mut writer = self.create_async_arrow_writer( + &path, + Arc::clone(&object_store), + context, + parquet_props.clone(), + )?; let reservation = MemoryConsumer::new(format!("ParquetSink[{path}]")) .register(context.memory_pool()); file_write_tasks.spawn( diff --git a/datafusion/execution/src/async_stream.rs b/datafusion/execution/src/async_stream.rs index a84984d192d1f..e271145e03de6 100644 --- a/datafusion/execution/src/async_stream.rs +++ b/datafusion/execution/src/async_stream.rs @@ -388,6 +388,7 @@ mod test { async fn unit_emit_in_select() { use tokio::select; + #[expect(clippy::unused_async)] async fn do_stuff_async() {} let s = async_stream(|mut emitter| async move { @@ -405,7 +406,9 @@ mod test { async fn emit_with_select() { use tokio::select; + #[expect(clippy::unused_async)] async fn do_stuff_async() {} + #[expect(clippy::unused_async)] async fn more_async_work() {} let s = async_stream(|mut emitter| async move { @@ -556,6 +559,7 @@ mod test { fn inner_try_stream() { use tokio::select; + #[expect(clippy::unused_async)] async fn do_stuff_async() {} let _ = async_stream(|mut emitter| async move { diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index a1f6074cb9ae6..ddce680fc18ad 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -839,80 +839,73 @@ mod tests { Ok(()) } - #[tokio::test] - async fn test_row_number_statistics_for_global_limit() -> Result<()> { - let row_count = row_number_statistics_for_global_limit(0, Some(10)).await?; + #[test] + fn test_row_number_statistics_for_global_limit() -> Result<()> { + let row_count = row_number_statistics_for_global_limit(0, Some(10))?; assert_eq!(row_count, Precision::Exact(10)); - let row_count = row_number_statistics_for_global_limit(5, Some(10)).await?; + let row_count = row_number_statistics_for_global_limit(5, Some(10))?; assert_eq!(row_count, Precision::Exact(10)); - let row_count = row_number_statistics_for_global_limit(400, Some(10)).await?; + let row_count = row_number_statistics_for_global_limit(400, Some(10))?; assert_eq!(row_count, Precision::Exact(0)); - let row_count = row_number_statistics_for_global_limit(398, Some(10)).await?; + let row_count = row_number_statistics_for_global_limit(398, Some(10))?; assert_eq!(row_count, Precision::Exact(2)); - let row_count = row_number_statistics_for_global_limit(398, Some(1)).await?; + let row_count = row_number_statistics_for_global_limit(398, Some(1))?; assert_eq!(row_count, Precision::Exact(1)); - let row_count = row_number_statistics_for_global_limit(398, None).await?; + let row_count = row_number_statistics_for_global_limit(398, None)?; assert_eq!(row_count, Precision::Exact(2)); - let row_count = - row_number_statistics_for_global_limit(0, Some(usize::MAX)).await?; + let row_count = row_number_statistics_for_global_limit(0, Some(usize::MAX))?; assert_eq!(row_count, Precision::Exact(400)); - let row_count = - row_number_statistics_for_global_limit(398, Some(usize::MAX)).await?; + let row_count = row_number_statistics_for_global_limit(398, Some(usize::MAX))?; assert_eq!(row_count, Precision::Exact(2)); - let row_count = - row_number_inexact_statistics_for_global_limit(0, Some(10)).await?; + let row_count = row_number_inexact_statistics_for_global_limit(0, Some(10))?; assert_eq!(row_count, Precision::Inexact(10)); - let row_count = - row_number_inexact_statistics_for_global_limit(5, Some(10)).await?; + let row_count = row_number_inexact_statistics_for_global_limit(5, Some(10))?; assert_eq!(row_count, Precision::Inexact(10)); // Input was Inexact, so an `nr <= skip` outcome must remain Inexact: // the inexact estimate could be wrong, so we cannot promote 0 to // Exact. - let row_count = - row_number_inexact_statistics_for_global_limit(400, Some(10)).await?; + let row_count = row_number_inexact_statistics_for_global_limit(400, Some(10))?; assert_eq!(row_count, Precision::Inexact(0)); - let row_count = - row_number_inexact_statistics_for_global_limit(398, Some(10)).await?; + let row_count = row_number_inexact_statistics_for_global_limit(398, Some(10))?; assert_eq!(row_count, Precision::Inexact(2)); - let row_count = - row_number_inexact_statistics_for_global_limit(398, Some(1)).await?; + let row_count = row_number_inexact_statistics_for_global_limit(398, Some(1))?; assert_eq!(row_count, Precision::Inexact(1)); - let row_count = row_number_inexact_statistics_for_global_limit(398, None).await?; + let row_count = row_number_inexact_statistics_for_global_limit(398, None)?; assert_eq!(row_count, Precision::Inexact(2)); let row_count = - row_number_inexact_statistics_for_global_limit(0, Some(usize::MAX)).await?; + row_number_inexact_statistics_for_global_limit(0, Some(usize::MAX))?; assert_eq!(row_count, Precision::Inexact(400)); let row_count = - row_number_inexact_statistics_for_global_limit(398, Some(usize::MAX)).await?; + row_number_inexact_statistics_for_global_limit(398, Some(usize::MAX))?; assert_eq!(row_count, Precision::Inexact(2)); Ok(()) } - #[tokio::test] - async fn test_row_number_statistics_for_local_limit() -> Result<()> { - let row_count = row_number_statistics_for_local_limit(4, 10).await?; + #[test] + fn test_row_number_statistics_for_local_limit() -> Result<()> { + let row_count = row_number_statistics_for_local_limit(4, 10)?; assert_eq!(row_count, Precision::Exact(10)); Ok(()) } - async fn row_number_statistics_for_global_limit( + fn row_number_statistics_for_global_limit( skip: usize, fetch: Option, ) -> Result> { @@ -940,7 +933,7 @@ mod tests { PhysicalGroupBy::new_single(group_by_expr.clone()) } - async fn row_number_inexact_statistics_for_global_limit( + fn row_number_inexact_statistics_for_global_limit( skip: usize, fetch: Option, ) -> Result> { @@ -971,7 +964,7 @@ mod tests { .num_rows) } - async fn row_number_statistics_for_local_limit( + fn row_number_statistics_for_local_limit( num_partitions: usize, fetch: usize, ) -> Result> { diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index a9b754dee68bd..4b30aede7d02a 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -406,7 +406,7 @@ impl ExternalSorter { /// Appending globally sorted batches to the in-progress spill file, and clears /// the `globally_sorted_batches` (also its memory reservation) afterwards. - async fn consume_and_spill_append( + fn consume_and_spill_append( &mut self, globally_sorted_batches: &mut Vec, ) -> Result<()> { @@ -445,7 +445,7 @@ impl ExternalSorter { } /// Finishes the in-progress spill file and moves it to the finished spill files. - async fn spill_finish(&mut self) -> Result<()> { + fn spill_finish(&mut self) -> Result<()> { let (mut in_progress_file, max_record_batch_memory) = self.in_progress_spill_file.take().ok_or_else(|| { internal_datafusion_err!("Should be called after `spill_append`") @@ -500,8 +500,7 @@ impl ExternalSorter { // already in memory, so it's okay to combine it with previously // sorted batches, and spill together. globally_sorted_batches.push(batch); - self.consume_and_spill_append(&mut globally_sorted_batches) - .await?; // reservation is freed in spill() + self.consume_and_spill_append(&mut globally_sorted_batches)?; // reservation is freed in spill() } else { globally_sorted_batches.push(batch); } @@ -511,9 +510,8 @@ impl ExternalSorter { // upcoming `self.reserve_memory_for_merge()` may fail due to insufficient memory. drop(sorted_stream); - self.consume_and_spill_append(&mut globally_sorted_batches) - .await?; - self.spill_finish().await?; + self.consume_and_spill_append(&mut globally_sorted_batches)?; + self.spill_finish()?; // Sanity check after spilling let buffers_cleared_property = diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index cd51dc47ef5fc..da0beb0c29a28 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -473,6 +473,10 @@ async fn run_test_file_substrait_round_trip( } #[cfg(not(feature = "substrait"))] +#[expect( + clippy::unused_async, + reason = "matches the substrait-enabled implementation" +)] async fn run_test_file_substrait_round_trip( _test_file: TestFile, _validator: Validator, @@ -646,6 +650,10 @@ async fn run_test_file_with_postgres( } #[cfg(not(feature = "postgres"))] +#[expect( + clippy::unused_async, + reason = "matches the postgres-enabled implementation" +)] async fn run_test_file_with_postgres( _test_file: TestFile, _validator: Validator, @@ -771,6 +779,10 @@ async fn run_complete_file_with_postgres( } #[cfg(not(feature = "postgres"))] +#[expect( + clippy::unused_async, + reason = "matches the postgres-enabled implementation" +)] async fn run_complete_file_with_postgres( _test_file: TestFile, _validator: Validator, diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index fdb04edc05101..99c3179ef1056 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -142,15 +142,15 @@ impl TestContext { } "information_schema_table_types.slt" => { info!("Registering local temporary table"); - register_temp_table(test_ctx.session_ctx()).await; + register_temp_table(test_ctx.session_ctx()); } "information_schema_columns.slt" => { info!("Registering table with many types"); - register_table_with_many_types(test_ctx.session_ctx()).await; + register_table_with_many_types(test_ctx.session_ctx()); } "map.slt" => { info!("Registering table with map"); - register_table_with_map(test_ctx.session_ctx()).await; + register_table_with_map(test_ctx.session_ctx()); } "avro.slt" => { #[cfg(feature = "avro")] @@ -173,7 +173,7 @@ impl TestContext { test_ctx.ctx.register_udf(example_udf); register_partition_table(&mut test_ctx).await; info!("Registering table with many types"); - register_table_with_many_types(test_ctx.session_ctx()).await; + register_table_with_many_types(test_ctx.session_ctx()); } "range_partitioning.slt" => { info!("Registering range partitioned table"); @@ -181,7 +181,7 @@ impl TestContext { } "metadata.slt" | "arrow_field.slt" => { info!("Registering metadata table tables"); - register_metadata_tables(test_ctx.session_ctx()).await; + register_metadata_tables(test_ctx.session_ctx()); } "union_function.slt" => { info!("Registering table with union column"); @@ -370,7 +370,7 @@ pub async fn register_partition_table(test_ctx: &mut TestContext) { } // registers a LOCAL TEMPORARY table. -pub async fn register_temp_table(ctx: &SessionContext) { +pub fn register_temp_table(ctx: &SessionContext) { #[derive(Debug)] struct TestTable(TableType); @@ -402,7 +402,7 @@ pub async fn register_temp_table(ctx: &SessionContext) { .unwrap(); } -pub async fn register_table_with_many_types(ctx: &SessionContext) { +pub fn register_table_with_many_types(ctx: &SessionContext) { let catalog = MemoryCatalogProvider::new(); let schema = MemorySchemaProvider::new(); @@ -418,7 +418,7 @@ pub async fn register_table_with_many_types(ctx: &SessionContext) { .unwrap(); } -pub async fn register_table_with_map(ctx: &SessionContext) { +pub fn register_table_with_map(ctx: &SessionContext) { let key = Field::new("key", DataType::Int64, false); let value = Field::new("value", DataType::Int64, true); let map_field = @@ -468,7 +468,7 @@ fn table_with_many_types() -> Arc { } /// Registers a table_with_metadata that contains both field level and Table level metadata -pub async fn register_metadata_tables(ctx: &SessionContext) { +pub fn register_metadata_tables(ctx: &SessionContext) { let id = Field::new("id", DataType::Int32, true).with_metadata(HashMap::from([( String::from("metadata_key"), String::from("the id field"), diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs index 4cd856fc562e8..47a944504c510 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs @@ -88,7 +88,7 @@ pub async fn from_scalar_function( // In those cases we build a balanced tree of BinaryExprs arg_list_to_binary_op_tree(op, args) } else if let Some(builder) = BuiltinExprBuilder::try_from_name(fn_name) { - builder.build(consumer, f, args).await + builder.build(consumer, f, args) } else { not_impl_err!("Unsupported function name: {fn_name:?}") } @@ -206,34 +206,32 @@ impl BuiltinExprBuilder { } } - pub async fn build( + pub fn build( self, consumer: &impl SubstraitConsumer, f: &ScalarFunction, args: Vec, ) -> Result { match self.expr_name.as_str() { - "like" => Self::build_like_expr(false, false, f, args).await, - "ilike" => Self::build_like_expr(true, false, f, args).await, - "like_match" => Self::build_like_expr(false, false, f, args).await, - "like_imatch" => Self::build_like_expr(true, false, f, args).await, - "like_not_match" => Self::build_like_expr(false, true, f, args).await, - "like_not_imatch" => Self::build_like_expr(true, true, f, args).await, + "like" => Self::build_like_expr(false, false, f, args), + "ilike" => Self::build_like_expr(true, false, f, args), + "like_match" => Self::build_like_expr(false, false, f, args), + "like_imatch" => Self::build_like_expr(true, false, f, args), + "like_not_match" => Self::build_like_expr(false, true, f, args), + "like_not_imatch" => Self::build_like_expr(true, true, f, args), "not" | "negative" | "negate" | "is_null" | "is_not_null" | "is_true" | "is_false" | "is_not_true" | "is_not_false" | "is_unknown" - | "is_not_unknown" => Self::build_unary_expr(&self.expr_name, args).await, - "and_not" | "xor" => Self::build_binary_expr(&self.expr_name, args).await, - "between" => Self::build_between_expr(&self.expr_name, args).await, - "logb" => { - Self::build_custom_handling_expr(consumer, &self.expr_name, args).await - } + | "is_not_unknown" => Self::build_unary_expr(&self.expr_name, args), + "and_not" | "xor" => Self::build_binary_expr(&self.expr_name, args), + "between" => Self::build_between_expr(&self.expr_name, args), + "logb" => Self::build_custom_handling_expr(consumer, &self.expr_name, args), _ => { not_impl_err!("Unsupported builtin expression: {}", self.expr_name) } } } - async fn build_unary_expr(fn_name: &str, args: Vec) -> Result { + fn build_unary_expr(fn_name: &str, args: Vec) -> Result { let [arg] = match args.try_into() { Ok(args_arr) => args_arr, Err(_) => return substrait_err!("Expected one argument for {fn_name} expr"), @@ -257,7 +255,7 @@ impl BuiltinExprBuilder { Ok(expr) } - async fn build_like_expr( + fn build_like_expr( case_insensitive: bool, negated: bool, f: &ScalarFunction, @@ -306,7 +304,7 @@ impl BuiltinExprBuilder { })) } - async fn build_binary_expr(fn_name: &str, args: Vec) -> Result { + fn build_binary_expr(fn_name: &str, args: Vec) -> Result { let [a, b] = match args.try_into() { Ok(args_arr) => args_arr, Err(_) => { @@ -330,7 +328,7 @@ impl BuiltinExprBuilder { Self::build_and_not_expr(or_expr, and_expr) } - async fn build_between_expr(fn_name: &str, args: Vec) -> Result { + fn build_between_expr(fn_name: &str, args: Vec) -> Result { let [expression, low, high] = match args.try_into() { Ok(args_arr) => args_arr, Err(_) => { @@ -347,18 +345,18 @@ impl BuiltinExprBuilder { } //This handles any functions that require custom handling - async fn build_custom_handling_expr( + fn build_custom_handling_expr( consumer: &impl SubstraitConsumer, fn_name: &str, args: Vec, ) -> Result { match fn_name { - "logb" => Self::build_logb_expr(consumer, args).await, + "logb" => Self::build_logb_expr(consumer, args), _ => not_impl_err!("Unsupported custom handled expression: {}", fn_name), } } - async fn build_logb_expr( + fn build_logb_expr( consumer: &impl SubstraitConsumer, args: Vec, ) -> Result { diff --git a/datafusion/substrait/src/serializer.rs b/datafusion/substrait/src/serializer.rs index ee71bc3121afe..bcc9f5cf50eac 100644 --- a/datafusion/substrait/src/serializer.rs +++ b/datafusion/substrait/src/serializer.rs @@ -70,12 +70,12 @@ pub async fn deserialize(path: impl AsRef) -> Result> { let mut file = OpenOptions::new().read(true).open(path).await?; file.read_to_end(&mut protobuf_in).await?; - deserialize_bytes(protobuf_in).await + deserialize_bytes(&protobuf_in) } /// Deserializes a plan from the bytes. -pub async fn deserialize_bytes(proto_bytes: Vec) -> Result> { - Ok(Box::new(Message::decode(&*proto_bytes).map_err(|e| { +pub fn deserialize_bytes(proto_bytes: &[u8]) -> Result> { + Ok(Box::new(Message::decode(proto_bytes).map_err(|e| { DataFusionError::Substrait(format!("Failed to decode plan: {e}")) })?)) } diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs index 018e1aef80ea1..f084d3170edcc 100644 --- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs @@ -2224,7 +2224,7 @@ fn check_post_join_filters(rel: &Rel) -> Result<()> { } } -async fn verify_post_join_filter_value(proto: Box) -> Result<()> { +fn verify_post_join_filter_value(proto: &Plan) -> Result<()> { for relation in &proto.relations { match relation.rel_type.as_ref() { Some(rt) => match rt { @@ -2263,10 +2263,7 @@ fn count_read_filters(rel: &Rel, filter_count: &mut u32) -> Result<()> { } } -async fn assert_read_filter_count( - proto: Box, - expected_filter_count: u32, -) -> Result<()> { +fn assert_read_filter_count(proto: &Plan, expected_filter_count: u32) -> Result<()> { let mut filter_count: u32 = 0; for relation in &proto.relations { match relation.rel_type.as_ref() { @@ -2644,7 +2641,7 @@ async fn roundtrip_verify_post_join_filter(sql: &str) -> Result<()> { let proto = roundtrip_with_ctx(sql, ctx).await?; // verify that the join filters are None - verify_post_join_filter_value(proto).await + verify_post_join_filter_value(&proto) } async fn roundtrip_verify_read_filter_count( @@ -2655,7 +2652,7 @@ async fn roundtrip_verify_read_filter_count( let proto = roundtrip_with_ctx(sql, ctx).await?; // verify that filter counts in read relations are as expected - assert_read_filter_count(proto, expected_filter_count).await + assert_read_filter_count(&proto, expected_filter_count) } async fn roundtrip_all_types(sql: &str) -> Result<()> { diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index 5f1609af30fc6..6ca4d71a7883f 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -847,6 +847,35 @@ fn catalog_list(&self) -> Arc { See [PR #23703](https://github.com/apache/datafusion/pull/23703) for details. +### Unused `async` removed from several public functions + +Public functions that were declared `async` but never awaited anything are now +synchronous: + +- `CsvFormat::read_to_delimited_chunks_from_stream` (in + `datafusion_datasource_csv`, re-exported as + `datafusion::datasource::file_format::csv::CsvFormat`) +- `datafusion_substrait::serializer::deserialize_bytes`, which now also borrows + its input as `&[u8]` instead of taking an owned `Vec` +- `datafusion::test_util::parquet::TestParquetFile::create_scan` + +**Migration guide:** + +Remove `.await` from call sites; the compiler flags each one, since `.await` +on a non-future value does not compile: + +```rust,ignore +// Before +let stream = csv_format + .read_to_delimited_chunks_from_stream(input) + .await; +let plan = deserialize_bytes(proto_bytes).await?; + +// After +let stream = csv_format.read_to_delimited_chunks_from_stream(input); +let plan = deserialize_bytes(&proto_bytes)?; +``` + ### `MSRV` updated to 1.94.0 The Minimum Supported Rust Version (MSRV) has been updated to [`1.94.0`]. From 0d8482c76984e615990b594a44fbc596c6b24a34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:56:39 -0400 Subject: [PATCH 015/109] chore(deps-dev): bump webpack-dev-server from 5.2.6 to 6.0.0 in /datafusion/wasmtest/datafusion-wasm-app (#23868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [webpack-dev-server](https://github.com/webpack/webpack-dev-server) from 5.2.6 to 6.0.0.
Release notes

Sourced from webpack-dev-server's releases.

v6.0.0

Major Changes

  • Bump Express to v5. See the Express 5 migration guide for the full list of breaking changes. (by @​bjohansebas in #5674)

  • Bump the webpack peer dependency range from ^5.0.0 to ^5.101.0. (by @​bjohansebas in #5674)

  • Drop support for Node.js < 22.15.0. (by @​bjohansebas in #5674)

  • Convert the source to native ES modules. The package keeps "type": "module" and now exposes both an ESM and a CommonJS build via the exports field: ESM consumers import the native lib/, while CommonJS consumers require() a transpiled dist/ build, allowing the package to be consumed from both ESM and CommonJS without relying on require(ESM) for CommonJS consumers. (by @​bjohansebas in #5674)

  • Remove CLI flags. Use the serve command from webpack-cli together with a configuration file or the programmatic API instead. (by @​bjohansebas in #5674)

  • Remove the internalIP and internalIPSync static methods from Server. Resolve the local IP yourself if you need it. (by @​bjohansebas in #5674)

  • Remove the bypass option from proxy configuration. Use the router or context options provided by http-proxy-middleware instead. (by @​bjohansebas in #5674)

  • Remove SockJS support. The webSocketServer option no longer accepts "sockjs"; use the default "ws" transport instead. (by @​bjohansebas in #5674)

  • Remove the spdy dependency. Use the built-in node:http2 module via the server option for HTTP/2 support. (by @​bjohansebas in #5674)

  • Update http-proxy-middleware to v4. See the http-proxy-middleware v3 release notes and v4 release notes for the full list of breaking changes. (by @​bjohansebas in #5674)

  • Update webpack-dev-middleware to v8 and sync originalUrl for middleware compatibility. server.middleware.getFilenameFromUrl() is now asynchronous and resolves to { filename, extra: { stats, outputFileSystem } }. See the webpack-dev-middleware v8 release notes for details. (by @​bjohansebas in #5674)

Minor Changes

  • Add plugin support. webpack-dev-server can now be used as a webpack plugin, integrating with the compiler lifecycle without explicitly passing a compiler, preventing multiple server starts on recompilation, ensuring clean shutdown, and supporting MultiCompiler setups with multiple independent plugin servers. (by @​bjohansebas in #5674)

  • Enable the compression middleware for HTTP/2 connections. (by @​bjohansebas in #5674)

  • Remove the colorette dependency in favor of native ANSI styling. (by @​bjohansebas in #5674)

  • Update chokidar to v5 and extend watchFiles.options.ignored to support glob string patterns via tinyglobby. (by @​bjohansebas in #5674)

  • Use compiler.platform to determine the target environment instead of inspecting the resolved target string. Universal targets ("universal" or ["web", "node"], where compiler.platform.universal is true since webpack 5.108.0) are treated as web targets so the client runtime is injected. (by @​bjohansebas in #5674)

  • Use the WHATWG URL API instead of the deprecated url.parse. (by @​bjohansebas in #5674)

Patch Changes

  • Bump production dependencies, notably open to v11 and p-retry to v8. (by @​bjohansebas in #5674)

  • Reject cross-site requests to the internal open-editor and invalidate endpoints. They performed state-changing actions (opening a file in the editor, forcing a recompilation) on any GET request, so a page the developer visited could trigger them. They now require a same-origin request, validated via Sec-Fetch-Site with an Origin/Host fallback. (by @​bjohansebas in #5691)

  • Treat loopback aliases (127.0.0.1, ::1, localhost) as equivalent in isSameOrigin so the WebSocket client does not reject valid same-origin connections. (by @​bjohansebas in #5674)

  • Migrate the test suite from Jest to node:test and set up the jsdom environment. (by @​bjohansebas in #5674)

... (truncated)

Changelog

Sourced from webpack-dev-server's changelog.

6.0.0

Major Changes

  • Bump Express to v5. See the Express 5 migration guide for the full list of breaking changes. (by @​bjohansebas in #5674)

  • Bump the webpack peer dependency range from ^5.0.0 to ^5.101.0. (by @​bjohansebas in #5674)

  • Drop support for Node.js < 22.15.0. (by @​bjohansebas in #5674)

  • Convert the source to native ES modules. The package keeps "type": "module" and now exposes both an ESM and a CommonJS build via the exports field: ESM consumers import the native lib/, while CommonJS consumers require() a transpiled dist/ build — so the package works from both ESM and CommonJS, including environments where require(ESM) is not supported. (by @​bjohansebas in #5674)

  • Remove CLI flags. Use the serve command from webpack-cli together with a configuration file or the programmatic API instead. (by @​bjohansebas in #5674)

  • Remove the internalIP and internalIPSync static methods from Server. Resolve the local IP yourself if you need it. (by @​bjohansebas in #5674)

  • Remove the bypass option from proxy configuration. Use the router or context options provided by http-proxy-middleware instead. (by @​bjohansebas in #5674)

  • Remove SockJS support. The webSocketServer option no longer accepts "sockjs"; use the default "ws" transport instead. (by @​bjohansebas in #5674)

  • Remove the spdy dependency. Use the built-in node:http2 module via the server option for HTTP/2 support. (by @​bjohansebas in #5674)

  • Update http-proxy-middleware to v4. See the http-proxy-middleware v3 release notes and v4 release notes for the full list of breaking changes. (by @​bjohansebas in #5674)

  • Update webpack-dev-middleware to v8 and sync originalUrl for middleware compatibility. server.middleware.getFilenameFromUrl() is now asynchronous and resolves to { filename, extra: { stats, outputFileSystem } }. See the webpack-dev-middleware v8 release notes for details. (by @​bjohansebas in #5674)

Minor Changes

  • Add plugin support. webpack-dev-server can now be used as a webpack plugin, integrating with the compiler lifecycle without explicitly passing a compiler, preventing multiple server starts on recompilation, ensuring clean shutdown, and supporting MultiCompiler setups with multiple independent plugin servers. (by @​bjohansebas in #5674)

  • Enable the compression middleware for HTTP/2 connections. (by @​bjohansebas in #5674)

  • Remove the colorette dependency in favor of native ANSI styling. (by @​bjohansebas in #5674)

  • Update chokidar to v5 and extend watchFiles.options.ignored to support glob string patterns via tinyglobby. (by @​bjohansebas in #5674)

  • Use compiler.platform to determine the target environment instead of inspecting the resolved target string. Universal targets ("universal" or ["web", "node"], where compiler.platform.universal is true since webpack 5.108.0) are treated as web targets so the client runtime is injected. (by @​bjohansebas in #5674)

  • Use the WHATWG URL API instead of the deprecated url.parse. (by @​bjohansebas in #5674)

Patch Changes

  • Bump production dependencies, notably open to v11 and p-retry to v8. (by @​bjohansebas in #5674)

  • Reject cross-site requests to the internal open-editor and invalidate endpoints. They performed state-changing actions (opening a file in the editor, forcing a recompilation) on any GET request, so a page the developer visited could trigger them. They now require a same-origin request, validated via Sec-Fetch-Site with an Origin/Host fallback. (by @​bjohansebas in #5691)

  • Treat loopback aliases (127.0.0.1, ::1, localhost) as equivalent in isSameOrigin so the WebSocket client does not reject valid same-origin connections. (by @​bjohansebas in #5674)

  • Migrate the test suite from Jest to node:test and set up the jsdom environment. (by @​bjohansebas in #5674)

... (truncated)

Commits
  • 05cb792 chore(release): new release (#5692)
  • a451839 fix: handle middleware teardown in plugin mode (#5703)
  • c2d23a7 fix: load ESM-only dependencies with native import() in the CommonJS build (#...
  • ba54764 fix: reject cross-site requests to open-editor and invalidate endpoints (#5691)
  • 2b369b3 fixup!
  • 08a0ea7 fix: ensure undefined options default to an empty object in Server constructor
  • 797b9e7 fix: handle undefined options in Server constructor
  • e90221c feat: plugin support (#5650)
  • 4c351e1 feat: support universal platform as a web target (#5690)
  • 2236aa4 chore: update http-proxy-middleware to version 4.1.1 and add tests for pathRe...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=webpack-dev-server&package-manager=npm_and_yarn&previous-version=5.2.6&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/apache/datafusion/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../datafusion-wasm-app/package-lock.json | 3635 ++++++++--------- .../wasmtest/datafusion-wasm-app/package.json | 2 +- 2 files changed, 1709 insertions(+), 1928 deletions(-) diff --git a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json index e863fe5e8da15..6fd3fb8ab0646 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json @@ -15,7 +15,7 @@ "copy-webpack-plugin": "14.0.0", "webpack": "5.105.0", "webpack-cli": "5.1.4", - "webpack-dev-server": "5.2.6" + "webpack-dev-server": "6.0.0" } }, "../pkg": { @@ -391,21 +391,20 @@ "dev": true }, "node_modules/@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "dependencies": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "node_modules/@types/express-serve-static-core": { - "version": "4.17.36", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.36.tgz", - "integrity": "sha512-zbivROJ0ZqLAtMzgzIUC4oNqDG9iF0lSsAqpOD9kbs5xcIM3dTiyuHvBc7R8MtWBp3AAWGaovJa+wzWPjLYW7Q==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.2.tgz", + "integrity": "sha512-d3KvEXBSo/lOAMc2u6fkyDHBvetBHeqD7wm/AcXfLpSOQwlmG9D/aQ0SFswVjv05p7ullQS7Mjohj6/VdbZuTg==", "dev": true, "dependencies": { "@types/node": "*", @@ -415,20 +414,11 @@ } }, "node_modules/@types/http-errors": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.2.tgz", - "integrity": "sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", "dev": true }, - "node_modules/@types/http-proxy": { - "version": "1.17.12", - "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.12.tgz", - "integrity": "sha512-kQtujO08dVtQ2wXAuSFfk9ASy3sug4+ogFR8Kd8UgP8PEuc1/G/8yjYRmp//PcDNJEUKOza/MrQu15bouEUCiw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -459,13 +449,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, - "node_modules/@types/retry": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", - "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz", @@ -487,24 +470,12 @@ } }, "node_modules/@types/serve-static": { - "version": "1.15.7", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", - "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "*" - } - }, - "node_modules/@types/sockjs": { - "version": "0.3.36", - "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz", - "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", - "dev": true, - "license": "MIT", - "dependencies": { "@types/node": "*" } }, @@ -721,18 +692,43 @@ "dev": true }, "node_modules/accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "dev": true, "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" }, "engines": { "node": ">= 0.6" } }, + "node_modules/accepts/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -814,27 +810,6 @@ "ansi-html": "bin/ansi-html" } }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "dev": true, - "license": "MIT" - }, "node_modules/asn1js": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.10.tgz", @@ -867,101 +842,46 @@ "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", - "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "dev": true }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "dev": true, "dependencies": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" }, "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/body-parser/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.0.0" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/body-parser/node_modules/depd": { + "node_modules/body-parser/node_modules/content-type": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/body-parser/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", "dev": true, - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, "engines": { - "node": ">= 0.8" + "node": ">=18" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/body-parser/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/body-parser/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/bonjour-service": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz", @@ -1029,7 +949,6 @@ "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", "dev": true, - "license": "MIT", "dependencies": { "run-applescript": "^7.0.0" }, @@ -1063,7 +982,6 @@ "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", "dev": true, - "license": "MIT", "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" @@ -1077,7 +995,6 @@ "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", "dev": true, - "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" @@ -1110,28 +1027,18 @@ ] }, "node_modules/chokidar": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "dev": true, - "license": "MIT", "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" + "readdirp": "^5.0.0" }, "engines": { - "node": ">= 8.10.0" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" } }, "node_modules/chrome-trace-event": { @@ -1250,65 +1157,44 @@ } }, "node_modules/content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/content-disposition/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, "node_modules/content-type": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", "dev": true, - "license": "MIT" + "engines": { + "node": ">=6.6.0" + } }, "node_modules/copy-webpack-plugin": { "version": "14.0.0", @@ -1355,12 +1241,6 @@ "node": ">=20.0.0" } }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true - }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -1380,27 +1260,33 @@ "link": true }, "node_modules/debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dev": true, "dependencies": { - "ms": "^2.1.1" + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, "node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, "node_modules/default-browser": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz", - "integrity": "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", + "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", "dev": true, - "license": "MIT", "dependencies": { "bundle-name": "^4.1.0", "default-browser-id": "^5.0.0" @@ -1413,11 +1299,10 @@ } }, "node_modules/default-browser-id": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.0.tgz", - "integrity": "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", "dev": true, - "license": "MIT", "engines": { "node": ">=18" }, @@ -1430,7 +1315,6 @@ "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", "dev": true, - "license": "MIT", "engines": { "node": ">=12" }, @@ -1439,31 +1323,14 @@ } }, "node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", - "dev": true, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true, - "license": "MIT", "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" + "node": ">= 0.8" } }, - "node_modules/detect-node": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.0.4.tgz", - "integrity": "sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==", - "dev": true - }, "node_modules/dns-packet": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", @@ -1482,7 +1349,6 @@ "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", "dev": true, - "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", @@ -1496,8 +1362,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "dev": true, - "license": "MIT" + "dev": true }, "node_modules/electron-to-chromium": { "version": "1.5.286", @@ -1510,7 +1375,6 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -1545,7 +1409,6 @@ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" } @@ -1555,7 +1418,6 @@ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" } @@ -1567,11 +1429,10 @@ "dev": true }, "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", "dev": true, - "license": "MIT", "dependencies": { "es-errors": "^1.3.0" }, @@ -1591,7 +1452,7 @@ "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "dev": true }, "node_modules/eslint-scope": { @@ -1642,17 +1503,10 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true - }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -1663,112 +1517,83 @@ } }, "node_modules/express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", - "dev": true, - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" }, "engines": { - "node": ">= 0.10.0" + "node": ">= 18" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "node_modules/express/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.0.0" + "engines": { + "node": ">= 0.6" } }, - "node_modules/express/node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "node_modules/express/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "dev": true, - "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, "engines": { - "node": ">= 0.8" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/express/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/express/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true - }, - "node_modules/fast-uri": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", - "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/fast-uri": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "dev": true, "funding": [ { @@ -1790,18 +1615,6 @@ "node": ">= 4.9.1" } }, - "node_modules/faye-websocket": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", - "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", - "dev": true, - "dependencies": { - "websocket-driver": ">=0.5.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1815,42 +1628,24 @@ } }, "node_modules/finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "dev": true, - "license": "MIT", "dependencies": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/finalhandler/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/finalhandler/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/find-up": { @@ -1866,59 +1661,22 @@ "node": ">=8" } }, - "node_modules/follow-redirects": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", - "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + "node": ">= 0.8" } }, "node_modules/function-bind": { @@ -1935,7 +1693,6 @@ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dev": true, - "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", @@ -1960,7 +1717,6 @@ "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", "dev": true, - "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" @@ -1969,18 +1725,6 @@ "node": ">= 0.4" } }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/glob-to-regexp": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", @@ -1992,7 +1736,6 @@ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -2006,12 +1749,6 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, - "node_modules/handle-thing": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", - "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", - "dev": true - }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -2038,7 +1775,6 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -2047,11 +1783,10 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "dev": true, - "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, @@ -2059,132 +1794,71 @@ "node": ">= 0.4" } }, - "node_modules/hpack.js": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", - "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=", - "dev": true, - "dependencies": { - "inherits": "^2.0.1", - "obuf": "^1.0.0", - "readable-stream": "^2.0.1", - "wbuf": "^1.1.0" - } - }, - "node_modules/http-deceiver": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", - "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc=", - "dev": true - }, "node_modules/http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "dev": true, - "license": "MIT", "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" }, "engines": { "node": ">= 0.8" - } - }, - "node_modules/http-errors/node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-errors/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/http-errors/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-parser-js": { - "version": "0.5.8", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz", - "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==", - "dev": true - }, - "node_modules/http-proxy": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", - "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", - "dev": true, - "dependencies": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" }, - "engines": { - "node": ">=8.0.0" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/http-proxy-middleware": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", - "integrity": "sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-4.2.0.tgz", + "integrity": "sha512-ZA+oNOoM+GLoFTIzhkJptVQov73Srep2LBqhF8hG8CIPKO3nam1jonXVQ/QUH8RbwsmaaVz2SOJdzBNBHNtKbw==", "dev": true, "dependencies": { - "@types/http-proxy": "^1.17.8", - "http-proxy": "^1.18.1", - "is-glob": "^4.0.1", - "is-plain-obj": "^3.0.0", - "micromatch": "^4.0.2" + "debug": "^4.4.3", + "httpxy": "^0.5.4", + "is-glob": "^4.0.3", + "is-plain-obj": "^4.1.0", + "micromatch": "^4.0.8" }, "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "@types/express": "^4.17.13" - }, - "peerDependenciesMeta": { - "@types/express": { - "optional": true - } + "node": "^22.15.0 || ^24.0.0 || >=26.0.0" } }, + "node_modules/httpxy": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/httpxy/-/httpxy-0.5.5.tgz", + "integrity": "sha512-uDjmnPyp1q4Sgzf3w+J/Fc6UqcCEj0x4Wjp7OqK5dGhNeDgpyrAmnS6ey8QWrX3SWDon2DMKf9sBa5X9+CVyMA==", + "dev": true + }, "node_modules/hyperdyperid": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", "integrity": "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==", "dev": true, - "license": "MIT", "engines": { "node": ">=10.18" } }, "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "dev": true, "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" }, "engines": { "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/import-local": { @@ -2207,9 +1881,9 @@ } }, "node_modules/inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, "node_modules/interpret": { @@ -2222,27 +1896,14 @@ } }, "node_modules/ipaddr.js": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", - "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", + "integrity": "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==", "dev": true, "engines": { "node": ">= 10" } }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "license": "MIT", - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", @@ -2260,7 +1921,6 @@ "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", "dev": true, - "license": "MIT", "bin": { "is-docker": "cli.js" }, @@ -2292,12 +1952,23 @@ "node": ">=0.10.0" } }, + "node_modules/is-in-ssh": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", + "integrity": "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw==", + "dev": true, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-inside-container": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", "dev": true, - "license": "MIT", "dependencies": { "is-docker": "^3.0.0" }, @@ -2312,11 +1983,10 @@ } }, "node_modules/is-network-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", - "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", "dev": true, - "license": "MIT", "engines": { "node": ">=16" }, @@ -2334,12 +2004,12 @@ } }, "node_modules/is-plain-obj": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", - "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", "dev": true, "engines": { - "node": ">=10" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2357,12 +2027,17 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true + }, "node_modules/is-wsl": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", - "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", "dev": true, - "license": "MIT", "dependencies": { "is-inside-container": "^1.0.0" }, @@ -2373,12 +2048,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2469,39 +2138,50 @@ "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" } }, "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", "dev": true, - "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/memfs": { - "version": "4.17.2", - "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.17.2.tgz", - "integrity": "sha512-NgYhCOWgovOXSzvYgUW0LQ7Qy72rWQMGGFJDoWg4G30RHd3z77VbYdtJ4fembJXBy8pMIUA31XNAupobOQlwdg==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jsonjoy.com/json-pack": "^1.0.3", - "@jsonjoy.com/util": "^1.3.0", - "tree-dump": "^1.0.1", + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.64.0.tgz", + "integrity": "sha512-Kw72fgY7Wn+sD8KmtNWSafl1dz0UvAsE/PHs3YVfLiaZuA3HxNm9sRLqAu0ATiBGJvME1PxZXbBZPv5GycDeAw==", + "dev": true, + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-to-fsa": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "@jsonjoy.com/json-pack": "^1.11.0", + "@jsonjoy.com/util": "^1.9.0", + "glob-to-regex.js": "^1.0.1", + "thingies": "^2.5.0", + "tree-dump": "^1.0.3", "tslib": "^2.0.0" }, - "engines": { - "node": ">= 4.0.0" - }, "funding": { "type": "github", "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" } }, "node_modules/memfs/node_modules/@jsonjoy.com/base64": { @@ -2509,7 +2189,6 @@ "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-1.1.2.tgz", "integrity": "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==", "dev": true, - "license": "Apache-2.0", "engines": { "node": ">=10.0" }, @@ -2521,18 +2200,11 @@ "tslib": "2" } }, - "node_modules/memfs/node_modules/@jsonjoy.com/json-pack": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.2.0.tgz", - "integrity": "sha512-io1zEbbYcElht3tdlqEOFxZ0dMTYrHz9iMf0gqn1pPjZFTCgM5R4R5IMA20Chb2UPYYsxjzs8CgZ7Nb5n2K2rA==", + "node_modules/memfs/node_modules/@jsonjoy.com/buffers": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-17.67.0.tgz", + "integrity": "sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==", "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jsonjoy.com/base64": "^1.1.1", - "@jsonjoy.com/util": "^1.1.2", - "hyperdyperid": "^1.2.0", - "thingies": "^1.20.0" - }, "engines": { "node": ">=10.0" }, @@ -2544,12 +2216,11 @@ "tslib": "2" } }, - "node_modules/memfs/node_modules/@jsonjoy.com/util": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.6.0.tgz", - "integrity": "sha512-sw/RMbehRhN68WRtcKCpQOPfnH6lLP4GJfqzi3iYej8tnzpZUDr6UkZYJjcjjC0FWEJOJbyM3PTIwxucUmDG2A==", + "node_modules/memfs/node_modules/@jsonjoy.com/codegen": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-1.0.0.tgz", + "integrity": "sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==", "dev": true, - "license": "Apache-2.0", "engines": { "node": ">=10.0" }, @@ -2561,25 +2232,38 @@ "tslib": "2" } }, - "node_modules/memfs/node_modules/thingies": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", - "integrity": "sha512-hsqsJsFMsV+aD4s3CWKk85ep/3I9XzYV/IXaSouJMYIoDlgyi11cBhsqYe9/geRfB0YIikBQg6raRaM+nIMP9g==", + "node_modules/memfs/node_modules/@jsonjoy.com/fs-core": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-core/-/fs-core-4.64.0.tgz", + "integrity": "sha512-zs2TAq7Six5jgMuoMNjpspAvOP3mhtgq/k1UyQodEzCtQi/N83y2/y+zcvnZSGp/Rxq96DBN+bValOBQAyn/ew==", "dev": true, - "license": "Unlicense", + "dependencies": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + }, "engines": { - "node": ">=10.18" + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" }, "peerDependencies": { - "tslib": "^2" + "tslib": "2" } }, - "node_modules/memfs/node_modules/tree-dump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.0.3.tgz", - "integrity": "sha512-il+Cv80yVHFBwokQSfd4bldvr1Md951DpgAGfmhydt04L+YzHgubm2tQ7zueWDcGENKHq0ZvGFR/hjvNXilHEg==", + "node_modules/memfs/node_modules/@jsonjoy.com/fs-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-fsa/-/fs-fsa-4.64.0.tgz", + "integrity": "sha512-nMWOVbkLFyEgmXZih3wyvxA9XpgyyqyfrINMHvEFqhi7uqfRl7c9ERJt6yX7vgMPrB9Uo+OJO+Spa0cFzPD01w==", "dev": true, - "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + }, "engines": { "node": ">=10.0" }, @@ -2591,19 +2275,387 @@ "tslib": "2" } }, - "node_modules/memfs/node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "node_modules/memfs/node_modules/@jsonjoy.com/fs-node": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node/-/fs-node-4.64.0.tgz", + "integrity": "sha512-dO+NNkODbUli4uV42bcNrrLvq5rE7SNpdZ5TNd0dtbLsAaNK3MDiIC9lUi+brboGoIjW6vd2fB1qao60nrk5xA==", + "dev": true, + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "glob-to-regex.js": "^1.0.0", + "thingies": "^2.5.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-node-builtins": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-builtins/-/fs-node-builtins-4.64.0.tgz", + "integrity": "sha512-/o7WRFhUWaM/fOrslwLZGnzn4RmRILykn+lAL+mNObqqRNw+CQSiij6hpCeZ+C7buhdoVo7go/OYqzaSUfDYmA==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-node-to-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-to-fsa/-/fs-node-to-fsa-4.64.0.tgz", + "integrity": "sha512-WDD9WVs0hb7UAEKTgZW2f66WDrbj7gIIWwpP3spbLyXa0rghtUaFTB8L4gdR3ZCWwiKIsj38/CNijpVmpnuPUw==", + "dev": true, + "dependencies": { + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-node-utils": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-utils/-/fs-node-utils-4.64.0.tgz", + "integrity": "sha512-k5Indsx9hWW9xSF7Y6oSKKwtCUNhzZxadub3owhIlitc+iMRVlPPdX2duTKQWBL3qNWpXya8jykgaaWpheeS4w==", + "dev": true, + "dependencies": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "glob-to-regex.js": "^1.0.1" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-print": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-print/-/fs-print-4.64.0.tgz", + "integrity": "sha512-PHZFccchvkhWrwPWHjmVAhbC3vSHCtyZvlZfJJ3ho2bnzl450hXri6/8e6pbkWdH+SkmLXNml0sV8e5HDAfxKw==", + "dev": true, + "dependencies": { + "@jsonjoy.com/fs-node-utils": "4.64.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-snapshot/-/fs-snapshot-4.64.0.tgz", + "integrity": "sha512-oM7UDeL83q6NBzzsfKAsYKXKVXlykKFqqOLh4xZZKAzzROTlInkPbc6LTDGThEOnPiFiUzA7tYziHG9xavd76Q==", + "dev": true, + "dependencies": { + "@jsonjoy.com/buffers": "^17.65.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/json-pack": "^17.65.0", + "@jsonjoy.com/util": "^17.65.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/base64": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-17.67.0.tgz", + "integrity": "sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/codegen": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-17.67.0.tgz", + "integrity": "sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==", "dev": true, - "license": "0BSD" + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/json-pack": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-17.67.0.tgz", + "integrity": "sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==", + "dev": true, + "dependencies": { + "@jsonjoy.com/base64": "17.67.0", + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0", + "@jsonjoy.com/json-pointer": "17.67.0", + "@jsonjoy.com/util": "17.67.0", + "hyperdyperid": "^1.2.0", + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/json-pointer": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-17.67.0.tgz", + "integrity": "sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==", + "dev": true, + "dependencies": { + "@jsonjoy.com/util": "17.67.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/util": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-17.67.0.tgz", + "integrity": "sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==", + "dev": true, + "dependencies": { + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/json-pack": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.21.0.tgz", + "integrity": "sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==", + "dev": true, + "dependencies": { + "@jsonjoy.com/base64": "^1.1.2", + "@jsonjoy.com/buffers": "^1.2.0", + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/json-pointer": "^1.0.2", + "@jsonjoy.com/util": "^1.9.0", + "hyperdyperid": "^1.2.0", + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/json-pack/node_modules/@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/json-pointer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-1.0.2.tgz", + "integrity": "sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==", + "dev": true, + "dependencies": { + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/util": "^1.9.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/util": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.9.0.tgz", + "integrity": "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==", + "dev": true, + "dependencies": { + "@jsonjoy.com/buffers": "^1.0.0", + "@jsonjoy.com/codegen": "^1.0.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/@jsonjoy.com/util/node_modules/@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/glob-to-regex.js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/glob-to-regex.js/-/glob-to-regex.js-1.2.0.tgz", + "integrity": "sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/thingies": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/thingies/-/thingies-2.6.0.tgz", + "integrity": "sha512-rMHRjmlFLM1R96UYPvpmnc3LYtdFrT33JIB7L9hetGue1qAPfn1N2LJeEjxUSidu1Iku+haLZXDuEXUHNGO/lg==", + "dev": true, + "engines": { + "node": ">=10.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "^2" + } + }, + "node_modules/memfs/node_modules/tree-dump": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.1.0.tgz", + "integrity": "sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==", + "dev": true, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/memfs/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true }, "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "dev": true, - "license": "MIT", + "engines": { + "node": ">=18" + }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } @@ -2614,22 +2666,11 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "dev": true, - "license": "MIT", "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" @@ -2638,19 +2679,6 @@ "node": ">=8.6" } }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true, - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -2672,12 +2700,6 @@ "node": ">= 0.6" } }, - "node_modules/minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", - "dev": true - }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -2699,9 +2721,9 @@ } }, "node_modules/negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "dev": true, "engines": { "node": ">= 0.6" @@ -2733,7 +2755,6 @@ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -2741,18 +2762,11 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obuf": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", - "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", - "dev": true - }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "dev": true, - "license": "MIT", "dependencies": { "ee-first": "1.1.1" }, @@ -2769,20 +2783,30 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, "node_modules/open": { - "version": "10.1.2", - "resolved": "https://registry.npmjs.org/open/-/open-10.1.2.tgz", - "integrity": "sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/open/-/open-11.0.0.tgz", + "integrity": "sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==", "dev": true, - "license": "MIT", "dependencies": { - "default-browser": "^5.2.1", + "default-browser": "^5.4.0", "define-lazy-prop": "^3.0.0", + "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", - "is-wsl": "^3.1.0" + "powershell-utils": "^0.1.0", + "wsl-utils": "^0.3.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2816,18 +2840,15 @@ } }, "node_modules/p-retry": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", - "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-8.0.0.tgz", + "integrity": "sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A==", "dev": true, - "license": "MIT", "dependencies": { - "@types/retry": "0.12.2", - "is-network-error": "^1.0.0", - "retry": "^0.13.1" + "is-network-error": "^1.3.0" }, "engines": { - "node": ">=16.17" + "node": ">=22" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2876,11 +2897,14 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "dev": true, - "license": "MIT" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } }, "node_modules/picocolors": { "version": "1.1.1", @@ -2935,18 +2959,23 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "dev": true }, - "node_modules/process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", - "dev": true + "node_modules/powershell-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", + "integrity": "sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==", + "dev": true, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "dev": true, - "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" @@ -2960,7 +2989,6 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.10" } @@ -2990,12 +3018,13 @@ } }, "node_modules/qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", "dev": true, "dependencies": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" }, "engines": { "node": ">=0.6" @@ -3005,100 +3034,44 @@ } }, "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "dev": true, "dependencies": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" }, "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/raw-body/node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/raw-body/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "node_modules/raw-body/node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dev": true, - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "node": ">= 0.10" } }, "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", "dev": true, - "license": "MIT", - "dependencies": { - "picomatch": "^2.2.1" - }, "engines": { - "node": ">=8.10.0" + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" } }, "node_modules/rechoir": { @@ -3128,12 +3101,6 @@ "node": ">=0.10.0" } }, - "node_modules/requires-port": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true - }, "node_modules/resolve": { "version": "1.22.6", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.6.tgz", @@ -3172,22 +3139,27 @@ "node": ">=8" } }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", "dev": true, - "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, "engines": { - "node": ">= 4" + "node": ">= 18" } }, "node_modules/run-applescript": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", - "integrity": "sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", "dev": true, - "license": "MIT", "engines": { "node": ">=18" }, @@ -3195,12 +3167,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -3226,12 +3192,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/select-hose": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", - "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=", - "dev": true - }, "node_modules/selfsigned": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", @@ -3246,100 +3206,95 @@ } }, "node_modules/send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.0.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, - "node_modules/send/node_modules/debug/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true, - "license": "MIT" - }, - "node_modules/send/node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "node_modules/send/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, - "license": "MIT", "engines": { - "node": ">= 0.8" + "node": ">= 0.6" } }, - "node_modules/send/node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "node_modules/send/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "dev": true, - "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, "engines": { - "node": ">= 0.8" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/send/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } + "dev": true }, "node_modules/serve-index": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", - "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.2.tgz", + "integrity": "sha512-KDj11HScOaLmrPxl70KYNW1PksP4Nb/CLL2yvC+Qd2kHMPEEpfc4Re2e4FOay+bC/+XQl/7zAcWON3JVo5v3KQ==", "dev": true, "dependencies": { - "accepts": "~1.3.4", + "accepts": "~1.3.8", "batch": "0.6.1", "debug": "2.6.9", "escape-html": "~1.0.3", - "http-errors": "~1.6.2", - "mime-types": "~2.1.17", - "parseurl": "~1.3.2" + "http-errors": "~1.8.0", + "mime-types": "~2.1.35", + "parseurl": "~1.3.3" }, "engines": { "node": ">= 0.8.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-index/node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" } }, "node_modules/serve-index/node_modules/debug": { @@ -3351,49 +3306,73 @@ "ms": "2.0.0" } }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/serve-index/node_modules/http-errors": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", "dev": true, "dependencies": { "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" }, "engines": { "node": ">= 0.6" } }, - "node_modules/serve-index/node_modules/setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", - "dev": true + "node_modules/serve-index/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } }, "node_modules/serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "dev": true, - "license": "MIT", "dependencies": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" }, "engines": { - "node": ">= 0.8.0" + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "dev": true, - "license": "ISC" + "dev": true }, "node_modules/shallow-clone": { "version": "3.0.1", @@ -3441,15 +3420,14 @@ } }, "node_modules/side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "dev": true, - "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" }, @@ -3461,14 +3439,13 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "dev": true, - "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -3482,7 +3459,6 @@ "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", "dev": true, - "license": "MIT", "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", @@ -3501,7 +3477,6 @@ "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", "dev": true, - "license": "MIT", "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", @@ -3516,17 +3491,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/sockjs": { - "version": "0.3.24", - "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", - "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", - "dev": true, - "dependencies": { - "faye-websocket": "^0.11.3", - "uuid": "^8.3.2", - "websocket-driver": "^0.7.4" - } - }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -3546,66 +3510,13 @@ "source-map": "^0.6.0" } }, - "node_modules/spdy": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", - "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", - "dev": true, - "dependencies": { - "debug": "^4.1.0", - "handle-thing": "^2.0.0", - "http-deceiver": "^1.2.7", - "select-hose": "^2.0.0", - "spdy-transport": "^3.0.0" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/spdy-transport": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", - "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", - "dev": true, - "dependencies": { - "debug": "^4.1.0", - "detect-node": "^2.0.4", - "hpack.js": "^2.1.6", - "obuf": "^1.1.2", - "readable-stream": "^3.0.6", - "wbuf": "^1.7.3" - } - }, - "node_modules/spdy-transport/node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "dev": true, "engines": { - "node": ">= 0.6" - } - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "dependencies": { - "safe-buffer": "~5.1.0" + "node": ">= 0.8" } }, "node_modules/supports-color": { @@ -3768,7 +3679,6 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true, - "license": "MIT", "engines": { "node": ">=0.6" } @@ -3792,25 +3702,66 @@ } }, "node_modules/type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "dev": true, - "license": "MIT", "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "dev": true, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, "engines": { "node": ">= 0.6" } }, + "node_modules/type-is/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", "dev": true, - "license": "MIT", "engines": { "node": ">= 0.8" } @@ -3845,31 +3796,6 @@ "browserslist": ">= 4.21.0" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -3892,15 +3818,6 @@ "node": ">=10.13.0" } }, - "node_modules/wbuf": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", - "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", - "dev": true, - "dependencies": { - "minimalistic-assert": "^1.0.0" - } - }, "node_modules/webpack": { "version": "5.105.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", @@ -4004,28 +3921,25 @@ } }, "node_modules/webpack-dev-middleware": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-7.4.2.tgz", - "integrity": "sha512-xOO8n6eggxnwYpy1NlzUKpvrjfJTvae5/D6WOK0S2LSo7vjmo5gCM1DbLUmFqrMTJP+W/0YZNctm7jasWvLuBA==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-8.0.4.tgz", + "integrity": "sha512-9dFzIvIfbdnkOlRjXDHEmEKlY/KPsELNIyKWdoNfK4WaHN9Db+JyVG0gi4/APUPX2UVhnCZ6jp7x0EyM7yTq1Q==", "dev": true, - "license": "MIT", "dependencies": { - "colorette": "^2.0.10", - "memfs": "^4.6.0", - "mime-types": "^2.1.31", - "on-finished": "^2.4.1", + "memfs": "^4.56.10", + "mime-types": "^3.0.2", "range-parser": "^1.2.1", - "schema-utils": "^4.0.0" + "schema-utils": "^4.3.3" }, "engines": { - "node": ">= 18.12.0" + "node": ">= 20.9.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/webpack" }, "peerDependencies": { - "webpack": "^5.0.0" + "webpack": "^5.101.0" }, "peerDependenciesMeta": { "webpack": { @@ -4033,53 +3947,75 @@ } } }, + "node_modules/webpack-dev-middleware/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/webpack-dev-middleware/node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/webpack-dev-server": { - "version": "5.2.6", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.6.tgz", - "integrity": "sha512-HNLRmamRvVavZQ+avceZifmv8hmdUjg43t6MI4SqJDwFdW7RPQwH5vzGhDRZSX59SgfbeHhLnq3g+uooWo7pVw==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-6.0.0.tgz", + "integrity": "sha512-q9SD4ItOGhZLeU6EGT10caDZdHjF50Pz1DtkRZZOPsfluMXOkacWKKOtSBSLVkPqKiF67eFUC0rI88U/tSFPEw==", "dev": true, "dependencies": { "@types/bonjour": "^3.5.13", "@types/connect-history-api-fallback": "^1.5.4", - "@types/express": "^4.17.25", - "@types/express-serve-static-core": "^4.17.21", + "@types/express": "^5.0.6", + "@types/express-serve-static-core": "^5.1.1", "@types/serve-index": "^1.9.4", - "@types/serve-static": "^1.15.5", - "@types/sockjs": "^0.3.36", - "@types/ws": "^8.5.10", + "@types/serve-static": "^2.2.0", + "@types/ws": "^8.18.1", "ansi-html-community": "^0.0.8", - "bonjour-service": "^1.2.1", - "chokidar": "^3.6.0", - "colorette": "^2.0.10", + "bonjour-service": "^1.3.0", + "chokidar": "^5.0.0", "compression": "^1.8.1", "connect-history-api-fallback": "^2.0.0", - "express": "^4.22.1", - "graceful-fs": "^4.2.6", - "http-proxy-middleware": "^2.0.9", - "ipaddr.js": "^2.1.0", + "express": "^5.2.1", + "graceful-fs": "^4.2.11", + "http-proxy-middleware": "^4.1.1", + "ipaddr.js": "^2.3.0", "launch-editor": "^2.14.1", - "open": "^10.0.3", - "p-retry": "^6.2.0", - "schema-utils": "^4.2.0", + "open": "^11.0.0", + "p-retry": "^8.0.0", + "schema-utils": "^4.3.3", "selfsigned": "^5.5.0", - "serve-index": "^1.9.1", - "sockjs": "^0.3.24", - "spdy": "^4.0.2", - "webpack-dev-middleware": "^7.4.2", - "ws": "^8.18.0" + "serve-index": "^1.9.2", + "tinyglobby": "^0.2.15", + "webpack-dev-middleware": "^8.0.3", + "ws": "^8.20.0" }, "bin": { "webpack-dev-server": "bin/webpack-dev-server.js" }, "engines": { - "node": ">= 18.12.0" + "node": ">= 22.15.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/webpack" }, "peerDependencies": { - "webpack": "^5.0.0" + "webpack": "^5.101.0" }, "peerDependenciesMeta": { "webpack": { @@ -4112,29 +4048,6 @@ "node": ">=10.13.0" } }, - "node_modules/websocket-driver": { - "version": "0.7.5", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.5.tgz", - "integrity": "sha512-ZL2+3c7kMBdIRCMz6l8jQMHyGVxj+UL+xVk74Ombiciboca8rHa15L86B19E5oh1pL9Ii/uj54gtsIrZGMo6zA==", - "dev": true, - "dependencies": { - "http-parser-js": ">=0.5.1", - "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4156,6 +4069,12 @@ "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, "node_modules/ws": { "version": "8.21.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", @@ -4176,6 +4095,22 @@ "optional": true } } + }, + "node_modules/wsl-utils": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.3.1.tgz", + "integrity": "sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==", + "dev": true, + "dependencies": { + "is-wsl": "^3.1.0", + "powershell-utils": "^0.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } }, "dependencies": { @@ -4554,21 +4489,20 @@ "dev": true }, "@types/express": { - "version": "4.17.25", - "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", - "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "requires": { "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "^1" + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" } }, "@types/express-serve-static-core": { - "version": "4.17.36", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.36.tgz", - "integrity": "sha512-zbivROJ0ZqLAtMzgzIUC4oNqDG9iF0lSsAqpOD9kbs5xcIM3dTiyuHvBc7R8MtWBp3AAWGaovJa+wzWPjLYW7Q==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.2.tgz", + "integrity": "sha512-d3KvEXBSo/lOAMc2u6fkyDHBvetBHeqD7wm/AcXfLpSOQwlmG9D/aQ0SFswVjv05p7ullQS7Mjohj6/VdbZuTg==", "dev": true, "requires": { "@types/node": "*", @@ -4578,20 +4512,11 @@ } }, "@types/http-errors": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.2.tgz", - "integrity": "sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", "dev": true }, - "@types/http-proxy": { - "version": "1.17.12", - "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.12.tgz", - "integrity": "sha512-kQtujO08dVtQ2wXAuSFfk9ASy3sug4+ogFR8Kd8UgP8PEuc1/G/8yjYRmp//PcDNJEUKOza/MrQu15bouEUCiw==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -4622,12 +4547,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, - "@types/retry": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", - "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", - "dev": true - }, "@types/send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.1.tgz", @@ -4648,22 +4567,12 @@ } }, "@types/serve-static": { - "version": "1.15.7", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", - "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", "dev": true, "requires": { "@types/http-errors": "*", - "@types/node": "*", - "@types/send": "*" - } - }, - "@types/sockjs": { - "version": "0.3.36", - "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.36.tgz", - "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", - "dev": true, - "requires": { "@types/node": "*" } }, @@ -4856,13 +4765,30 @@ "dev": true }, "accepts": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "dev": true, "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "dependencies": { + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + }, + "mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "requires": { + "mime-db": "^1.54.0" + } + } } }, "acorn": { @@ -4914,22 +4840,6 @@ "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", "dev": true }, - "anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "requires": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - } - }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", - "dev": true - }, "asn1js": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.10.tgz", @@ -4958,73 +4868,30 @@ "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", - "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=", - "dev": true - }, - "binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "dev": true }, "body-parser": { - "version": "1.20.5", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", - "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", + "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", "dev": true, "requires": { - "bytes": "~3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "~1.2.0", - "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", - "on-finished": "~2.4.1", - "qs": "~6.15.1", - "raw-body": "~2.5.3", - "type-is": "~1.6.18", - "unpipe": "~1.0.0" + "bytes": "^3.1.2", + "content-type": "^2.0.0", + "debug": "^4.4.3", + "http-errors": "^2.0.1", + "iconv-lite": "^0.7.2", + "on-finished": "^2.4.1", + "qs": "^6.15.2", + "raw-body": "^3.0.2", + "type-is": "^2.1.0" }, "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "depd": { + "content-type": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", "dev": true } } @@ -5115,19 +4982,12 @@ "dev": true }, "chokidar": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "dev": true, "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" + "readdirp": "^5.0.0" } }, "chrome-trace-event": { @@ -5216,21 +5076,10 @@ "dev": true }, "content-disposition": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", - "dev": true, - "requires": { - "safe-buffer": "5.2.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - } - } + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "dev": true }, "content-type": { "version": "1.0.5", @@ -5239,15 +5088,15 @@ "dev": true }, "cookie": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", - "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", "dev": true }, "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", "dev": true }, "copy-webpack-plugin": { @@ -5280,12 +5129,6 @@ } } }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true - }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -5301,26 +5144,26 @@ "version": "file:../pkg" }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "^2.1.3" }, "dependencies": { "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true } } }, "default-browser": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz", - "integrity": "sha512-WY/3TUME0x3KPYdRRxEJJvXRHV4PyPoUsxtZa78lwItwRQRHhd2U9xOscaT/YTf8uCXIAjeJOFBVEh/7FtD8Xg==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", + "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", "dev": true, "requires": { "bundle-name": "^4.1.0", @@ -5328,9 +5171,9 @@ } }, "default-browser-id": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.0.tgz", - "integrity": "sha512-A6p/pu/6fyBcA1TRz/GqWYPViplrftcW2gZC9q79ngNCKAeR/X3gcEdXQHl4KNXV+3wgIJ1CPkJQ3IHM6lcsyA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", "dev": true }, "define-lazy-prop": { @@ -5340,21 +5183,9 @@ "dev": true }, "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=", - "dev": true - }, - "destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", - "dev": true - }, - "detect-node": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.0.4.tgz", - "integrity": "sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true }, "dns-packet": { @@ -5430,9 +5261,9 @@ "dev": true }, "es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", "dev": true, "requires": { "es-errors": "^1.3.0" @@ -5447,7 +5278,7 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "dev": true }, "eslint-scope": { @@ -5489,12 +5320,6 @@ "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", "dev": true }, - "eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true - }, "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -5502,70 +5327,55 @@ "dev": true }, "express": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", - "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", - "dev": true, - "requires": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "~1.20.5", - "content-disposition": "~0.5.4", - "content-type": "~1.0.4", - "cookie": "~0.7.1", - "cookie-signature": "~1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "~1.3.1", - "fresh": "~0.5.2", - "http-errors": "~2.0.0", - "merge-descriptors": "1.0.3", - "methods": "~1.1.2", - "on-finished": "~2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "~0.1.12", - "proxy-addr": "~2.0.7", - "qs": "~6.15.1", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "~0.19.0", - "serve-static": "~1.16.2", - "setprototypeof": "1.2.0", - "statuses": "~2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "dev": true, + "requires": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "dependencies": { + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + }, + "mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "dev": true, "requires": { - "ms": "2.0.0" + "mime-db": "^1.54.0" } - }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - }, - "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true } } }, @@ -5587,15 +5397,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true }, - "faye-websocket": { - "version": "0.11.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.4.tgz", - "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", - "dev": true, - "requires": { - "websocket-driver": ">=0.5.1" - } - }, "fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -5606,35 +5407,17 @@ } }, "finalhandler": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", - "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", "dev": true, "requires": { - "debug": "2.6.9", - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true - } + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" } }, "find-up": { @@ -5647,12 +5430,6 @@ "path-exists": "^4.0.0" } }, - "follow-redirects": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", - "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", - "dev": true - }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -5660,18 +5437,11 @@ "dev": true }, "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", "dev": true }, - "fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "optional": true - }, "function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -5706,15 +5476,6 @@ "es-object-atoms": "^1.0.0" } }, - "glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - }, "glob-to-regexp": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", @@ -5733,12 +5494,6 @@ "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", "dev": true }, - "handle-thing": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", - "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", - "dev": true - }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -5761,95 +5516,46 @@ "dev": true }, "hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", "dev": true, "requires": { "function-bind": "^1.1.2" } }, - "hpack.js": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", - "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=", - "dev": true, - "requires": { - "inherits": "^2.0.1", - "obuf": "^1.0.0", - "readable-stream": "^2.0.1", - "wbuf": "^1.1.0" - } - }, - "http-deceiver": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", - "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc=", - "dev": true - }, "http-errors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", - "dev": true, - "requires": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - }, - "dependencies": { - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true - } - } - }, - "http-parser-js": { - "version": "0.5.8", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz", - "integrity": "sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==", - "dev": true - }, - "http-proxy": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", - "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", "dev": true, "requires": { - "eventemitter3": "^4.0.0", - "follow-redirects": "^1.0.0", - "requires-port": "^1.0.0" + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" } }, "http-proxy-middleware": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.10.tgz", - "integrity": "sha512-RKzRWNPxUZqbuk3BC5mGVJbBnWgr+diEnjJexIOytFbBzDy88Fbh/YvBr3DsNrl1jYAfjWfpATEv0NO35FDuPQ==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-4.2.0.tgz", + "integrity": "sha512-ZA+oNOoM+GLoFTIzhkJptVQov73Srep2LBqhF8hG8CIPKO3nam1jonXVQ/QUH8RbwsmaaVz2SOJdzBNBHNtKbw==", "dev": true, "requires": { - "@types/http-proxy": "^1.17.8", - "http-proxy": "^1.18.1", - "is-glob": "^4.0.1", - "is-plain-obj": "^3.0.0", - "micromatch": "^4.0.2" + "debug": "^4.4.3", + "httpxy": "^0.5.4", + "is-glob": "^4.0.3", + "is-plain-obj": "^4.1.0", + "micromatch": "^4.0.8" } }, + "httpxy": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/httpxy/-/httpxy-0.5.5.tgz", + "integrity": "sha512-uDjmnPyp1q4Sgzf3w+J/Fc6UqcCEj0x4Wjp7OqK5dGhNeDgpyrAmnS6ey8QWrX3SWDon2DMKf9sBa5X9+CVyMA==", + "dev": true + }, "hyperdyperid": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", @@ -5857,12 +5563,12 @@ "dev": true }, "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", "dev": true, "requires": { - "safer-buffer": ">= 2.1.2 < 3" + "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "import-local": { @@ -5876,9 +5582,9 @@ } }, "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, "interpret": { @@ -5888,20 +5594,11 @@ "dev": true }, "ipaddr.js": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", - "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", + "integrity": "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==", "dev": true }, - "is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "requires": { - "binary-extensions": "^2.0.0" - } - }, "is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", @@ -5932,6 +5629,12 @@ "is-extglob": "^2.1.1" } }, + "is-in-ssh": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", + "integrity": "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw==", + "dev": true + }, "is-inside-container": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", @@ -5942,9 +5645,9 @@ } }, "is-network-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", - "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", "dev": true }, "is-number": { @@ -5954,9 +5657,9 @@ "dev": true }, "is-plain-obj": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", - "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", "dev": true }, "is-plain-object": { @@ -5968,21 +5671,21 @@ "isobject": "^3.0.1" } }, + "is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true + }, "is-wsl": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.0.tgz", - "integrity": "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", "dev": true, "requires": { "is-inside-container": "^1.0.0" } }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true - }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -6056,20 +5759,30 @@ "dev": true }, "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", + "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", "dev": true }, "memfs": { - "version": "4.17.2", - "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.17.2.tgz", - "integrity": "sha512-NgYhCOWgovOXSzvYgUW0LQ7Qy72rWQMGGFJDoWg4G30RHd3z77VbYdtJ4fembJXBy8pMIUA31XNAupobOQlwdg==", - "dev": true, - "requires": { - "@jsonjoy.com/json-pack": "^1.0.3", - "@jsonjoy.com/util": "^1.3.0", - "tree-dump": "^1.0.1", + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.64.0.tgz", + "integrity": "sha512-Kw72fgY7Wn+sD8KmtNWSafl1dz0UvAsE/PHs3YVfLiaZuA3HxNm9sRLqAu0ATiBGJvME1PxZXbBZPv5GycDeAw==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-to-fsa": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "@jsonjoy.com/json-pack": "^1.11.0", + "@jsonjoy.com/util": "^1.9.0", + "glob-to-regex.js": "^1.0.1", + "thingies": "^2.5.0", + "tree-dump": "^1.0.3", "tslib": "^2.0.0" }, "dependencies": { @@ -6080,36 +5793,231 @@ "dev": true, "requires": {} }, + "@jsonjoy.com/buffers": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-17.67.0.tgz", + "integrity": "sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==", + "dev": true, + "requires": {} + }, + "@jsonjoy.com/codegen": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-1.0.0.tgz", + "integrity": "sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==", + "dev": true, + "requires": {} + }, + "@jsonjoy.com/fs-core": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-core/-/fs-core-4.64.0.tgz", + "integrity": "sha512-zs2TAq7Six5jgMuoMNjpspAvOP3mhtgq/k1UyQodEzCtQi/N83y2/y+zcvnZSGp/Rxq96DBN+bValOBQAyn/ew==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + } + }, + "@jsonjoy.com/fs-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-fsa/-/fs-fsa-4.64.0.tgz", + "integrity": "sha512-nMWOVbkLFyEgmXZih3wyvxA9XpgyyqyfrINMHvEFqhi7uqfRl7c9ERJt6yX7vgMPrB9Uo+OJO+Spa0cFzPD01w==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + } + }, + "@jsonjoy.com/fs-node": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node/-/fs-node-4.64.0.tgz", + "integrity": "sha512-dO+NNkODbUli4uV42bcNrrLvq5rE7SNpdZ5TNd0dtbLsAaNK3MDiIC9lUi+brboGoIjW6vd2fB1qao60nrk5xA==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "glob-to-regex.js": "^1.0.0", + "thingies": "^2.5.0" + } + }, + "@jsonjoy.com/fs-node-builtins": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-builtins/-/fs-node-builtins-4.64.0.tgz", + "integrity": "sha512-/o7WRFhUWaM/fOrslwLZGnzn4RmRILykn+lAL+mNObqqRNw+CQSiij6hpCeZ+C7buhdoVo7go/OYqzaSUfDYmA==", + "dev": true, + "requires": {} + }, + "@jsonjoy.com/fs-node-to-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-to-fsa/-/fs-node-to-fsa-4.64.0.tgz", + "integrity": "sha512-WDD9WVs0hb7UAEKTgZW2f66WDrbj7gIIWwpP3spbLyXa0rghtUaFTB8L4gdR3ZCWwiKIsj38/CNijpVmpnuPUw==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0" + } + }, + "@jsonjoy.com/fs-node-utils": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-utils/-/fs-node-utils-4.64.0.tgz", + "integrity": "sha512-k5Indsx9hWW9xSF7Y6oSKKwtCUNhzZxadub3owhIlitc+iMRVlPPdX2duTKQWBL3qNWpXya8jykgaaWpheeS4w==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "glob-to-regex.js": "^1.0.1" + } + }, + "@jsonjoy.com/fs-print": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-print/-/fs-print-4.64.0.tgz", + "integrity": "sha512-PHZFccchvkhWrwPWHjmVAhbC3vSHCtyZvlZfJJ3ho2bnzl450hXri6/8e6pbkWdH+SkmLXNml0sV8e5HDAfxKw==", + "dev": true, + "requires": { + "@jsonjoy.com/fs-node-utils": "4.64.0", + "tree-dump": "^1.1.0" + } + }, + "@jsonjoy.com/fs-snapshot": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-snapshot/-/fs-snapshot-4.64.0.tgz", + "integrity": "sha512-oM7UDeL83q6NBzzsfKAsYKXKVXlykKFqqOLh4xZZKAzzROTlInkPbc6LTDGThEOnPiFiUzA7tYziHG9xavd76Q==", + "dev": true, + "requires": { + "@jsonjoy.com/buffers": "^17.65.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/json-pack": "^17.65.0", + "@jsonjoy.com/util": "^17.65.0" + }, + "dependencies": { + "@jsonjoy.com/base64": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-17.67.0.tgz", + "integrity": "sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==", + "dev": true, + "requires": {} + }, + "@jsonjoy.com/codegen": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-17.67.0.tgz", + "integrity": "sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==", + "dev": true, + "requires": {} + }, + "@jsonjoy.com/json-pack": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-17.67.0.tgz", + "integrity": "sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==", + "dev": true, + "requires": { + "@jsonjoy.com/base64": "17.67.0", + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0", + "@jsonjoy.com/json-pointer": "17.67.0", + "@jsonjoy.com/util": "17.67.0", + "hyperdyperid": "^1.2.0", + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + } + }, + "@jsonjoy.com/json-pointer": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-17.67.0.tgz", + "integrity": "sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==", + "dev": true, + "requires": { + "@jsonjoy.com/util": "17.67.0" + } + }, + "@jsonjoy.com/util": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-17.67.0.tgz", + "integrity": "sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==", + "dev": true, + "requires": { + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0" + } + } + } + }, "@jsonjoy.com/json-pack": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.2.0.tgz", - "integrity": "sha512-io1zEbbYcElht3tdlqEOFxZ0dMTYrHz9iMf0gqn1pPjZFTCgM5R4R5IMA20Chb2UPYYsxjzs8CgZ7Nb5n2K2rA==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.21.0.tgz", + "integrity": "sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==", "dev": true, "requires": { - "@jsonjoy.com/base64": "^1.1.1", - "@jsonjoy.com/util": "^1.1.2", + "@jsonjoy.com/base64": "^1.1.2", + "@jsonjoy.com/buffers": "^1.2.0", + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/json-pointer": "^1.0.2", + "@jsonjoy.com/util": "^1.9.0", "hyperdyperid": "^1.2.0", - "thingies": "^1.20.0" + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + }, + "dependencies": { + "@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "dev": true, + "requires": {} + } + } + }, + "@jsonjoy.com/json-pointer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-1.0.2.tgz", + "integrity": "sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==", + "dev": true, + "requires": { + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/util": "^1.9.0" } }, "@jsonjoy.com/util": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.6.0.tgz", - "integrity": "sha512-sw/RMbehRhN68WRtcKCpQOPfnH6lLP4GJfqzi3iYej8tnzpZUDr6UkZYJjcjjC0FWEJOJbyM3PTIwxucUmDG2A==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.9.0.tgz", + "integrity": "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==", + "dev": true, + "requires": { + "@jsonjoy.com/buffers": "^1.0.0", + "@jsonjoy.com/codegen": "^1.0.0" + }, + "dependencies": { + "@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "dev": true, + "requires": {} + } + } + }, + "glob-to-regex.js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/glob-to-regex.js/-/glob-to-regex.js-1.2.0.tgz", + "integrity": "sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ==", "dev": true, "requires": {} }, "thingies": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/thingies/-/thingies-1.21.0.tgz", - "integrity": "sha512-hsqsJsFMsV+aD4s3CWKk85ep/3I9XzYV/IXaSouJMYIoDlgyi11cBhsqYe9/geRfB0YIikBQg6raRaM+nIMP9g==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/thingies/-/thingies-2.6.0.tgz", + "integrity": "sha512-rMHRjmlFLM1R96UYPvpmnc3LYtdFrT33JIB7L9hetGue1qAPfn1N2LJeEjxUSidu1Iku+haLZXDuEXUHNGO/lg==", "dev": true, "requires": {} }, "tree-dump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.0.3.tgz", - "integrity": "sha512-il+Cv80yVHFBwokQSfd4bldvr1Md951DpgAGfmhydt04L+YzHgubm2tQ7zueWDcGENKHq0ZvGFR/hjvNXilHEg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.1.0.tgz", + "integrity": "sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==", "dev": true, "requires": {} }, @@ -6122,9 +6030,9 @@ } }, "merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", "dev": true }, "merge-stream": { @@ -6133,12 +6041,6 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "dev": true - }, "micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", @@ -6149,12 +6051,6 @@ "picomatch": "^2.3.1" } }, - "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "dev": true - }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -6170,12 +6066,6 @@ "mime-db": "1.52.0" } }, - "minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", - "dev": true - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -6193,9 +6083,9 @@ } }, "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", "dev": true }, "neo-async": { @@ -6222,12 +6112,6 @@ "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", "dev": true }, - "obuf": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", - "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", - "dev": true - }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -6243,16 +6127,27 @@ "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", "dev": true }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, "open": { - "version": "10.1.2", - "resolved": "https://registry.npmjs.org/open/-/open-10.1.2.tgz", - "integrity": "sha512-cxN6aIDPz6rm8hbebcP7vrQNhvRcveZoJU72Y7vskh4oIm+BZwBECnx5nTmrlres1Qapvx27Qo1Auukpf8PKXw==", + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/open/-/open-11.0.0.tgz", + "integrity": "sha512-smsWv2LzFjP03xmvFoJ331ss6h+jixfA4UUV/Bsiyuu4YJPfN+FIQGOIiv4w9/+MoHkfkJ22UIaQWRVFRfH6Vw==", "dev": true, "requires": { - "default-browser": "^5.2.1", + "default-browser": "^5.4.0", "define-lazy-prop": "^3.0.0", + "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", - "is-wsl": "^3.1.0" + "powershell-utils": "^0.1.0", + "wsl-utils": "^0.3.0" } }, "p-locate": { @@ -6276,14 +6171,12 @@ } }, "p-retry": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", - "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-8.0.0.tgz", + "integrity": "sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A==", "dev": true, "requires": { - "@types/retry": "0.12.2", - "is-network-error": "^1.0.0", - "retry": "^0.13.1" + "is-network-error": "^1.3.0" } }, "p-try": { @@ -6317,9 +6210,9 @@ "dev": true }, "path-to-regexp": { - "version": "0.1.12", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", - "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "dev": true }, "picocolors": { @@ -6365,10 +6258,10 @@ } } }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "powershell-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.1.0.tgz", + "integrity": "sha512-dM0jVuXJPsDN6DvRpea484tCUaMiXWjuCn++HGTqUWzGDjv5tZkEZldAJ/UMlqRYGFrD/etByo4/xOuC/snX2A==", "dev": true }, "proxy-addr": { @@ -6413,88 +6306,38 @@ "dev": true }, "qs": { - "version": "6.15.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", - "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "version": "6.15.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", + "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", "dev": true, "requires": { - "side-channel": "^1.1.0" + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" } }, "range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", "dev": true }, "raw-body": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", - "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", "dev": true, "requires": { "bytes": "~3.1.2", "http-errors": "~2.0.1", - "iconv-lite": "~0.4.24", + "iconv-lite": "~0.7.0", "unpipe": "~1.0.0" - }, - "dependencies": { - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "requires": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true - }, - "statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true - } - } - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "dev": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" } }, "readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.0.0.tgz", + "integrity": "sha512-9u/XQ1pvrQtYyMpZe7DXKv2p5CNvyVwzUB6uhLAnQwHMSgKMBR62lc7AHljaeteeHXn11XTAaLLUVZYVZyuRBQ==", + "dev": true }, "rechoir": { "version": "0.8.0", @@ -6517,12 +6360,6 @@ "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", "dev": true }, - "requires-port": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", - "dev": true - }, "resolve": { "version": "1.22.6", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.6.tgz", @@ -6549,22 +6386,23 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true }, - "retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "dev": true + "router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "dev": true, + "requires": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + } }, "run-applescript": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", - "integrity": "sha512-9by4Ij99JUr/MCFBUkDKLWK3G9HVXmabKz9U5MlIAIuvuzkiOicRYs8XJLxX+xahD+mLiiCYDqF9dKAgtzKP1A==", - "dev": true - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", "dev": true }, "safer-buffer": { @@ -6585,12 +6423,6 @@ "ajv-keywords": "^5.1.0" } }, - "select-hose": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", - "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=", - "dev": true - }, "selfsigned": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", @@ -6602,84 +6434,72 @@ } }, "send": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", - "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", - "dev": true, - "requires": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "dev": true, + "requires": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" }, "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + }, + "mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", "dev": true, "requires": { - "ms": "2.0.0" - }, - "dependencies": { - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "dev": true - } + "mime-db": "^1.54.0" } }, - "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", - "dev": true - }, "ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true - }, - "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "dev": true } } }, "serve-index": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", - "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.2.tgz", + "integrity": "sha512-KDj11HScOaLmrPxl70KYNW1PksP4Nb/CLL2yvC+Qd2kHMPEEpfc4Re2e4FOay+bC/+XQl/7zAcWON3JVo5v3KQ==", "dev": true, "requires": { - "accepts": "~1.3.4", + "accepts": "~1.3.8", "batch": "0.6.1", "debug": "2.6.9", "escape-html": "~1.0.3", - "http-errors": "~1.6.2", - "mime-types": "~2.1.17", - "parseurl": "~1.3.2" + "http-errors": "~1.8.0", + "mime-types": "~2.1.35", + "parseurl": "~1.3.3" }, "dependencies": { + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -6689,36 +6509,49 @@ "ms": "2.0.0" } }, + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true + }, "http-errors": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", - "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", "dev": true, "requires": { "depd": "~1.1.2", - "inherits": "2.0.3", - "setprototypeof": "1.1.0", - "statuses": ">= 1.4.0 < 2" + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" } }, - "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true + }, + "statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", "dev": true } } }, "serve-static": { - "version": "1.16.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", - "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", "dev": true, "requires": { - "encodeurl": "~2.0.0", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.19.0" + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" } }, "setprototypeof": { @@ -6758,26 +6591,26 @@ "dev": true }, "side-channel": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", - "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", "dev": true, "requires": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3", - "side-channel-list": "^1.0.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "dev": true, "requires": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" } }, "side-channel-map": { @@ -6805,17 +6638,6 @@ "side-channel-map": "^1.0.1" } }, - "sockjs": { - "version": "0.3.24", - "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.24.tgz", - "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", - "dev": true, - "requires": { - "faye-websocket": "^0.11.3", - "uuid": "^8.3.2", - "websocket-driver": "^0.7.4" - } - }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -6832,61 +6654,12 @@ "source-map": "^0.6.0" } }, - "spdy": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.2.tgz", - "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", - "dev": true, - "requires": { - "debug": "^4.1.0", - "handle-thing": "^2.0.0", - "http-deceiver": "^1.2.7", - "select-hose": "^2.0.0", - "spdy-transport": "^3.0.0" - } - }, - "spdy-transport": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", - "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", - "dev": true, - "requires": { - "debug": "^4.1.0", - "detect-node": "^2.0.4", - "hpack.js": "^2.1.6", - "obuf": "^1.1.2", - "readable-stream": "^3.0.6", - "wbuf": "^1.7.3" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", "dev": true }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, "supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -6994,13 +6767,37 @@ } }, "type-is": { - "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", "dev": true, "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "dependencies": { + "content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "dev": true + }, + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + }, + "mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "requires": { + "mime-db": "^1.54.0" + } + } } }, "unpipe": { @@ -7019,24 +6816,6 @@ "picocolors": "^1.1.1" } }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true - }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", - "dev": true - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true - }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -7053,15 +6832,6 @@ "graceful-fs": "^4.1.2" } }, - "wbuf": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz", - "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", - "dev": true, - "requires": { - "minimalistic-assert": "^1.0.0" - } - }, "webpack": { "version": "5.105.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", @@ -7125,53 +6895,65 @@ } }, "webpack-dev-middleware": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-7.4.2.tgz", - "integrity": "sha512-xOO8n6eggxnwYpy1NlzUKpvrjfJTvae5/D6WOK0S2LSo7vjmo5gCM1DbLUmFqrMTJP+W/0YZNctm7jasWvLuBA==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-8.0.4.tgz", + "integrity": "sha512-9dFzIvIfbdnkOlRjXDHEmEKlY/KPsELNIyKWdoNfK4WaHN9Db+JyVG0gi4/APUPX2UVhnCZ6jp7x0EyM7yTq1Q==", "dev": true, "requires": { - "colorette": "^2.0.10", - "memfs": "^4.6.0", - "mime-types": "^2.1.31", - "on-finished": "^2.4.1", + "memfs": "^4.56.10", + "mime-types": "^3.0.2", "range-parser": "^1.2.1", - "schema-utils": "^4.0.0" + "schema-utils": "^4.3.3" + }, + "dependencies": { + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + }, + "mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "requires": { + "mime-db": "^1.54.0" + } + } } }, "webpack-dev-server": { - "version": "5.2.6", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-5.2.6.tgz", - "integrity": "sha512-HNLRmamRvVavZQ+avceZifmv8hmdUjg43t6MI4SqJDwFdW7RPQwH5vzGhDRZSX59SgfbeHhLnq3g+uooWo7pVw==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-6.0.0.tgz", + "integrity": "sha512-q9SD4ItOGhZLeU6EGT10caDZdHjF50Pz1DtkRZZOPsfluMXOkacWKKOtSBSLVkPqKiF67eFUC0rI88U/tSFPEw==", "dev": true, "requires": { "@types/bonjour": "^3.5.13", "@types/connect-history-api-fallback": "^1.5.4", - "@types/express": "^4.17.25", - "@types/express-serve-static-core": "^4.17.21", + "@types/express": "^5.0.6", + "@types/express-serve-static-core": "^5.1.1", "@types/serve-index": "^1.9.4", - "@types/serve-static": "^1.15.5", - "@types/sockjs": "^0.3.36", - "@types/ws": "^8.5.10", + "@types/serve-static": "^2.2.0", + "@types/ws": "^8.18.1", "ansi-html-community": "^0.0.8", - "bonjour-service": "^1.2.1", - "chokidar": "^3.6.0", - "colorette": "^2.0.10", + "bonjour-service": "^1.3.0", + "chokidar": "^5.0.0", "compression": "^1.8.1", "connect-history-api-fallback": "^2.0.0", - "express": "^4.22.1", - "graceful-fs": "^4.2.6", - "http-proxy-middleware": "^2.0.9", - "ipaddr.js": "^2.1.0", + "express": "^5.2.1", + "graceful-fs": "^4.2.11", + "http-proxy-middleware": "^4.1.1", + "ipaddr.js": "^2.3.0", "launch-editor": "^2.14.1", - "open": "^10.0.3", - "p-retry": "^6.2.0", - "schema-utils": "^4.2.0", + "open": "^11.0.0", + "p-retry": "^8.0.0", + "schema-utils": "^4.3.3", "selfsigned": "^5.5.0", - "serve-index": "^1.9.1", - "sockjs": "^0.3.24", - "spdy": "^4.0.2", - "webpack-dev-middleware": "^7.4.2", - "ws": "^8.18.0" + "serve-index": "^1.9.2", + "tinyglobby": "^0.2.15", + "webpack-dev-middleware": "^8.0.3", + "ws": "^8.20.0" } }, "webpack-merge": { @@ -7190,23 +6972,6 @@ "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", "dev": true }, - "websocket-driver": { - "version": "0.7.5", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.5.tgz", - "integrity": "sha512-ZL2+3c7kMBdIRCMz6l8jQMHyGVxj+UL+xVk74Ombiciboca8rHa15L86B19E5oh1pL9Ii/uj54gtsIrZGMo6zA==", - "dev": true, - "requires": { - "http-parser-js": ">=0.5.1", - "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - } - }, - "websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", - "dev": true - }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -7222,12 +6987,28 @@ "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, "ws": { "version": "8.21.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", "dev": true, "requires": {} + }, + "wsl-utils": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.3.1.tgz", + "integrity": "sha512-g/eziiSUNBSsdDJtCLB8bdYEUMj4jR7AGeUo96p/3dTafgjHhpF4RiCFPiRILwjQoDXx5MqkBr4fwWtR3Ky4Wg==", + "dev": true, + "requires": { + "is-wsl": "^3.1.0", + "powershell-utils": "^0.1.0" + } } } } diff --git a/datafusion/wasmtest/datafusion-wasm-app/package.json b/datafusion/wasmtest/datafusion-wasm-app/package.json index 1377df28463bd..e9e98f49495f9 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package.json @@ -29,7 +29,7 @@ "devDependencies": { "webpack": "5.105.0", "webpack-cli": "5.1.4", - "webpack-dev-server": "5.2.6", + "webpack-dev-server": "6.0.0", "copy-webpack-plugin": "14.0.0" } } From 562f87dff5e2ee381cb37e45ce6802ddefdebbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C5=9Een?= Date: Fri, 24 Jul 2026 23:45:35 +0300 Subject: [PATCH 016/109] refactor(proto): migrate HashJoinExec serde (#23853) ## Which issue does this PR close? - Closes #23507. ## Rationale for this change Part of epic #23494. Moves `HashJoinExec` protobuf serialization from central dispatch. ## What changes are included in this PR? Add protobuf serialization and deserialization to the `HashJoinExec` physical plan implementation and deprecate the corresponding central proto methods. The wire format remains unchanged. ## Are these changes tested? Yes, existing HashJoin round-trip tests cover this plan. ## Are there any user-facing changes? Existing methods are deprecated with no immediate API change. --- .../physical-plan/src/joins/hash_join/exec.rs | 191 ++++++++++++++ datafusion/proto/src/physical_plan/mod.rs | 249 ++---------------- 2 files changed, 217 insertions(+), 223 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index c746aba028990..ccdb050d168e0 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1761,6 +1761,197 @@ impl ExecutionPlan for HashJoinExec { .ok() .map(|exec| Arc::new(exec) as _) } + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let left = ctx.encode_child(self.left())?; + let right = ctx.encode_child(self.right())?; + + let on = self + .on() + .iter() + .map(|(l, r)| -> Result { + Ok(protobuf::JoinOn { + left: Some(ctx.encode_expr(l)?), + right: Some(ctx.encode_expr(r)?), + }) + }) + .collect::>>()?; + + let join_type = crate::joins::proto::join_type_to_proto(*self.join_type()); + let null_equality = + crate::joins::proto::null_equality_to_proto(self.null_equality()); + // `PartitionMode` is specific to `HashJoinExec`, so its conversion stays + // inline (by-name on purpose: the enums are numbered differently). + let partition_mode = match self.partition_mode() { + PartitionMode::CollectLeft => protobuf::PartitionMode::CollectLeft, + PartitionMode::Partitioned => protobuf::PartitionMode::Partitioned, + PartitionMode::Auto => protobuf::PartitionMode::Auto, + }; + + let filter = self + .filter() + .map(|f| crate::joins::proto::join_filter_to_proto(f, ctx)) + .transpose()?; + + let dynamic_filter = self + .dynamic_filter_expr() + .map(|df| { + let df_expr: Arc = + Arc::clone(df) as Arc; + ctx.encode_expr(&df_expr) + }) + .transpose()?; + + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::HashJoin(Box::new( + protobuf::HashJoinExecNode { + left: Some(Box::new(left)), + right: Some(Box::new(right)), + on, + join_type: join_type.into(), + partition_mode: partition_mode.into(), + null_equality: null_equality.into(), + filter, + // Proto3 `repeated` cannot distinguish `None` from + // `Some(vec![])`. `Some(vec![])` (reachable via + // `try_embed_projection` for e.g. `SELECT count(1) … JOIN …`) + // changes the output schema, so it is encoded with the + // single-element sentinel `[u32::MAX]` (never a valid column + // index); every other state is sent as-is. See + // `try_from_proto` for the matching decoder. + projection: match self.projection.as_ref() { + None => Vec::new(), + Some(v) if v.is_empty() => vec![u32::MAX], + Some(v) => v.iter().map(|x| *x as u32).collect(), + }, + null_aware: self.null_aware, + dynamic_filter, + }, + )), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl HashJoinExec { + /// Reconstruct a [`HashJoinExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_common::internal_datafusion_err; + use datafusion_proto_models::protobuf; + use std::any::Any; + + let hashjoin = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::HashJoin, + "HashJoinExec", + ); + + let left = + ctx.decode_required_child(hashjoin.left.as_deref(), "HashJoinExec", "left")?; + let right = ctx.decode_required_child( + hashjoin.right.as_deref(), + "HashJoinExec", + "right", + )?; + let left_schema = left.schema(); + let right_schema = right.schema(); + + let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin + .on + .iter() + .map(|col| { + let l = ctx.decode_required_expr( + col.left.as_ref(), + left_schema.as_ref(), + "HashJoinExec", + "on.left", + )?; + let r = ctx.decode_required_expr( + col.right.as_ref(), + right_schema.as_ref(), + "HashJoinExec", + "on.right", + )?; + Ok((l, r)) + }) + .collect::>()?; + + let join_type = crate::joins::proto::join_type_from_proto( + hashjoin.join_type, + "HashJoinExec", + )?; + let null_equality = crate::joins::proto::null_equality_from_proto( + hashjoin.null_equality, + "HashJoinExec", + )?; + // `PartitionMode` is specific to `HashJoinExec`, so its conversion stays + // inline (by-name on purpose: the enums are numbered differently). + let partition_mode = match protobuf::PartitionMode::try_from( + hashjoin.partition_mode, + ) + .map_err(|_| { + internal_datafusion_err!( + "HashJoinExec: unknown PartitionMode {}", + hashjoin.partition_mode + ) + })? { + protobuf::PartitionMode::CollectLeft => PartitionMode::CollectLeft, + protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned, + protobuf::PartitionMode::Auto => PartitionMode::Auto, + }; + + let filter = hashjoin + .filter + .as_ref() + .map(|f| crate::joins::proto::join_filter_from_proto(f, ctx, "HashJoinExec")) + .transpose()?; + + // Preserve the empty-projection sentinel written by `try_to_proto`. + let projection = match hashjoin.projection.as_slice() { + [] => None, + [u32::MAX] => Some(Vec::new()), + indices => Some(indices.iter().map(|i| *i as usize).collect()), + }; + + let mut hash_join = HashJoinExec::try_new( + left, + right, + on, + filter, + &join_type, + projection, + partition_mode, + null_equality, + hashjoin.null_aware, + )?; + + if let Some(dynamic_filter_proto) = &hashjoin.dynamic_filter { + // The dynamic filter is a `DynamicFilterPhysicalExpr` over the probe + // (right) side; decode against the right schema then downcast. + let dynamic_filter_expr = + ctx.decode_expr(dynamic_filter_proto, right_schema.as_ref())?; + let df = (dynamic_filter_expr as Arc) + .downcast::() + .map_err(|_| { + internal_datafusion_err!( + "HashJoinExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr" + ) + })?; + hash_join = hash_join.with_dynamic_filter_expr(df)?; + } + + Ok(Arc::new(hash_join)) + } } /// Determines which sides of a join are "preserved" for filter pushdown. diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index b459368bcb1da..345ab87bf5d5b 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -27,8 +27,7 @@ use datafusion_common::config::CsvOptions; use datafusion_common::display::StringifiedPlan; use datafusion_common::format::ExplainFormat; use datafusion_common::{ - DataFusionError, JoinType, NullEquality, Result, internal_datafusion_err, - internal_err, not_impl_err, + DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err, }; #[cfg(feature = "parquet")] use datafusion_datasource::file::FileSource; @@ -60,7 +59,7 @@ use datafusion_functions_table::generate_series::{ use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr}; use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr; use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; -use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef}; +use datafusion_physical_expr::{LexOrdering, LexRequirement}; use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; use datafusion_physical_plan::aggregates::{ @@ -80,9 +79,8 @@ use datafusion_physical_plan::empty::EmptyExec; use datafusion_physical_plan::explain::ExplainExec; use datafusion_physical_plan::expressions::PhysicalSortExpr; use datafusion_physical_plan::filter::FilterExec; -use datafusion_physical_plan::joins::utils::{ColumnIndex, JoinFilter}; use datafusion_physical_plan::joins::{ - CrossJoinExec, HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec, + CrossJoinExec, HashJoinExec, NestedLoopJoinExec, SortMergeJoinExec, SymmetricHashJoinExec, }; use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; @@ -799,8 +797,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::Aggregate(hash_agg) => { self.try_into_aggregate_physical_plan(hash_agg, ctx, proto_converter) } - PhysicalPlanType::HashJoin(hashjoin) => { - self.try_into_hash_join_physical_plan(hashjoin, ctx, proto_converter) + PhysicalPlanType::HashJoin(_) => { + HashJoinExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::SymmetricHashJoin(_) => { SymmetricHashJoinExec::try_from_proto(self.node(), &decode_ctx) @@ -909,14 +907,6 @@ pub trait PhysicalPlanNodeExt: Sized { ); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_hash_join_exec( - exec, - codec, - proto_converter, - ); - } - if let Some(exec) = plan.downcast_ref::() { return protobuf::PhysicalPlanNode::try_from_aggregate_exec( exec, @@ -1703,137 +1693,27 @@ pub trait PhysicalPlanNodeExt: Sized { Ok(Arc::new(agg)) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `HashJoinExec` deserializes itself via `HashJoinExec::try_from_proto`" + )] fn try_into_hash_join_physical_plan( &self, hashjoin: &protobuf::HashJoinExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let left: Arc = - into_physical_plan(&hashjoin.left, ctx, proto_converter)?; - let right: Arc = - into_physical_plan(&hashjoin.right, ctx, proto_converter)?; - let left_schema = left.schema(); - let right_schema = right.schema(); - let on: Vec<(PhysicalExprRef, PhysicalExprRef)> = hashjoin - .on - .iter() - .map(|col| { - let left = proto_converter.proto_to_physical_expr( - &col.left.clone().unwrap(), - left_schema.as_ref(), - ctx, - )?; - let right = proto_converter.proto_to_physical_expr( - &col.right.clone().unwrap(), - right_schema.as_ref(), - ctx, - )?; - Ok((left, right)) - }) - .collect::>()?; - let join_type = - protobuf::JoinType::try_from(hashjoin.join_type).map_err(|_| { - proto_error(format!( - "Received a HashJoinNode message with unknown JoinType {}", - hashjoin.join_type - )) - })?; - let null_equality = protobuf::NullEquality::try_from(hashjoin.null_equality) - .map_err(|_| { - proto_error(format!( - "Received a HashJoinNode message with unknown NullEquality {}", - hashjoin.null_equality - )) - })?; - let filter = hashjoin - .filter - .as_ref() - .map(|f| { - let schema = f - .schema - .as_ref() - .ok_or_else(|| proto_error("Missing JoinFilter schema"))? - .try_into()?; - - let expression = proto_converter.proto_to_physical_expr( - f.expression.as_ref().ok_or_else(|| { - proto_error("Unexpected empty filter expression") - })?, - &schema, - ctx, - )?; - let column_indices = f.column_indices - .iter() - .map(|i| { - let side = protobuf::JoinSide::try_from(i.side) - .map_err(|_| proto_error(format!( - "Received a HashJoinNode message with JoinSide in Filter {}", - i.side)) - )?; - - Ok(ColumnIndex { - index: i.index as usize, - side: side.into(), - }) - }) - .collect::>>()?; - - Ok(JoinFilter::new(expression, column_indices, Arc::new(schema))) - }) - .map_or(Ok(None), |v: Result| v.map(Some))?; - - let partition_mode = protobuf::PartitionMode::try_from(hashjoin.partition_mode) - .map_err(|_| { - proto_error(format!( - "Received a HashJoinNode message with unknown PartitionMode {}", - hashjoin.partition_mode - )) - })?; - let partition_mode = match partition_mode { - protobuf::PartitionMode::CollectLeft => PartitionMode::CollectLeft, - protobuf::PartitionMode::Partitioned => PartitionMode::Partitioned, - protobuf::PartitionMode::Auto => PartitionMode::Auto, + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::HashJoin(Box::new( + hashjoin.clone(), + ))), }; - // Proto3 `repeated` cannot distinguish `None` from `Some(vec![])`. The latter - // is reachable via `try_embed_projection` for `SELECT count(1) … JOIN …` and - // changes the join's output schema, so the encoder reserves the single-element - // sentinel `[u32::MAX]` (never a valid column index) to mean "explicitly empty"; - // every other state is sent as-is. See `try_from_hash_join_exec`. - let projection = match hashjoin.projection.as_slice() { - [] => None, - [u32::MAX] => Some(Vec::new()), - indices => Some(indices.iter().map(|i| *i as usize).collect()), + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, }; - let mut hash_join = HashJoinExec::try_new( - left, - right, - on, - filter, - &JoinType::from_proto(join_type), - projection, - partition_mode, - NullEquality::from_proto(null_equality), - hashjoin.null_aware, - )?; - - if let Some(dynamic_filter_proto) = &hashjoin.dynamic_filter { - let dynamic_filter_expr = proto_converter.proto_to_physical_expr( - dynamic_filter_proto, - right_schema.as_ref(), - ctx, - )?; - let df = (dynamic_filter_expr as Arc) - .downcast::() - .map_err(|_| { - internal_datafusion_err!( - "HashJoinExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr" - ) - })?; - hash_join = hash_join.with_dynamic_filter_expr(df)?; - } - - Ok(Arc::new(hash_join)) + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + HashJoinExec::try_from_proto(&node, &decode_ctx) } #[deprecated( @@ -2585,99 +2465,22 @@ pub trait PhysicalPlanNodeExt: Sized { .ok_or_else(|| internal_datafusion_err!("LocalLimitExec is not serializable")) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `HashJoinExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_hash_join_exec( exec: &HashJoinExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let left = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.left().to_owned(), - codec, - proto_converter, - )?; - let right = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.right().to_owned(), + let encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - let on: Vec = exec - .on() - .iter() - .map(|tuple| { - let l = proto_converter.physical_expr_to_proto(&tuple.0, codec)?; - let r = proto_converter.physical_expr_to_proto(&tuple.1, codec)?; - Ok::<_, DataFusionError>(protobuf::JoinOn { - left: Some(l), - right: Some(r), - }) - }) - .collect::>()?; - let join_type = protobuf::JoinType::from_proto(exec.join_type().to_owned()); - let null_equality = protobuf::NullEquality::from_proto(exec.null_equality()); - let filter = exec - .filter() - .as_ref() - .map(|f| { - let expression = - proto_converter.physical_expr_to_proto(f.expression(), codec)?; - let column_indices = f - .column_indices() - .iter() - .map(|i| { - let side: protobuf::JoinSide = i.side.to_owned().into(); - protobuf::ColumnIndex { - index: i.index as u32, - side: side.into(), - } - }) - .collect(); - let schema = f.schema().as_ref().try_into()?; - Ok(protobuf::JoinFilter { - expression: Some(expression), - column_indices, - schema: Some(schema), - }) - }) - .map_or(Ok(None), |v: Result| v.map(Some))?; - - let partition_mode = match exec.partition_mode() { - PartitionMode::CollectLeft => protobuf::PartitionMode::CollectLeft, - PartitionMode::Partitioned => protobuf::PartitionMode::Partitioned, - PartitionMode::Auto => protobuf::PartitionMode::Auto, }; - - let dynamic_filter = exec - .dynamic_filter_expr() - .map(|df| { - let df_expr: Arc = - Arc::clone(df) as Arc; - proto_converter.physical_expr_to_proto(&df_expr, codec) - }) - .transpose()?; - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::HashJoin(Box::new( - protobuf::HashJoinExecNode { - left: Some(Box::new(left)), - right: Some(Box::new(right)), - on, - join_type: join_type.into(), - partition_mode: partition_mode.into(), - null_equality: null_equality.into(), - filter, - // Send `Some(vec![])` as `[u32::MAX]` (never a valid index) so the - // wire format can distinguish it from `None` (which stays empty). - // See `try_into_hash_join_physical_plan` for the matching decoder. - projection: match exec.projection.as_ref() { - None => Vec::new(), - Some(v) if v.is_empty() => vec![u32::MAX], - Some(v) => v.iter().map(|x| *x as u32).collect(), - }, - null_aware: exec.null_aware, - dynamic_filter, - }, - ))), - }) + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)? + .ok_or_else(|| internal_datafusion_err!("HashJoinExec is not serializable")) } #[deprecated( From 16471eeb91635fcb874c91249f9b57128a20a746 Mon Sep 17 00:00:00 2001 From: Matthew Patton Date: Fri, 24 Jul 2026 18:10:00 -0400 Subject: [PATCH 017/109] refactor(proto): migrate AsyncFuncExec to self-serializing proto (#23825) Closes #23514. Part of #23494. Migrate `AsyncFuncExec` proto encode/decode into the plan itself via `try_to_proto` / `try_from_proto`, removing its central-arm handling in `physical_plan/mod.rs`. ## Rationale `datafusion-proto` currently downcasts `AsyncFuncExec` in the central encode match and rebuilds it inline on decode. #23495 introduced self-serializing hooks so each plan owns its own wire format. This moves `AsyncFuncExec` onto that pattern. ## Changes - Added `try_to_proto` / `AsyncFuncExec::try_from_proto`, wired into the decode dispatch - Removed the central encode downcast branch - Old helper methods kept as `#[deprecated]` stubs, per existing convention - Wire format unchanged ## Testing Existing `roundtrip_async_func_exec` integration test now exercises the new hooks (old path deleted, so it's the only path left). `cargo fmt` + `cargo clippy --all-targets --features proto -- -D warnings` clean. ## User-facing changes No Co-authored-by: Matthew Patton --- datafusion/physical-plan/src/async_func.rs | 77 ++++++++++++++++++ datafusion/proto/src/physical_plan/mod.rs | 92 +++++++--------------- 2 files changed, 105 insertions(+), 64 deletions(-) diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rs index 5a65c9aedc2f1..e13a5b986aa2c 100644 --- a/datafusion/physical-plan/src/async_func.rs +++ b/datafusion/physical-plan/src/async_func.rs @@ -246,6 +246,83 @@ impl ExecutionPlan for AsyncFuncExec { fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let input = ctx.encode_child(self.input())?; + let async_exprs = + ctx.encode_expressions(self.async_exprs.iter().map(|e| &e.func))?; + let async_expr_names = self + .async_exprs + .iter() + .map(|e| e.name().to_string()) + .collect(); + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc(Box::new( + protobuf::AsyncFuncExecNode { + input: Some(Box::new(input)), + async_exprs, + async_expr_names, + }, + )), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl AsyncFuncExec { + /// Reconstruct an [`AsyncFuncExec`] from its protobuf representation. + /// + /// The exact inverse of [`ExecutionPlan::try_to_proto`]: it takes the whole + /// [`PhysicalPlanNode`] so every plan's `try_from_proto` shares one + /// signature. Child plans and expressions are decoded recursively via the + /// [`ExecutionPlanDecodeCtx`]. + /// + /// [`PhysicalPlanNode`]: datafusion_proto_models::protobuf::PhysicalPlanNode + /// [`ExecutionPlan::try_to_proto`]: crate::ExecutionPlan::try_to_proto + /// [`ExecutionPlanDecodeCtx`]: crate::proto::ExecutionPlanDecodeCtx + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + let async_func = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::AsyncFunc, + "AsyncFuncExec", + ); + let input = ctx.decode_required_child( + async_func.input.as_deref(), + "AsyncFuncExec", + "input", + )?; + let input_schema = input.schema(); + assert_eq_or_internal_err!( + async_func.async_exprs.len(), + async_func.async_expr_names.len(), + "AsyncFuncExecNode async_exprs length does not match async_expr_names" + ); + let async_exprs = async_func + .async_exprs + .iter() + .zip(async_func.async_expr_names.iter()) + .map(|(expr, name)| { + let physical_expr = ctx.decode_expr(expr, input_schema.as_ref())?; + Ok(Arc::new(AsyncFuncExpr::try_new( + name.clone(), + physical_expr, + input_schema.as_ref(), + )?)) + }) + .collect::>>()?; + Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?)) + } } struct CoalesceInputStream { diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 345ab87bf5d5b..5cb9a922a8826 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -57,7 +57,6 @@ use datafusion_functions_table::generate_series::{ Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue, }; use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr}; -use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr; use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use datafusion_physical_expr::{LexOrdering, LexRequirement}; use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; @@ -855,8 +854,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::SortMergeJoin(_) => { SortMergeJoinExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::AsyncFunc(async_func) => { - self.try_into_async_func_physical_plan(async_func, ctx, proto_converter) + PhysicalPlanType::AsyncFunc(_) => { + AsyncFuncExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::Buffer(_) => { BufferExec::try_from_proto(self.node(), &decode_ctx) @@ -958,14 +957,6 @@ pub trait PhysicalPlanNodeExt: Sized { return Ok(node); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_async_func_exec( - exec, - codec, - proto_converter, - ); - } - if let Some(exec) = plan.downcast_ref::() { return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec( exec, @@ -2244,41 +2235,27 @@ pub trait PhysicalPlanNodeExt: Sized { CooperativeExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AsyncFuncExec` deserializes itself via `AsyncFuncExec::try_from_proto`" + )] fn try_into_async_func_physical_plan( &self, async_func: &protobuf::AsyncFuncExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let input: Arc = - into_physical_plan(&async_func.input, ctx, proto_converter)?; - - if async_func.async_exprs.len() != async_func.async_expr_names.len() { - return internal_err!( - "AsyncFuncExecNode async_exprs length does not match async_expr_names" - ); - } - - let async_exprs = async_func - .async_exprs - .iter() - .zip(async_func.async_expr_names.iter()) - .map(|(expr, name)| { - let physical_expr = proto_converter.proto_to_physical_expr( - expr, - input.schema().as_ref(), - ctx, - )?; - - Ok(Arc::new(AsyncFuncExpr::try_new( - name.clone(), - physical_expr, - input.schema().as_ref(), - )?)) - }) - .collect::>>()?; - - Ok(Arc::new(AsyncFuncExec::try_new(async_exprs, input)?)) + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::AsyncFunc(Box::new( + async_func.clone(), + ))), + }; + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + AsyncFuncExec::try_from_proto(&node, &decode_ctx) } #[deprecated( @@ -3333,35 +3310,22 @@ pub trait PhysicalPlanNodeExt: Sized { Ok(None) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AsyncFuncExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_async_func_exec( exec: &AsyncFuncExec, - codec: &dyn PhysicalExtensionCodec, + extension_codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - Arc::clone(exec.input()), - codec, + let encoder = ConverterPlanEncoder { + codec: extension_codec, proto_converter, - )?; - - let mut async_exprs = vec![]; - let mut async_expr_names = vec![]; - - for async_expr in exec.async_exprs() { - async_exprs - .push(proto_converter.physical_expr_to_proto(&async_expr.func, codec)?); - async_expr_names.push(async_expr.name.clone()) - } - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::AsyncFunc(Box::new( - protobuf::AsyncFuncExecNode { - input: Some(Box::new(input)), - async_exprs, - async_expr_names, - }, - ))), - }) + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)? + .ok_or_else(|| internal_datafusion_err!("AsyncFuncExec is not serializable")) } #[deprecated( From 9808e83a9c3d3c7b2ef297e41b4532ee30cb8860 Mon Sep 17 00:00:00 2001 From: Phoenix Date: Sat, 25 Jul 2026 10:18:33 +0800 Subject: [PATCH 018/109] refactor(proto): migrate window serde (#23780) ## Which issue does this PR close? - Closes #23513. ## Rationale for this change Window plans still relied on centralized protobuf dispatch, keeping serialization separate from the execution plans that own their state. Both window executors share one protobuf variant. Decoding must inspect `input_order_mode` before selecting the concrete plan. Add plan-local encoders for both executors and a decoder for the shared node. Keep the legacy helpers as deprecated delegates while preserving window frames and UDF payloads on the existing wire format. ## What changes are included in this PR? ## Are these changes tested? Yes ## Are there any user-facing changes? --------- Signed-off-by: Jiawei Zhao --- .../src/windows/bounded_window_agg_exec.rs | 49 +++ datafusion/physical-plan/src/windows/mod.rs | 2 + datafusion/physical-plan/src/windows/proto.rs | 288 ++++++++++++++++++ .../src/windows/window_agg_exec.rs | 102 +++++++ datafusion/proto/src/physical_plan/mod.rs | 177 +++-------- 5 files changed, 480 insertions(+), 138 deletions(-) create mode 100644 datafusion/physical-plan/src/windows/proto.rs diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs index 3ca612bbdb775..d5863080895f6 100644 --- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs @@ -401,6 +401,55 @@ impl ExecutionPlan for BoundedWindowAggExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use super::proto::encode_physical_window_expr; + use datafusion_proto_common::protobuf_common::EmptyMessage; + use datafusion_proto_models::protobuf; + use protobuf::window_agg_exec_node::InputOrderMode as ProtoInputOrderMode; + + let input = ctx.encode_child(self.input())?; + let window_expr = self + .window_expr() + .iter() + .map(|expr| encode_physical_window_expr(expr, ctx)) + .collect::>>()?; + let partition_keys = self + .partition_keys() + .iter() + .map(|expr| ctx.encode_expr(expr)) + .collect::>>()?; + // A `Some(input_order_mode)` is what tells the shared `Window` decode + // arm to rebuild a `BoundedWindowAggExec` rather than a `WindowAggExec`. + let input_order_mode = match &self.input_order_mode { + InputOrderMode::Linear => ProtoInputOrderMode::Linear(EmptyMessage {}), + InputOrderMode::PartiallySorted(columns) => { + ProtoInputOrderMode::PartiallySorted( + protobuf::PartiallySortedInputOrderMode { + columns: columns.iter().map(|column| *column as u64).collect(), + }, + ) + } + InputOrderMode::Sorted => ProtoInputOrderMode::Sorted(EmptyMessage {}), + }; + + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Window(Box::new( + protobuf::WindowAggExecNode { + input: Some(Box::new(input)), + window_expr, + partition_keys, + input_order_mode: Some(input_order_mode), + }, + )), + ), + })) + } } /// Trait that specifies how we search for (or calculate) partitions. It has two diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index b72a65cf996be..baa6abd839175 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -18,6 +18,8 @@ //! Physical expressions for window functions mod bounded_window_agg_exec; +#[cfg(feature = "proto")] +mod proto; mod utils; mod window_agg_exec; diff --git a/datafusion/physical-plan/src/windows/proto.rs b/datafusion/physical-plan/src/windows/proto.rs new file mode 100644 index 0000000000000..aa62158d18fa0 --- /dev/null +++ b/datafusion/physical-plan/src/windows/proto.rs @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Protobuf conversions shared by window execution plans. + +use std::sync::Arc; + +use arrow::compute::SortOptions; +use arrow::datatypes::Schema; +use datafusion_common::{ + Result, ScalarValue, internal_datafusion_err, internal_err, not_impl_err, +}; +use datafusion_expr::{ + WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, +}; +use datafusion_physical_expr::window::SlidingAggregateWindowExpr; +use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; +use datafusion_proto_common::protobuf_common; +use datafusion_proto_models::protobuf::{self, physical_window_expr_node}; + +use super::{ + PlainAggregateWindowExpr, StandardWindowExpr, WindowExpr, WindowUDFExpr, + create_window_expr, schema_add_window_field, +}; + +pub(super) fn encode_physical_window_expr( + window_expr: &Arc, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, +) -> Result { + let expr = window_expr.as_any(); + let mut args = window_expr.expressions().to_vec(); + let window_frame = window_expr.get_window_frame(); + let (window_function, fun_definition, ignore_nulls, distinct) = + if let Some(plain) = expr.downcast_ref::() { + let aggregate_expr = plain.get_aggregate_expr(); + ( + physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( + aggregate_expr.fun().name().to_string(), + ), + ctx.encode_udaf(aggregate_expr.fun())?, + aggregate_expr.ignore_nulls(), + aggregate_expr.is_distinct(), + ) + } else if let Some(sliding) = expr.downcast_ref::() { + let aggregate_expr = sliding.get_aggregate_expr(); + ( + physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( + aggregate_expr.fun().name().to_string(), + ), + ctx.encode_udaf(aggregate_expr.fun())?, + aggregate_expr.ignore_nulls(), + aggregate_expr.is_distinct(), + ) + } else if let Some(standard) = expr.downcast_ref::() { + if let Some(window_udf) = standard + .get_standard_func_expr() + .as_any() + .downcast_ref::() + { + // `WindowUDFExpr::args` returns the full, unfiltered argument list so + // every argument survives the round-trip. + args = window_udf.args().to_vec(); + ( + physical_window_expr_node::WindowFunction::UserDefinedWindowFunction( + window_udf.fun().name().to_string(), + ), + ctx.encode_udwf(window_udf.fun().as_ref())?, + false, + false, + ) + } else { + return not_impl_err!( + "User-defined window function not supported: {window_expr:?}" + ); + } + } else { + return not_impl_err!("WindowExpr not supported: {window_expr:?}"); + }; + + let args = ctx.encode_expressions(&args)?; + let partition_by = ctx.encode_expressions(window_expr.partition_by())?; + let order_by = window_expr + .order_by() + .iter() + .map(|sort_expr| { + Ok(protobuf::PhysicalSortExprNode { + expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)), + asc: !sort_expr.options.descending, + nulls_first: sort_expr.options.nulls_first, + }) + }) + .collect::>>()?; + + Ok(protobuf::PhysicalWindowExprNode { + args, + partition_by, + order_by, + window_frame: Some(encode_window_frame(window_frame.as_ref())?), + window_function: Some(window_function), + name: window_expr.name().to_string(), + fun_definition, + ignore_nulls, + distinct, + }) +} + +pub(super) fn decode_physical_window_expr( + proto: &protobuf::PhysicalWindowExprNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + input_schema: &Schema, +) -> Result> { + let args = proto + .args + .iter() + .map(|expr| ctx.decode_expr(expr, input_schema)) + .collect::>>()?; + let partition_by = proto + .partition_by + .iter() + .map(|expr| ctx.decode_expr(expr, input_schema)) + .collect::>>()?; + let order_by = proto + .order_by + .iter() + .map(|sort_expr| { + let expr = sort_expr.expr.as_ref().ok_or_else(|| { + internal_datafusion_err!( + "Missing expr in window order_by sort expression" + ) + })?; + Ok(PhysicalSortExpr { + expr: ctx.decode_expr(expr, input_schema)?, + options: SortOptions { + descending: !sort_expr.asc, + nulls_first: sort_expr.nulls_first, + }, + }) + }) + .collect::>>()?; + let window_frame = proto + .window_frame + .as_ref() + .map(decode_window_frame) + .transpose()? + .ok_or_else(|| { + internal_datafusion_err!("Missing required field 'window_frame' in protobuf") + })?; + let function = match proto.window_function.as_ref() { + Some(physical_window_expr_node::WindowFunction::UserDefinedAggrFunction( + name, + )) => WindowFunctionDefinition::AggregateUDF( + ctx.decode_udaf(name, proto.fun_definition.as_deref())?, + ), + Some(physical_window_expr_node::WindowFunction::UserDefinedWindowFunction( + name, + )) => WindowFunctionDefinition::WindowUDF( + ctx.decode_udwf(name, proto.fun_definition.as_deref())?, + ), + None => { + return internal_err!("Missing required field 'window_function' in protobuf"); + } + }; + + let name = proto.name.clone(); + // TODO: Remove extended_schema if functions are all UDAF + let extended_schema = schema_add_window_field(&args, input_schema, &function, &name)?; + create_window_expr( + &function, + name, + &args, + &partition_by, + &order_by, + Arc::new(window_frame), + extended_schema, + proto.ignore_nulls, + proto.distinct, + None, + ) +} + +fn encode_window_frame(window_frame: &WindowFrame) -> Result { + let units = match window_frame.units { + WindowFrameUnits::Rows => protobuf::WindowFrameUnits::Rows, + WindowFrameUnits::Range => protobuf::WindowFrameUnits::Range, + WindowFrameUnits::Groups => protobuf::WindowFrameUnits::Groups, + }; + Ok(protobuf::WindowFrame { + window_frame_units: units.into(), + start_bound: Some(encode_window_frame_bound(&window_frame.start_bound)?), + end_bound: Some(protobuf::window_frame::EndBound::Bound( + encode_window_frame_bound(&window_frame.end_bound)?, + )), + }) +} + +fn encode_window_frame_bound( + bound: &WindowFrameBound, +) -> Result { + let encode_value = |value: &ScalarValue| -> Result { + Ok(value.try_into()?) + }; + Ok(match bound { + WindowFrameBound::CurrentRow => protobuf::WindowFrameBound { + window_frame_bound_type: protobuf::WindowFrameBoundType::CurrentRow.into(), + bound_value: None, + }, + WindowFrameBound::Preceding(value) => protobuf::WindowFrameBound { + window_frame_bound_type: protobuf::WindowFrameBoundType::Preceding.into(), + bound_value: Some(encode_value(value)?), + }, + WindowFrameBound::Following(value) => protobuf::WindowFrameBound { + window_frame_bound_type: protobuf::WindowFrameBoundType::Following.into(), + bound_value: Some(encode_value(value)?), + }, + }) +} + +fn decode_window_frame(window_frame: &protobuf::WindowFrame) -> Result { + let units = protobuf::WindowFrameUnits::try_from(window_frame.window_frame_units) + .map_err(|_| { + internal_datafusion_err!( + "Received a WindowFrame message with unknown WindowFrameUnits {}", + window_frame.window_frame_units + ) + })?; + let units = match units { + protobuf::WindowFrameUnits::Rows => WindowFrameUnits::Rows, + protobuf::WindowFrameUnits::Range => WindowFrameUnits::Range, + protobuf::WindowFrameUnits::Groups => WindowFrameUnits::Groups, + }; + let start_bound = + decode_window_frame_bound(window_frame.start_bound.as_ref().ok_or_else( + || internal_datafusion_err!("Missing start_bound in WindowFrame"), + )?)?; + let end_bound = window_frame + .end_bound + .as_ref() + .map(|end_bound| match end_bound { + protobuf::window_frame::EndBound::Bound(bound) => { + decode_window_frame_bound(bound) + } + }) + .transpose()? + .unwrap_or(WindowFrameBound::CurrentRow); + Ok(WindowFrame::new_bounds(units, start_bound, end_bound)) +} + +fn decode_window_frame_bound( + bound: &protobuf::WindowFrameBound, +) -> Result { + let decode_value = |value: &protobuf_common::ScalarValue| -> Result { + Ok(ScalarValue::try_from(value)?) + }; + let bound_type = protobuf::WindowFrameBoundType::try_from( + bound.window_frame_bound_type, + ) + .map_err(|_| { + internal_datafusion_err!( + "Received a WindowFrameBound message with unknown WindowFrameBoundType {}", + bound.window_frame_bound_type + ) + })?; + match bound_type { + protobuf::WindowFrameBoundType::CurrentRow => Ok(WindowFrameBound::CurrentRow), + protobuf::WindowFrameBoundType::Preceding => match &bound.bound_value { + Some(value) => Ok(WindowFrameBound::Preceding(decode_value(value)?)), + None => Ok(WindowFrameBound::Preceding(ScalarValue::UInt64(None))), + }, + protobuf::WindowFrameBoundType::Following => match &bound.bound_value { + Some(value) => Ok(WindowFrameBound::Following(decode_value(value)?)), + None => Ok(WindowFrameBound::Following(ScalarValue::UInt64(None))), + }, + } +} diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs index 3eb8edd298901..81838300cf5c7 100644 --- a/datafusion/physical-plan/src/windows/window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs @@ -21,6 +21,8 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +#[cfg(feature = "proto")] +use super::proto::{decode_physical_window_expr, encode_physical_window_expr}; use super::utils::create_schema; use crate::execution_plan::{CardinalityEffect, EmissionType}; use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; @@ -317,6 +319,106 @@ impl ExecutionPlan for WindowAggExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let input = ctx.encode_child(self.input())?; + let window_expr = self + .window_expr() + .iter() + .map(|expr| encode_physical_window_expr(expr, ctx)) + .collect::>>()?; + let partition_keys = self + .partition_keys() + .iter() + .map(|expr| ctx.encode_expr(expr)) + .collect::>>()?; + + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Window(Box::new( + protobuf::WindowAggExecNode { + input: Some(Box::new(input)), + window_expr, + partition_keys, + // `None` distinguishes a `WindowAggExec` from a + // `BoundedWindowAggExec` on the shared `Window` variant. + input_order_mode: None, + }, + )), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl WindowAggExec { + /// Reconstruct a window plan from its protobuf representation. + /// + /// This returns a [`WindowAggExec`] when `input_order_mode` is absent and a + /// [`BoundedWindowAggExec`] when it is present. + /// + /// [`BoundedWindowAggExec`]: crate::windows::BoundedWindowAggExec + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use super::BoundedWindowAggExec; + use crate::InputOrderMode; + use datafusion_proto_models::protobuf; + use protobuf::window_agg_exec_node::InputOrderMode as ProtoInputOrderMode; + + let window_agg = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::Window, + "WindowAggExec", + ); + let input = ctx.decode_required_child( + window_agg.input.as_deref(), + "WindowAggExec", + "input", + )?; + let input_schema = input.schema(); + let window_expr = window_agg + .window_expr + .iter() + .map(|expr| decode_physical_window_expr(expr, ctx, input_schema.as_ref())) + .collect::>>()?; + let partition_keys = window_agg + .partition_keys + .iter() + .map(|expr| ctx.decode_expr(expr, input_schema.as_ref())) + .collect::>>()?; + + if let Some(input_order_mode) = window_agg.input_order_mode.as_ref() { + let input_order_mode = match input_order_mode { + ProtoInputOrderMode::Linear(_) => InputOrderMode::Linear, + ProtoInputOrderMode::PartiallySorted( + protobuf::PartiallySortedInputOrderMode { columns }, + ) => InputOrderMode::PartiallySorted( + columns.iter().map(|column| *column as usize).collect(), + ), + ProtoInputOrderMode::Sorted(_) => InputOrderMode::Sorted, + }; + Ok(Arc::new(BoundedWindowAggExec::try_new( + window_expr, + input, + input_order_mode, + !partition_keys.is_empty(), + )?)) + } else { + Ok(Arc::new(WindowAggExec::try_new( + window_expr, + input, + !partition_keys.is_empty(), + )?)) + } + } } /// Compute the window aggregate columns diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 5cb9a922a8826..e1801f66ef64f 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -98,7 +98,7 @@ use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeE use datafusion_physical_plan::union::{InterleaveExec, UnionExec}; use datafusion_physical_plan::unnest::UnnestExec; use datafusion_physical_plan::windows::{BoundedWindowAggExec, WindowAggExec}; -use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, PhysicalExpr, WindowExpr}; +use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use prost::Message; use prost::bytes::BufMut; @@ -107,18 +107,18 @@ use crate::convert::{FromProto, TryFromProto}; use crate::convert_required; use crate::physical_plan::from_proto::{ parse_physical_expr_with_converter, parse_physical_sort_expr, - parse_physical_sort_exprs, parse_physical_window_expr, - parse_protobuf_file_scan_config, parse_record_batches, parse_table_schema_from_proto, + parse_physical_sort_exprs, parse_protobuf_file_scan_config, parse_record_batches, + parse_table_schema_from_proto, }; use crate::physical_plan::to_proto::{ serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr, serialize_physical_expr_with_converter, serialize_physical_sort_exprs, - serialize_physical_window_expr, serialize_record_batches, + serialize_record_batches, }; use crate::protobuf::physical_aggregate_expr_node::AggregateFunction; use crate::protobuf::physical_expr_node::ExprType; use crate::protobuf::physical_plan_node::PhysicalPlanType; -use crate::protobuf::{self, SortMergeJoinExecNode, proto_error, window_agg_exec_node}; +use crate::protobuf::{self, SortMergeJoinExecNode, proto_error}; pub mod from_proto; pub mod to_proto; @@ -790,8 +790,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::LocalLimit(_) => { LocalLimitExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::Window(window_agg) => { - self.try_into_window_physical_plan(window_agg, ctx, proto_converter) + PhysicalPlanType::Window(_) => { + WindowAggExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::Aggregate(hash_agg) => { self.try_into_aggregate_physical_plan(hash_agg, ctx, proto_converter) @@ -924,22 +924,6 @@ pub trait PhysicalPlanNodeExt: Sized { return Ok(node); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_window_agg_exec( - exec, - codec, - proto_converter, - ); - } - - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_bounded_window_agg_exec( - exec, - codec, - proto_converter, - ); - } - if let Some(exec) = plan.downcast_ref::() && let Some(node) = protobuf::PhysicalPlanNode::try_from_data_sink_exec( exec, @@ -1419,61 +1403,27 @@ pub trait PhysicalPlanNodeExt: Sized { LocalLimitExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; window plans deserialize via `WindowAggExec::try_from_proto`" + )] fn try_into_window_physical_plan( &self, window_agg: &protobuf::WindowAggExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let input: Arc = - into_physical_plan(&window_agg.input, ctx, proto_converter)?; - let input_schema = input.schema(); - - let physical_window_expr: Vec> = window_agg - .window_expr - .iter() - .map(|window_expr| { - parse_physical_window_expr( - window_expr, - ctx, - input_schema.as_ref(), - proto_converter, - ) - }) - .collect::, _>>()?; - - let partition_keys = window_agg - .partition_keys - .iter() - .map(|expr| { - proto_converter.proto_to_physical_expr(expr, input.schema().as_ref(), ctx) - }) - .collect::>>>()?; - - if let Some(input_order_mode) = window_agg.input_order_mode.as_ref() { - let input_order_mode = match input_order_mode { - window_agg_exec_node::InputOrderMode::Linear(_) => InputOrderMode::Linear, - window_agg_exec_node::InputOrderMode::PartiallySorted( - protobuf::PartiallySortedInputOrderMode { columns }, - ) => InputOrderMode::PartiallySorted( - columns.iter().map(|c| *c as usize).collect(), - ), - window_agg_exec_node::InputOrderMode::Sorted(_) => InputOrderMode::Sorted, - }; - - Ok(Arc::new(BoundedWindowAggExec::try_new( - physical_window_expr, - input, - input_order_mode, - !partition_keys.is_empty(), - )?)) - } else { - Ok(Arc::new(WindowAggExec::try_new( - physical_window_expr, - input, - !partition_keys.is_empty(), - )?)) - } + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::Window(Box::new( + window_agg.clone(), + ))), + }; + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + WindowAggExec::try_from_proto(&node, &decode_ctx) } fn try_into_aggregate_physical_plan( @@ -2985,89 +2935,40 @@ pub trait PhysicalPlanNodeExt: Sized { }) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `WindowAggExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_window_agg_exec( exec: &WindowAggExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.input().to_owned(), + let encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - - let window_expr = exec - .window_expr() - .iter() - .map(|e| serialize_physical_window_expr(e, codec, proto_converter)) - .collect::>>()?; - - let partition_keys = exec - .partition_keys() - .iter() - .map(|e| proto_converter.physical_expr_to_proto(e, codec)) - .collect::>>()?; - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Window(Box::new( - protobuf::WindowAggExecNode { - input: Some(Box::new(input)), - window_expr, - partition_keys, - input_order_mode: None, - }, - ))), - }) + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)? + .ok_or_else(|| internal_datafusion_err!("WindowAggExec is not serializable")) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `BoundedWindowAggExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_bounded_window_agg_exec( exec: &BoundedWindowAggExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.input().to_owned(), + let encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - - let window_expr = exec - .window_expr() - .iter() - .map(|e| serialize_physical_window_expr(e, codec, proto_converter)) - .collect::>>()?; - - let partition_keys = exec - .partition_keys() - .iter() - .map(|e| proto_converter.physical_expr_to_proto(e, codec)) - .collect::>>()?; - - let input_order_mode = match &exec.input_order_mode { - InputOrderMode::Linear => { - window_agg_exec_node::InputOrderMode::Linear(protobuf::EmptyMessage {}) - } - InputOrderMode::PartiallySorted(columns) => { - window_agg_exec_node::InputOrderMode::PartiallySorted( - protobuf::PartiallySortedInputOrderMode { - columns: columns.iter().map(|c| *c as u64).collect(), - }, - ) - } - InputOrderMode::Sorted => { - window_agg_exec_node::InputOrderMode::Sorted(protobuf::EmptyMessage {}) - } }; - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Window(Box::new( - protobuf::WindowAggExecNode { - input: Some(Box::new(input)), - window_expr, - partition_keys, - input_order_mode: Some(input_order_mode), - }, - ))), + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)?.ok_or_else(|| { + internal_datafusion_err!("BoundedWindowAggExec is not serializable") }) } From 4a40101c69174ca68ef89ea5732d133ebf3730be Mon Sep 17 00:00:00 2001 From: Phoenix Date: Sat, 25 Jul 2026 10:27:34 +0800 Subject: [PATCH 019/109] Migrate ExplainExec and AnalyzeExec protobuf serde (#23742) ## Which issue does this PR close? - Closes #23511. ## Rationale for this change Physical plan protobuf serialization is being moved from the central dispatch module into each `ExecutionPlan` implementation. Co-locating this logic with the corresponding execution plan makes the serialization code easier to maintain and incrementally reduces the central downcast chain. ## What changes are included in this PR? - Implement plan-local protobuf serialization and deserialization for `ExplainExec`. - Implement plan-local protobuf serialization and deserialization for `AnalyzeExec`. - Remove both plans from the live central serialization dispatch. - Retain the deprecated compatibility methods as delegates to the new implementations. - Preserve the existing protobuf wire format. - Strengthen roundtrip tests to cover all stored fields, including every `StringifiedPlan` variant, metric categories, and explain formats. Each execution plan migration is kept in a separate commit. ## Are these changes tested? Yes. The following checks pass: - `cargo fmt --all` - `cargo clippy --all-targets --all-features -- -D warnings` - Full extended workspace test suite - `cargo test -p datafusion-proto --test proto_integration` The proto integration suite passes all 209 tests after rebasing onto the latest `main`. ## Are there any user-facing changes? No. This is an internal refactor and does not change the protobuf wire format or user-facing behavior. --------- Signed-off-by: Jiawei Zhao --- datafusion/physical-plan/src/analyze.rs | 96 +++++++++ datafusion/physical-plan/src/explain.rs | 182 ++++++++++++++++++ datafusion/proto/src/physical_plan/mod.rs | 156 +++++---------- .../tests/cases/roundtrip_physical_plan.rs | 116 ++++++++++- 4 files changed, 437 insertions(+), 113 deletions(-) diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs index 72cd24ef95673..31e0a27410ff9 100644 --- a/datafusion/physical-plan/src/analyze.rs +++ b/datafusion/physical-plan/src/analyze.rs @@ -303,6 +303,102 @@ impl ExecutionPlan for AnalyzeExec { futures::stream::once(output), ))) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let input = ctx.encode_child(self.input())?; + let (has_metric_categories, metric_categories) = match self.metric_categories() { + Some(categories) => { + (true, categories.iter().map(ToString::to_string).collect()) + } + None => (false, vec![]), + }; + let format = match self.format() { + ExplainFormat::Indent => protobuf::ExplainFormat::Indent, + ExplainFormat::Tree => protobuf::ExplainFormat::Tree, + ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson, + ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz, + } as i32; + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Analyze(Box::new( + protobuf::AnalyzeExecNode { + verbose: self.verbose(), + show_statistics: self.show_statistics(), + input: Some(Box::new(input)), + schema: Some(self.schema().as_ref().try_into()?), + has_metric_categories, + metric_categories, + format, + }, + )), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl AnalyzeExec { + /// Reconstruct an [`AnalyzeExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let analyze = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::Analyze, + "AnalyzeExec", + ); + let input = + ctx.decode_required_child(analyze.input.as_deref(), "AnalyzeExec", "input")?; + let metric_categories = if analyze.has_metric_categories { + Some( + analyze + .metric_categories + .iter() + .map(|category| category.parse::()) + .collect::>>()?, + ) + } else { + None + }; + let proto_format = + protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| { + DataFusionError::Internal(format!( + "Received an AnalyzeExecNode message with unknown ExplainFormat {}", + analyze.format + )) + })?; + let format = match proto_format { + protobuf::ExplainFormat::Indent => ExplainFormat::Indent, + protobuf::ExplainFormat::Tree => ExplainFormat::Tree, + protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON, + protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz, + }; + let schema = analyze.schema.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "AnalyzeExec is missing required field 'schema'" + ) + })?; + Ok(Arc::new( + AnalyzeExec::builder( + analyze.verbose, + analyze.show_statistics, + input, + Arc::new(arrow::datatypes::Schema::try_from(schema)?), + ) + .with_metric_categories(metric_categories) + .with_format(format) + .build(), + )) + } } /// Creates the output of AnalyzeExec as a RecordBatch diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs index 98eac3d28b5df..a270a003eba17 100644 --- a/datafusion/physical-plan/src/explain.rs +++ b/datafusion/physical-plan/src/explain.rs @@ -185,6 +185,188 @@ impl ExecutionPlan for ExplainExec { futures::stream::iter(vec![Ok(record_batch)]), ))) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + _ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Explain( + protobuf::ExplainExecNode { + schema: Some(self.schema().as_ref().try_into()?), + stringified_plans: self + .stringified_plans() + .iter() + .map(stringified_plan_to_proto) + .collect(), + verbose: self.verbose(), + }, + ), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl ExplainExec { + /// Reconstruct an [`ExplainExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + _ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let explain = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::Explain, + "ExplainExec", + ); + let schema = explain.schema.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "ExplainExec is missing required field 'schema'" + ) + })?; + Ok(Arc::new(ExplainExec::new( + Arc::new(arrow::datatypes::Schema::try_from(schema)?), + explain + .stringified_plans + .iter() + .map(stringified_plan_from_proto) + .collect(), + explain.verbose, + ))) + } +} + +#[cfg(feature = "proto")] +fn stringified_plan_to_proto( + stringified_plan: &StringifiedPlan, +) -> datafusion_proto_models::protobuf::StringifiedPlan { + use datafusion_common::display::PlanType; + use datafusion_proto_models::datafusion_common::EmptyMessage; + use datafusion_proto_models::protobuf; + use protobuf::plan_type::PlanTypeEnum::{ + AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan, + FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats, + InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema, + InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan, + PhysicalPlanError, + }; + + protobuf::StringifiedPlan { + plan_type: match stringified_plan.clone().plan_type { + PlanType::InitialLogicalPlan => Some(protobuf::PlanType { + plan_type_enum: Some(InitialLogicalPlan(EmptyMessage {})), + }), + PlanType::AnalyzedLogicalPlan { analyzer_name } => Some(protobuf::PlanType { + plan_type_enum: Some(AnalyzedLogicalPlan( + protobuf::AnalyzedLogicalPlanType { analyzer_name }, + )), + }), + PlanType::FinalAnalyzedLogicalPlan => Some(protobuf::PlanType { + plan_type_enum: Some(FinalAnalyzedLogicalPlan(EmptyMessage {})), + }), + PlanType::OptimizedLogicalPlan { optimizer_name } => { + Some(protobuf::PlanType { + plan_type_enum: Some(OptimizedLogicalPlan( + protobuf::OptimizedLogicalPlanType { optimizer_name }, + )), + }) + } + PlanType::FinalLogicalPlan => Some(protobuf::PlanType { + plan_type_enum: Some(FinalLogicalPlan(EmptyMessage {})), + }), + PlanType::InitialPhysicalPlan => Some(protobuf::PlanType { + plan_type_enum: Some(InitialPhysicalPlan(EmptyMessage {})), + }), + PlanType::OptimizedPhysicalPlan { optimizer_name } => { + Some(protobuf::PlanType { + plan_type_enum: Some(OptimizedPhysicalPlan( + protobuf::OptimizedPhysicalPlanType { optimizer_name }, + )), + }) + } + PlanType::FinalPhysicalPlan => Some(protobuf::PlanType { + plan_type_enum: Some(FinalPhysicalPlan(EmptyMessage {})), + }), + PlanType::InitialPhysicalPlanWithStats => Some(protobuf::PlanType { + plan_type_enum: Some(InitialPhysicalPlanWithStats(EmptyMessage {})), + }), + PlanType::InitialPhysicalPlanWithSchema => Some(protobuf::PlanType { + plan_type_enum: Some(InitialPhysicalPlanWithSchema(EmptyMessage {})), + }), + PlanType::FinalPhysicalPlanWithStats => Some(protobuf::PlanType { + plan_type_enum: Some(FinalPhysicalPlanWithStats(EmptyMessage {})), + }), + PlanType::FinalPhysicalPlanWithSchema => Some(protobuf::PlanType { + plan_type_enum: Some(FinalPhysicalPlanWithSchema(EmptyMessage {})), + }), + PlanType::PhysicalPlanError => Some(protobuf::PlanType { + plan_type_enum: Some(PhysicalPlanError(EmptyMessage {})), + }), + }, + plan: stringified_plan.plan.to_string(), + } +} + +#[cfg(feature = "proto")] +fn stringified_plan_from_proto( + stringified_plan: &datafusion_proto_models::protobuf::StringifiedPlan, +) -> StringifiedPlan { + use datafusion_common::display::PlanType; + use datafusion_proto_models::protobuf::plan_type::PlanTypeEnum::{ + AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan, + FinalPhysicalPlan, FinalPhysicalPlanWithSchema, FinalPhysicalPlanWithStats, + InitialLogicalPlan, InitialPhysicalPlan, InitialPhysicalPlanWithSchema, + InitialPhysicalPlanWithStats, OptimizedLogicalPlan, OptimizedPhysicalPlan, + PhysicalPlanError, + }; + use datafusion_proto_models::protobuf::{ + AnalyzedLogicalPlanType, OptimizedLogicalPlanType, OptimizedPhysicalPlanType, + }; + + StringifiedPlan { + plan_type: match stringified_plan + .plan_type + .as_ref() + .and_then(|plan_type| plan_type.plan_type_enum.as_ref()) + .unwrap_or_else(|| { + panic!( + "Cannot create protobuf::StringifiedPlan from {stringified_plan:?}" + ) + }) { + InitialLogicalPlan(_) => PlanType::InitialLogicalPlan, + AnalyzedLogicalPlan(AnalyzedLogicalPlanType { analyzer_name }) => { + PlanType::AnalyzedLogicalPlan { + analyzer_name: analyzer_name.clone(), + } + } + FinalAnalyzedLogicalPlan(_) => PlanType::FinalAnalyzedLogicalPlan, + OptimizedLogicalPlan(OptimizedLogicalPlanType { optimizer_name }) => { + PlanType::OptimizedLogicalPlan { + optimizer_name: optimizer_name.clone(), + } + } + FinalLogicalPlan(_) => PlanType::FinalLogicalPlan, + InitialPhysicalPlan(_) => PlanType::InitialPhysicalPlan, + InitialPhysicalPlanWithStats(_) => PlanType::InitialPhysicalPlanWithStats, + InitialPhysicalPlanWithSchema(_) => PlanType::InitialPhysicalPlanWithSchema, + OptimizedPhysicalPlan(OptimizedPhysicalPlanType { optimizer_name }) => { + PlanType::OptimizedPhysicalPlan { + optimizer_name: optimizer_name.clone(), + } + } + FinalPhysicalPlan(_) => PlanType::FinalPhysicalPlan, + FinalPhysicalPlanWithStats(_) => PlanType::FinalPhysicalPlanWithStats, + FinalPhysicalPlanWithSchema(_) => PlanType::FinalPhysicalPlanWithSchema, + PhysicalPlanError(_) => PlanType::PhysicalPlanError, + }, + plan: Arc::new(stringified_plan.plan.clone()), + } } /// If this plan should be shown, given the previous plan that was diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index e1801f66ef64f..748ca53505c4d 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -24,8 +24,6 @@ use std::sync::Arc; use arrow::datatypes::{IntervalMonthDayNanoType, Schema, SchemaRef}; use datafusion_catalog::memory::MemorySourceConfig; use datafusion_common::config::CsvOptions; -use datafusion_common::display::StringifiedPlan; -use datafusion_common::format::ExplainFormat; use datafusion_common::{ DataFusionError, Result, internal_datafusion_err, internal_err, not_impl_err, }; @@ -84,7 +82,6 @@ use datafusion_physical_plan::joins::{ }; use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use datafusion_physical_plan::memory::LazyMemoryExec; -use datafusion_physical_plan::metrics::MetricCategory; use datafusion_physical_plan::placeholder_row::PlaceholderRowExec; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::proto::{ @@ -103,7 +100,7 @@ use prost::Message; use prost::bytes::BufMut; use crate::common::{byte_to_string, str_to_byte}; -use crate::convert::{FromProto, TryFromProto}; +use crate::convert::TryFromProto; use crate::convert_required; use crate::physical_plan::from_proto::{ parse_physical_expr_with_converter, parse_physical_sort_expr, @@ -744,8 +741,8 @@ pub trait PhysicalPlanNodeExt: Sized { }; let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder); match plan { - PhysicalPlanType::Explain(explain) => { - self.try_into_explain_physical_plan(explain, ctx, proto_converter) + PhysicalPlanType::Explain(_) => { + ExplainExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::Projection(_) => { ProjectionExec::try_from_proto(self.node(), &decode_ctx) @@ -829,8 +826,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::NestedLoopJoin(_) => { NestedLoopJoinExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::Analyze(analyze) => { - self.try_into_analyze_physical_plan(analyze, ctx, proto_converter) + PhysicalPlanType::Analyze(_) => { + AnalyzeExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::JsonSink(sink) => { self.try_into_json_sink_physical_plan(sink, ctx, proto_converter) @@ -894,18 +891,6 @@ pub trait PhysicalPlanNodeExt: Sized { return Ok(node); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_explain_exec(exec, codec); - } - - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_analyze_exec( - exec, - codec, - proto_converter, - ); - } - if let Some(exec) = plan.downcast_ref::() { return protobuf::PhysicalPlanNode::try_from_aggregate_exec( exec, @@ -977,21 +962,22 @@ pub trait PhysicalPlanNodeExt: Sized { } } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `ExplainExec` deserializes itself via `ExplainExec::try_from_proto`" + )] fn try_into_explain_physical_plan( &self, - explain: &protobuf::ExplainExecNode, - _ctx: &PhysicalPlanDecodeContext<'_>, - _proto_converter: &dyn PhysicalProtoConverterExtension, + _explain: &protobuf::ExplainExecNode, + ctx: &PhysicalPlanDecodeContext<'_>, + proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - Ok(Arc::new(ExplainExec::new( - Arc::new(explain.schema.as_ref().unwrap().try_into()?), - explain - .stringified_plans - .iter() - .map(StringifiedPlan::from_proto) - .collect(), - explain.verbose, - ))) + let plan_decoder = ConverterPlanDecoder { + ctx, + proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder); + ExplainExec::try_from_proto(self.node(), &decode_ctx) } #[deprecated( @@ -1878,48 +1864,22 @@ pub trait PhysicalPlanNodeExt: Sized { NestedLoopJoinExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AnalyzeExec` deserializes itself via `AnalyzeExec::try_from_proto`" + )] fn try_into_analyze_physical_plan( &self, - analyze: &protobuf::AnalyzeExecNode, + _analyze: &protobuf::AnalyzeExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let input: Arc = - into_physical_plan(&analyze.input, ctx, proto_converter)?; - let metric_categories = if analyze.has_metric_categories { - let cats: Result> = analyze - .metric_categories - .iter() - .map(|s| s.parse::()) - .collect(); - Some(cats?) - } else { - None + let plan_decoder = ConverterPlanDecoder { + ctx, + proto_converter, }; - let pb_format = - protobuf::ExplainFormat::try_from(analyze.format).map_err(|_| { - DataFusionError::Internal(format!( - "Received an AnalyzeExecNode message with unknown ExplainFormat {}", - analyze.format - )) - })?; - let format = match pb_format { - protobuf::ExplainFormat::Indent => ExplainFormat::Indent, - protobuf::ExplainFormat::Tree => ExplainFormat::Tree, - protobuf::ExplainFormat::Pgjson => ExplainFormat::PostgresJSON, - protobuf::ExplainFormat::Graphviz => ExplainFormat::Graphviz, - }; - Ok(Arc::new( - AnalyzeExec::builder( - analyze.verbose, - analyze.show_statistics, - input, - Arc::new(convert_required!(analyze.schema)?), - ) - .with_metric_categories(metric_categories) - .with_format(format) - .build(), - )) + let decode_ctx = ExecutionPlanDecodeCtx::new(&plan_decoder); + AnalyzeExec::try_from_proto(self.node(), &decode_ctx) } fn try_into_json_sink_physical_plan( @@ -2263,22 +2223,22 @@ pub trait PhysicalPlanNodeExt: Sized { ))) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `ExplainExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_explain_exec( exec: &ExplainExec, - _codec: &dyn PhysicalExtensionCodec, + codec: &dyn PhysicalExtensionCodec, ) -> Result { - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Explain( - protobuf::ExplainExecNode { - schema: Some(exec.schema().as_ref().try_into()?), - stringified_plans: exec - .stringified_plans() - .iter() - .map(protobuf::StringifiedPlan::from_proto) - .collect(), - verbose: exec.verbose(), - }, - )), + let proto_converter = DefaultPhysicalProtoConverter {}; + let plan_encoder = ConverterPlanEncoder { + codec, + proto_converter: &proto_converter, + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder); + exec.try_to_proto(&encode_ctx)?.ok_or_else(|| { + internal_datafusion_err!("ExplainExec did not serialize itself") }) } @@ -2301,38 +2261,22 @@ pub trait PhysicalPlanNodeExt: Sized { }) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AnalyzeExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_analyze_exec( exec: &AnalyzeExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.input().to_owned(), + let plan_encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - let (has_metric_categories, metric_categories) = match exec.metric_categories() { - Some(cats) => (true, cats.iter().map(|c| c.to_string()).collect()), - None => (false, vec![]), - }; - let format = match exec.format() { - ExplainFormat::Indent => protobuf::ExplainFormat::Indent, - ExplainFormat::Tree => protobuf::ExplainFormat::Tree, - ExplainFormat::PostgresJSON => protobuf::ExplainFormat::Pgjson, - ExplainFormat::Graphviz => protobuf::ExplainFormat::Graphviz, - } as i32; - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Analyze(Box::new( - protobuf::AnalyzeExecNode { - verbose: exec.verbose(), - show_statistics: exec.show_statistics(), - input: Some(Box::new(input)), - schema: Some(exec.schema().as_ref().try_into()?), - has_metric_categories, - metric_categories, - format, - }, - ))), + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&plan_encoder); + exec.try_to_proto(&encode_ctx)?.ok_or_else(|| { + internal_datafusion_err!("AnalyzeExec did not serialize itself") }) } diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 889d42df40e0e..3d13ffe16e8b9 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -66,6 +66,7 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::coop::CooperativeExec; use datafusion::physical_plan::empty::EmptyExec; +use datafusion::physical_plan::explain::ExplainExec; use datafusion::physical_plan::expressions::{ BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary, cast, col, in_list, like, lit, @@ -77,6 +78,7 @@ use datafusion::physical_plan::joins::{ StreamJoinPartitionMode, SymmetricHashJoinExec, }; use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; +use datafusion::physical_plan::metrics::MetricCategory; use datafusion::physical_plan::placeholder_row::PlaceholderRowExec; use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; use datafusion::physical_plan::repartition::RepartitionExec; @@ -98,8 +100,10 @@ use datafusion::physical_plan::{ use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion::scalar::ScalarValue; use datafusion_common::config::{ConfigOptions, TableParquetOptions}; +use datafusion_common::display::{PlanType, StringifiedPlan}; use datafusion_common::file_options::csv_writer::CsvWriterOptions; use datafusion_common::file_options::json_writer::JsonWriterOptions; +use datafusion_common::format::ExplainFormat; use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::stats::Precision; use datafusion_common::{ @@ -1947,14 +1951,112 @@ fn roundtrip_like() -> Result<()> { #[test] fn roundtrip_analyze() -> Result<()> { - let field_a = Field::new("plan_type", DataType::Utf8, false); - let field_b = Field::new("plan", DataType::Utf8, false); - let schema = Schema::new(vec![field_a, field_b]); - let input = Arc::new(PlaceholderRowExec::new(Arc::new(schema.clone()))); + let schema = Arc::new(Schema::new(vec![ + Field::new("plan_type", DataType::Utf8, false), + Field::new("plan", DataType::Utf8, false), + ])); + let input = Arc::new(PlaceholderRowExec::new(Arc::clone(&schema))); + let metric_categories = vec![MetricCategory::Rows, MetricCategory::Timing]; + let analyze = Arc::new( + AnalyzeExec::builder(true, true, input, Arc::clone(&schema)) + .with_metric_categories(Some(metric_categories.clone())) + .with_format(ExplainFormat::Tree) + .build(), + ); - roundtrip_test(Arc::new( - AnalyzeExec::builder(false, false, input, Arc::new(schema)).build(), - )) + let ctx = SessionContext::new(); + let roundtripped = roundtrip_test_and_return( + analyze, + &ctx, + &DefaultPhysicalExtensionCodec {}, + &DefaultPhysicalProtoConverter {}, + )?; + let roundtripped = roundtripped.downcast_ref::().unwrap(); + + assert_eq!(roundtripped.schema(), schema); + assert!(roundtripped.verbose()); + assert!(roundtripped.show_statistics()); + assert_eq!( + roundtripped.metric_categories(), + Some(metric_categories.as_slice()) + ); + assert_eq!(roundtripped.format(), &ExplainFormat::Tree); + assert!( + roundtripped + .input() + .downcast_ref::() + .is_some() + ); + Ok(()) +} + +#[test] +fn roundtrip_explain() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("plan_type", DataType::Utf8, false), + Field::new("plan", DataType::Utf8, false), + ])); + let stringified_plans = vec![ + StringifiedPlan::new(PlanType::InitialLogicalPlan, "initial logical"), + StringifiedPlan::new( + PlanType::AnalyzedLogicalPlan { + analyzer_name: "analyzer".to_string(), + }, + "analyzed logical", + ), + StringifiedPlan::new(PlanType::FinalAnalyzedLogicalPlan, "final analyzed"), + StringifiedPlan::new( + PlanType::OptimizedLogicalPlan { + optimizer_name: "logical optimizer".to_string(), + }, + "optimized logical", + ), + StringifiedPlan::new(PlanType::FinalLogicalPlan, "final logical"), + StringifiedPlan::new(PlanType::InitialPhysicalPlan, "initial physical"), + StringifiedPlan::new( + PlanType::InitialPhysicalPlanWithStats, + "initial physical with stats", + ), + StringifiedPlan::new( + PlanType::InitialPhysicalPlanWithSchema, + "initial physical with schema", + ), + StringifiedPlan::new( + PlanType::OptimizedPhysicalPlan { + optimizer_name: "physical optimizer".to_string(), + }, + "optimized physical", + ), + StringifiedPlan::new(PlanType::FinalPhysicalPlan, "final physical"), + StringifiedPlan::new( + PlanType::FinalPhysicalPlanWithStats, + "final physical with stats", + ), + StringifiedPlan::new( + PlanType::FinalPhysicalPlanWithSchema, + "final physical with schema", + ), + StringifiedPlan::new(PlanType::PhysicalPlanError, "physical plan error"), + ]; + let explain = Arc::new(ExplainExec::new( + Arc::clone(&schema), + stringified_plans.clone(), + true, + )); + + let ctx = SessionContext::new(); + let roundtripped = roundtrip_test_and_return( + explain, + &ctx, + &DefaultPhysicalExtensionCodec {}, + &DefaultPhysicalProtoConverter {}, + )?; + let roundtripped = roundtripped.downcast_ref::().unwrap(); + + assert_eq!(roundtripped.schema(), schema); + assert_eq!(roundtripped.stringified_plans(), stringified_plans); + assert!(roundtripped.verbose()); + Ok(()) } #[tokio::test] From 9e3c71fea170a6c95f9bc4723725decf2c301a9f Mon Sep 17 00:00:00 2001 From: Phoenix Date: Sat, 25 Jul 2026 12:31:33 +0800 Subject: [PATCH 020/109] refactor(proto): migrate aggregate exec serde (#23779) ## Which issue does this PR close? - Closes #23512. ## Rationale for this change Aggregate protobuf conversion lived in the central proto crate, which prevented AggregateExec from owning its function-codec serialization. Move encoding and decoding into AggregateExec and keep the deprecated helpers as delegates so existing callers retain wire-compatible behavior. ## What changes are included in this PR? ## Are these changes tested? Yes ## Are there any user-facing changes? --------- Signed-off-by: Jiawei Zhao --- .../physical-plan/src/aggregates/mod.rs | 401 ++++++++++++++++++ datafusion/proto/src/physical_plan/mod.rs | 383 ++--------------- 2 files changed, 430 insertions(+), 354 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index e3cf1c4568009..787cd4f03ff6c 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -2168,6 +2168,385 @@ impl ExecutionPlan for AggregateExec { Ok(result) } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let input = ctx.encode_child(self.input())?; + let group_by = self.group_expr(); + let group_expr = + ctx.encode_expressions(group_by.expr().iter().map(|(expr, _)| expr))?; + let group_expr_name = group_by + .expr() + .iter() + .map(|(_, name)| name.to_owned()) + .collect(); + let null_expr = + ctx.encode_expressions(group_by.null_expr().iter().map(|(expr, _)| expr))?; + let groups = group_by.groups().iter().flatten().copied().collect(); + let aggr_expr = self + .aggr_expr() + .iter() + .map(|expr| encode_aggregate_expr(expr, ctx)) + .collect::>>()?; + let aggr_expr_name = self + .aggr_expr() + .iter() + .map(|expr| expr.name().to_string()) + .collect(); + let filter_expr = self + .filter_expr() + .iter() + .map(|filter| { + Ok(protobuf::MaybeFilter { + expr: filter + .as_ref() + .map(|expr| ctx.encode_expr(expr)) + .transpose()?, + }) + }) + .collect::>>()?; + // Match by name because the protobuf and execution enums use different + // discriminants, so a numeric cast would corrupt the wire format. + let mode = match self.mode() { + AggregateMode::Partial => protobuf::AggregateMode::Partial, + AggregateMode::Final => protobuf::AggregateMode::Final, + AggregateMode::FinalPartitioned => protobuf::AggregateMode::FinalPartitioned, + AggregateMode::Single => protobuf::AggregateMode::Single, + AggregateMode::SinglePartitioned => { + protobuf::AggregateMode::SinglePartitioned + } + AggregateMode::PartialReduce => protobuf::AggregateMode::PartialReduce, + }; + let limit = self.limit_options().map(|options| protobuf::AggLimit { + limit: options.limit() as u64, + descending: options.descending(), + }); + let dynamic_filter = match self.dynamic_filter_expr() { + Some(filter) => { + let expr: Arc = + Arc::clone(filter) as Arc; + Some(ctx.encode_expr(&expr)?) + } + None => None, + }; + + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::Aggregate(Box::new( + protobuf::AggregateExecNode { + group_expr, + group_expr_name, + aggr_expr, + filter_expr, + aggr_expr_name, + mode: mode as i32, + input: Some(Box::new(input)), + input_schema: Some(self.input_schema().as_ref().try_into()?), + null_expr, + groups, + limit, + has_grouping_set: group_by.has_grouping_set(), + dynamic_filter, + }, + )), + ), + })) + } +} + +/// Keep this marker byte-identical to the copy used by the deprecated +/// aggregate serializer in `datafusion-proto` until that path is removed. +#[cfg(feature = "proto")] +const HUMAN_DISPLAY_ALIAS_PREFIX: &str = "\u{1f}datafusion_human_display_alias_v1:"; + +#[cfg(feature = "proto")] +fn encode_human_display_alias(human_display: &str, alias: &str) -> String { + format!( + "{HUMAN_DISPLAY_ALIAS_PREFIX}{}:{alias}{human_display}", + alias.len() + ) +} + +#[cfg(feature = "proto")] +fn split_human_display_alias<'a>( + human_display: &'a str, + name: &'a str, +) -> (&'a str, Option<&'a str>) { + if let Some(encoded) = human_display.strip_prefix(HUMAN_DISPLAY_ALIAS_PREFIX) + && let Some((alias_len, encoded)) = encoded.split_once(':') + && let Ok(alias_len) = alias_len.parse::() + && let Some(alias) = encoded.get(..alias_len) + && let Some(human_display) = encoded.get(alias_len..) + && alias == name + && !human_display.is_empty() + { + return (human_display, Some(alias)); + } + + (human_display, None) +} + +#[cfg(feature = "proto")] +fn encode_aggregate_expr( + aggr_expr: &Arc, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, +) -> Result { + use datafusion_proto_models::protobuf; + + let expressions = aggr_expr.expressions(); + let expr = ctx.encode_expressions(expressions.iter())?; + let ordering_req = aggr_expr + .order_bys() + .iter() + .map(|sort_expr| { + Ok(protobuf::PhysicalSortExprNode { + expr: Some(Box::new(ctx.encode_expr(&sort_expr.expr)?)), + asc: !sort_expr.options.descending, + nulls_first: sort_expr.options.nulls_first, + }) + }) + .collect::>>()?; + let name = aggr_expr.fun().name().to_string(); + // The context already applies `(!buf.is_empty()).then_some(buf)`. + let fun_definition = ctx.encode_udaf(aggr_expr.fun())?; + let human_display = match (aggr_expr.human_display(), aggr_expr.human_display_alias()) + { + (Some(display), Some(alias)) => encode_human_display_alias(display, alias), + (Some(display), None) => display.to_string(), + (None, _) => String::new(), + }; + + Ok(protobuf::PhysicalExprNode { + expr_id: None, + expr_type: Some(protobuf::physical_expr_node::ExprType::AggregateExpr( + protobuf::PhysicalAggregateExprNode { + aggregate_function: Some( + protobuf::physical_aggregate_expr_node::AggregateFunction::UserDefinedAggrFunction(name), + ), + expr, + ordering_req, + distinct: aggr_expr.is_distinct(), + ignore_nulls: aggr_expr.ignore_nulls(), + fun_definition, + human_display, + }, + )), + }) +} + +#[cfg(feature = "proto")] +impl AggregateExec { + /// Reconstruct an [`AggregateExec`] from its protobuf representation. + /// + /// Grouping expressions are decoded against the child schema. Aggregate + /// arguments, ordering, filters, and the dynamic filter are decoded against + /// the aggregate input schema carried in the protobuf node. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_physical_expr::PhysicalSortExpr; + use datafusion_physical_expr::aggregate::AggregateExprBuilder; + use datafusion_proto_models::protobuf; + use protobuf::physical_aggregate_expr_node::AggregateFunction; + use protobuf::physical_expr_node::ExprType; + + let hash_agg = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::Aggregate, + "AggregateExec", + ); + let input = ctx.decode_required_child( + hash_agg.input.as_deref(), + "AggregateExec", + "input", + )?; + // Match by name because the protobuf and execution enums use different + // discriminants, so a numeric cast would corrupt the wire format. + let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| { + datafusion_common::internal_datafusion_err!( + "Received an AggregateNode message with unknown AggregateMode {}", + hash_agg.mode + ) + })?; + let mode = match mode { + protobuf::AggregateMode::Partial => AggregateMode::Partial, + protobuf::AggregateMode::Final => AggregateMode::Final, + protobuf::AggregateMode::FinalPartitioned => AggregateMode::FinalPartitioned, + protobuf::AggregateMode::Single => AggregateMode::Single, + protobuf::AggregateMode::SinglePartitioned => { + AggregateMode::SinglePartitioned + } + protobuf::AggregateMode::PartialReduce => AggregateMode::PartialReduce, + }; + let num_expr = hash_agg.group_expr.len(); + // Grouping expressions refer to the child plan's output schema. + let child_schema = input.schema(); + let group_expr = hash_agg + .group_expr + .iter() + .zip(hash_agg.group_expr_name.iter()) + .map(|(expr, name)| { + Ok(( + ctx.decode_expr(expr, child_schema.as_ref())?, + name.to_string(), + )) + }) + .collect::>>()?; + let null_expr = hash_agg + .null_expr + .iter() + .zip(hash_agg.group_expr_name.iter()) + .map(|(expr, name)| { + Ok(( + ctx.decode_expr(expr, child_schema.as_ref())?, + name.to_string(), + )) + }) + .collect::>>()?; + let groups = if hash_agg.groups.is_empty() { + vec![] + } else { + hash_agg + .groups + .chunks(num_expr) + .map(|group| group.to_vec()) + .collect() + }; + // Aggregate arguments, ordering, filters, and dynamic filters refer to + // the aggregate input schema carried in the protobuf node. + let input_schema = hash_agg.input_schema.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "input_schema in AggregateNode is missing." + ) + })?; + let input_schema: SchemaRef = SchemaRef::new(input_schema.try_into()?); + let filter_expr = hash_agg + .filter_expr + .iter() + .map(|filter| { + filter + .expr + .as_ref() + .map(|expr| ctx.decode_expr(expr, input_schema.as_ref())) + .transpose() + }) + .collect::>>()?; + let aggr_expr = hash_agg + .aggr_expr + .iter() + .zip(hash_agg.aggr_expr_name.iter()) + .map(|(expr, name)| { + let expr_type = expr.expr_type.as_ref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "Unexpected empty aggregate physical expression" + ) + })?; + let ExprType::AggregateExpr(aggregate) = expr_type else { + return internal_err!( + "Invalid aggregate expression for AggregateExec" + ); + }; + let args = aggregate + .expr + .iter() + .map(|expr| ctx.decode_expr(expr, input_schema.as_ref())) + .collect::>>()?; + let order_by = aggregate + .ordering_req + .iter() + .map(|sort_expr| { + let expr = sort_expr.expr.as_deref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "AggregateExec ordering expression is missing its inner expr" + ) + })?; + Ok(PhysicalSortExpr { + expr: ctx.decode_expr(expr, input_schema.as_ref())?, + options: arrow::compute::SortOptions { + descending: !sort_expr.asc, + nulls_first: sort_expr.nulls_first, + }, + }) + }) + .collect::>>()?; + let Some(AggregateFunction::UserDefinedAggrFunction(udaf_name)) = + aggregate.aggregate_function.as_ref() + else { + return internal_err!( + "Invalid AggregateExpr, missing aggregate_function" + ); + }; + // The context owns the payload-to-codec and + // registry-to-codec fallback order. + let udaf = ctx.decode_udaf( + udaf_name, + aggregate.fun_definition.as_deref(), + )?; + let (human_display, human_display_alias) = + split_human_display_alias(&aggregate.human_display, name); + let builder = AggregateExprBuilder::new(udaf, args) + .schema(Arc::clone(&input_schema)) + .alias(name) + .with_ignore_nulls(aggregate.ignore_nulls) + .with_distinct(aggregate.distinct) + .order_by(order_by) + .human_display(human_display); + let builder = if let Some(alias) = human_display_alias { + builder.human_display_alias(alias) + } else { + builder + }; + builder.build().map(Arc::new) + }) + .collect::>>()?; + let aggregate = AggregateExec::try_new( + mode, + PhysicalGroupBy::new( + group_expr, + null_expr, + groups, + hash_agg.has_grouping_set, + ), + aggr_expr, + filter_expr, + input, + Arc::clone(&input_schema), + )?; + let aggregate = if let Some(limit) = &hash_agg.limit { + let options = match limit.descending { + Some(descending) => { + LimitOptions::new_with_order(limit.limit as usize, descending) + } + None => LimitOptions::new(limit.limit as usize), + }; + aggregate.with_limit_options(Some(options)) + } else { + aggregate + }; + let aggregate = if let Some(dynamic_filter) = &hash_agg.dynamic_filter { + let dynamic_filter = + ctx.decode_expr(dynamic_filter, input_schema.as_ref())?; + let dynamic_filter = (dynamic_filter + as Arc) + .downcast::() + .map_err(|_| { + datafusion_common::internal_datafusion_err!( + "AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr" + ) + })?; + aggregate.with_dynamic_filter_expr(dynamic_filter)? + } else { + aggregate + }; + + Ok(Arc::new(aggregate)) + } } /// Creates the output schema for an [`AggregateExec`] containing the group by columns followed @@ -2708,6 +3087,28 @@ mod tests { use futures::{FutureExt, Stream, StreamExt}; use insta::{allow_duplicates, assert_snapshot}; + #[cfg(feature = "proto")] + #[test] + fn split_human_display_alias_ignores_mismatched_alias() { + let encoded = encode_human_display_alias("sum(value)", "revenue"); + + assert_eq!( + split_human_display_alias(&encoded, "other"), + (encoded.as_str(), None) + ); + } + + #[cfg(feature = "proto")] + #[test] + fn split_human_display_alias_keeps_malformed_prefix_literal() { + let display = format!("{HUMAN_DISPLAY_ALIAS_PREFIX}not-an-encoding"); + + assert_eq!( + split_human_display_alias(&display, "agg"), + (display.as_str(), None) + ); + } + // Generate a schema which consists of 5 columns (a, b, c, d, e) fn create_test_schema() -> Result { let a = Field::new("a", DataType::Int32, true); diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 748ca53505c4d..7ee173cb36868 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -54,14 +54,10 @@ use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF}; use datafusion_functions_table::generate_series::{ Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue, }; -use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr}; -use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr; use datafusion_physical_expr::{LexOrdering, LexRequirement}; use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; -use datafusion_physical_plan::aggregates::{ - AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy, -}; +use datafusion_physical_plan::aggregates::AggregateExec; use datafusion_physical_plan::analyze::AnalyzeExec; use datafusion_physical_plan::async_func::AsyncFuncExec; use datafusion_physical_plan::buffer::BufferExec; @@ -103,17 +99,13 @@ use crate::common::{byte_to_string, str_to_byte}; use crate::convert::TryFromProto; use crate::convert_required; use crate::physical_plan::from_proto::{ - parse_physical_expr_with_converter, parse_physical_sort_expr, - parse_physical_sort_exprs, parse_protobuf_file_scan_config, parse_record_batches, - parse_table_schema_from_proto, + parse_physical_expr_with_converter, parse_physical_sort_exprs, + parse_protobuf_file_scan_config, parse_record_batches, parse_table_schema_from_proto, }; use crate::physical_plan::to_proto::{ - serialize_file_scan_config, serialize_maybe_filter, serialize_physical_aggr_expr, - serialize_physical_expr_with_converter, serialize_physical_sort_exprs, - serialize_record_batches, + serialize_file_scan_config, serialize_physical_expr_with_converter, + serialize_physical_sort_exprs, serialize_record_batches, }; -use crate::protobuf::physical_aggregate_expr_node::AggregateFunction; -use crate::protobuf::physical_expr_node::ExprType; use crate::protobuf::physical_plan_node::PhysicalPlanType; use crate::protobuf::{self, SortMergeJoinExecNode, proto_error}; @@ -129,48 +121,10 @@ fn encode_human_display_alias(human_display: &str, alias: &str) -> String { ) } -fn split_human_display_alias<'a>( - human_display: &'a str, - name: &'a str, -) -> (&'a str, Option<&'a str>) { - if let Some(encoded) = human_display.strip_prefix(HUMAN_DISPLAY_ALIAS_PREFIX) - && let Some((alias_len, encoded)) = encoded.split_once(':') - && let Ok(alias_len) = alias_len.parse::() - && let Some(alias) = encoded.get(..alias_len) - && let Some(human_display) = encoded.get(alias_len..) - && alias == name - && !human_display.is_empty() - { - return (human_display, Some(alias)); - } - - (human_display, None) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn split_human_display_alias_ignores_mismatched_alias() { - let encoded = encode_human_display_alias("sum(value)", "revenue"); - - assert_eq!( - split_human_display_alias(&encoded, "other"), - (encoded.as_str(), None) - ); - } - - #[test] - fn split_human_display_alias_keeps_malformed_prefix_literal() { - let display = format!("{HUMAN_DISPLAY_ALIAS_PREFIX}not-an-encoding"); - - assert_eq!( - split_human_display_alias(&display, "agg"), - (display.as_str(), None) - ); - } - /// Unit tests for the bytes-only function serde exposed on /// [`ExecutionPlanEncodeCtx`] / [`ExecutionPlanDecodeCtx`] and backed by /// [`ConverterPlanEncoder`] / [`ConverterPlanDecoder`]. Function-carrying @@ -790,8 +744,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::Window(_) => { WindowAggExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::Aggregate(hash_agg) => { - self.try_into_aggregate_physical_plan(hash_agg, ctx, proto_converter) + PhysicalPlanType::Aggregate(_) => { + AggregateExec::try_from_proto(self.node(), &decode_ctx) } PhysicalPlanType::HashJoin(_) => { HashJoinExec::try_from_proto(self.node(), &decode_ctx) @@ -891,14 +845,6 @@ pub trait PhysicalPlanNodeExt: Sized { return Ok(node); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_aggregate_exec( - exec, - codec, - proto_converter, - ); - } - if let Some(data_source_exec) = plan.downcast_ref::() && let Some(node) = protobuf::PhysicalPlanNode::try_from_data_source_exec( data_source_exec, @@ -1412,212 +1358,27 @@ pub trait PhysicalPlanNodeExt: Sized { WindowAggExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AggregateExec` deserializes itself via `AggregateExec::try_from_proto`" + )] fn try_into_aggregate_physical_plan( &self, hash_agg: &protobuf::AggregateExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let input: Arc = - into_physical_plan(&hash_agg.input, ctx, proto_converter)?; - let mode = protobuf::AggregateMode::try_from(hash_agg.mode).map_err(|_| { - proto_error(format!( - "Received a AggregateNode message with unknown AggregateMode {}", - hash_agg.mode - )) - })?; - let agg_mode: AggregateMode = match mode { - protobuf::AggregateMode::Partial => AggregateMode::Partial, - protobuf::AggregateMode::Final => AggregateMode::Final, - protobuf::AggregateMode::FinalPartitioned => AggregateMode::FinalPartitioned, - protobuf::AggregateMode::Single => AggregateMode::Single, - protobuf::AggregateMode::SinglePartitioned => { - AggregateMode::SinglePartitioned - } - protobuf::AggregateMode::PartialReduce => AggregateMode::PartialReduce, - }; - - let num_expr = hash_agg.group_expr.len(); - - let group_expr = hash_agg - .group_expr - .iter() - .zip(hash_agg.group_expr_name.iter()) - .map(|(expr, name)| { - proto_converter - .proto_to_physical_expr(expr, input.schema().as_ref(), ctx) - .map(|expr| (expr, name.to_string())) - }) - .collect::, _>>()?; - - let null_expr = hash_agg - .null_expr - .iter() - .zip(hash_agg.group_expr_name.iter()) - .map(|(expr, name)| { - proto_converter - .proto_to_physical_expr(expr, input.schema().as_ref(), ctx) - .map(|expr| (expr, name.to_string())) - }) - .collect::, _>>()?; - - let groups: Vec> = if !hash_agg.groups.is_empty() { - hash_agg - .groups - .chunks(num_expr) - .map(|g| g.to_vec()) - .collect::>>() - } else { - vec![] - }; - - let has_grouping_set = hash_agg.has_grouping_set; - - let input_schema = hash_agg.input_schema.as_ref().ok_or_else(|| { - internal_datafusion_err!("input_schema in AggregateNode is missing.") - })?; - let physical_schema: SchemaRef = SchemaRef::new(input_schema.try_into()?); - - let physical_filter_expr = hash_agg - .filter_expr - .iter() - .map(|expr| { - expr.expr - .as_ref() - .map(|e| { - proto_converter.proto_to_physical_expr(e, &physical_schema, ctx) - }) - .transpose() - }) - .collect::, _>>()?; - - let physical_aggr_expr: Vec> = hash_agg - .aggr_expr - .iter() - .zip(hash_agg.aggr_expr_name.iter()) - .map(|(expr, name)| { - let expr_type = expr.expr_type.as_ref().ok_or_else(|| { - proto_error("Unexpected empty aggregate physical expression") - })?; - - match expr_type { - ExprType::AggregateExpr(agg_node) => { - let input_phy_expr: Vec> = agg_node - .expr - .iter() - .map(|e| { - proto_converter.proto_to_physical_expr( - e, - &physical_schema, - ctx, - ) - }) - .collect::>>()?; - let order_bys = agg_node - .ordering_req - .iter() - .map(|e| { - parse_physical_sort_expr( - e, - ctx, - &physical_schema, - proto_converter, - ) - }) - .collect::>()?; - agg_node - .aggregate_function - .as_ref() - .map(|func| match func { - AggregateFunction::UserDefinedAggrFunction(udaf_name) => { - let agg_udf = match &agg_node.fun_definition { - Some(buf) => { - ctx.codec().try_decode_udaf(udaf_name, buf)? - } - None => ctx.task_ctx().udaf(udaf_name).or_else( - |_| { - ctx.codec() - .try_decode_udaf(udaf_name, &[]) - }, - )?, - }; - - let (human_display, human_display_alias) = - split_human_display_alias( - &agg_node.human_display, - name, - ); - let builder = AggregateExprBuilder::new( - agg_udf, - input_phy_expr, - ) - .schema(Arc::clone(&physical_schema)) - .alias(name) - .with_ignore_nulls(agg_node.ignore_nulls) - .with_distinct(agg_node.distinct) - .order_by(order_bys) - .human_display(human_display); - let builder = if let Some(alias) = human_display_alias - { - builder.human_display_alias(alias) - } else { - builder - }; - builder.build().map(Arc::new) - } - }) - .transpose()? - .ok_or_else(|| { - proto_error( - "Invalid AggregateExpr, missing aggregate_function", - ) - }) - } - _ => internal_err!("Invalid aggregate expression for AggregateExec"), - } - }) - .collect::, _>>()?; - - let physical_schema_ref = Arc::clone(&physical_schema); - let agg = AggregateExec::try_new( - agg_mode, - PhysicalGroupBy::new(group_expr, null_expr, groups, has_grouping_set), - physical_aggr_expr, - physical_filter_expr, - input, - physical_schema, - )?; - - let agg = if let Some(limit_proto) = &hash_agg.limit { - let limit = limit_proto.limit as usize; - let limit_options = match limit_proto.descending { - Some(descending) => LimitOptions::new_with_order(limit, descending), - None => LimitOptions::new(limit), - }; - agg.with_limit_options(Some(limit_options)) - } else { - agg + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::Aggregate(Box::new( + hash_agg.clone(), + ))), }; - - let agg = if let Some(dynamic_filter_proto) = &hash_agg.dynamic_filter { - let dynamic_filter_expr = proto_converter.proto_to_physical_expr( - dynamic_filter_proto, - physical_schema_ref.as_ref(), - ctx, - )?; - let df = (dynamic_filter_expr as Arc) - .downcast::() - .map_err(|_| { - internal_datafusion_err!( - "AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr" - ) - })?; - agg.with_dynamic_filter_expr(df)? - } else { - agg + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, }; - - Ok(Arc::new(agg)) + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + AggregateExec::try_from_proto(&node, &decode_ctx) } #[deprecated( @@ -2410,108 +2171,22 @@ pub trait PhysicalPlanNodeExt: Sized { .ok_or_else(|| internal_datafusion_err!("CrossJoinExec is not serializable")) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `AggregateExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_aggregate_exec( exec: &AggregateExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let groups: Vec = exec - .group_expr() - .groups() - .iter() - .flatten() - .copied() - .collect(); - - let group_names = exec - .group_expr() - .expr() - .iter() - .map(|expr| expr.1.to_owned()) - .collect(); - - let filter = exec - .filter_expr() - .iter() - .map(|expr| serialize_maybe_filter(expr.to_owned(), codec, proto_converter)) - .collect::>>()?; - - let agg = exec - .aggr_expr() - .iter() - .map(|expr| { - serialize_physical_aggr_expr(expr.to_owned(), codec, proto_converter) - }) - .collect::>>()?; - - let agg_names = exec - .aggr_expr() - .iter() - .map(|expr| expr.name().to_string()) - .collect::>(); - - let agg_mode = match exec.mode() { - AggregateMode::Partial => protobuf::AggregateMode::Partial, - AggregateMode::Final => protobuf::AggregateMode::Final, - AggregateMode::FinalPartitioned => protobuf::AggregateMode::FinalPartitioned, - AggregateMode::Single => protobuf::AggregateMode::Single, - AggregateMode::SinglePartitioned => { - protobuf::AggregateMode::SinglePartitioned - } - AggregateMode::PartialReduce => protobuf::AggregateMode::PartialReduce, - }; - let input_schema = exec.input_schema(); - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - exec.input().to_owned(), + let encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - - let null_expr = exec - .group_expr() - .null_expr() - .iter() - .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec)) - .collect::>>()?; - - let group_expr = exec - .group_expr() - .expr() - .iter() - .map(|expr| proto_converter.physical_expr_to_proto(&expr.0, codec)) - .collect::>>()?; - - let limit = exec.limit_options().map(|config| protobuf::AggLimit { - limit: config.limit() as u64, - descending: config.descending(), - }); - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::Aggregate(Box::new( - protobuf::AggregateExecNode { - group_expr, - group_expr_name: group_names, - aggr_expr: agg, - filter_expr: filter, - aggr_expr_name: agg_names, - mode: agg_mode as i32, - input: Some(Box::new(input)), - input_schema: Some(input_schema.as_ref().try_into()?), - null_expr, - groups, - limit, - has_grouping_set: exec.group_expr().has_grouping_set(), - dynamic_filter: exec - .dynamic_filter_expr() - .map(|df| { - let df_expr: Arc = - Arc::clone(df) as Arc; - proto_converter.physical_expr_to_proto(&df_expr, codec) - }) - .transpose()?, - }, - ))), - }) + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)? + .ok_or_else(|| internal_datafusion_err!("AggregateExec is not serializable")) } #[deprecated( From abec3115b47d2c8ec89a20949f5cbcfe3efec8d4 Mon Sep 17 00:00:00 2001 From: Naman Modi Date: Sat, 25 Jul 2026 16:08:23 +0530 Subject: [PATCH 021/109] refactor(unparser): centralize aggregate-scope rendering in the SQL unparser (#23789) ## Which issue does this PR close? - Closes #23668. ## Rationale for this change - When printing a GROUP BY query, the unparser sometimes wraps the aggregate's input in an inner subquery (`... FROM (SELECT ...)`). - Table aliases like `cs` only exist inside that subquery. Any clause outside it (SELECT, GROUP BY, HAVING, QUALIFY, ORDER BY) must use the subquery's output columns, not the aliases, so the unparser drops the alias. - Each clause does that dropping on its own, so it is easy to miss one. - ORDER BY on an aggregate that is not in the SELECT list was missed: it printed `ORDER BY round(sum("cs"."total_revenue"), 2)`, while the SELECT list in the same query correctly printed `sum("total_revenue")`. - DataFusion reads that SQL back fine, but stricter databases reject it because `cs` is out of scope there. ## What changes are included in this PR? I moved the rule into one helper, `UnparserAggScope`, that checks once per aggregate whether the input is an inner subquery and then prepares expressions for each clause. SELECT, GROUP BY, HAVING, QUALIFY, and both ORDER BY paths now go through it, which fixes the ORDER BY case. The window-over-aggregate path is left as-is with a comment: it is only reachable from hand-built plans, since a window in SQL always sits inside a SELECT that already drops the alias. Only ORDER BY output changes; every already-correct case stays the same. ## Are these changes tested? Yes. New tests in `datafusion/core/tests/sql/unparser.rs` cover the inner-subquery shape for a window sorting by an aggregate, ORDER BY on an unselected aggregate (the fixed case), and a top-level ORDER BY (the second sort path). Existing unparser tests and the TPC-H/Clickbench roundtrips still pass. ## Are there any user-facing changes? ORDER BY over an aggregate is now printed without an out-of-scope table qualifier, so the generated SQL is valid for stricter databases. No API changes. --- datafusion/core/tests/sql/unparser.rs | 135 +++++++++++++++++++++++ datafusion/sql/src/unparser/plan.rs | 151 ++++++++++++++++---------- 2 files changed, 231 insertions(+), 55 deletions(-) diff --git a/datafusion/core/tests/sql/unparser.rs b/datafusion/core/tests/sql/unparser.rs index d689fb1496a2b..355a58fd6f45b 100644 --- a/datafusion/core/tests/sql/unparser.rs +++ b/datafusion/core/tests/sql/unparser.rs @@ -468,6 +468,59 @@ QUALIFY rn = 1 AND count(DISTINCT cs.customer_id) > 0 "#; +// https://github.com/apache/datafusion/issues/23668 +// +// Extends the #23317 aggregate-scope fix to the window and ORDER BY clauses. +// Reuses issue_23317_context() (same derived-projection shape). + +// Window sorting by an aggregate, over a derived-projection input. Already +// correct today; this locks the OVER clause against keeping the out-of-scope +// `cs` qualifier across the refactor. +const ISSUE_23668_WINDOW_QUERY: &str = r#" +SELECT + date_part('year', c.signup_date) AS signup_year, + count(DISTINCT cs.customer_id) AS customers, + row_number() OVER (ORDER BY count(DISTINCT cs.customer_id) DESC) AS rn +FROM + "warehouse"."main"."sales" cs + JOIN "warehouse"."main"."customers" c USING (customer_id) +GROUP BY + 1 +"#; + +// ORDER BY an aggregate that is NOT selected, so it can't use a select alias +// and is unprojected through the Aggregate. It must be normalized like the +// SELECT list, not keep the out-of-scope `cs` qualifier. +const ISSUE_23668_ORDER_BY_QUERY: &str = r#" +SELECT + date_part('year', c.signup_date) AS signup_year, + count(DISTINCT cs.customer_id) AS customers +FROM + "warehouse"."main"."sales" cs + JOIN "warehouse"."main"."customers" c USING (customer_id) +GROUP BY + 1 +ORDER BY + round(sum(cs.total_revenue), 2) DESC +"#; + +// ORDER BY a selected aggregate keeps a top-level Sort (the direct `Sort` arm, +// vs the projection-absorbed one above). It resolves to the select alias, so +// this covers routing only -- the normalization in that arm isn't reachable +// from SQL (an unselected aggregate takes the absorbed path above instead). +const ISSUE_23668_TOP_LEVEL_SORT_QUERY: &str = r#" +SELECT + date_part('year', c.signup_date) AS signup_year, + count(DISTINCT cs.customer_id) AS customers +FROM + "warehouse"."main"."sales" cs + JOIN "warehouse"."main"."customers" c USING (customer_id) +GROUP BY + 1 +ORDER BY + customers DESC +"#; + fn issue_23317_context() -> Result { let ctx = SessionContext::new(); @@ -612,6 +665,88 @@ async fn optimized_duckdb_unparse_qualify_unqualifies_agg_input() -> Result<()> Ok(()) } +#[tokio::test] +async fn optimized_duckdb_unparse_window_over_agg_unqualifies_input() -> Result<()> { + let ctx = issue_23317_context()?; + assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name())); + + let plan = ctx + .sql(ISSUE_23668_WINDOW_QUERY) + .await? + .into_optimized_plan()?; + let dialect = DuckDBDialect::new(); + let unparser = Unparser::new(&dialect); + let sql = unparser.plan_to_sql(&plan)?.to_string(); + + assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?; + + assert!( + sql.contains(r#"OVER (ORDER BY count(DISTINCT "customer_id")"#), + "window ORDER BY aggregate should resolve against the derived projection output: {sql}", + ); + assert!( + !sql.contains(r#"count(DISTINCT "cs"."customer_id")"#), + "window OVER clause must not reference out-of-scope alias cs: {sql}", + ); + + Ok(()) +} + +#[tokio::test] +async fn optimized_duckdb_unparse_order_by_unqualifies_agg_input() -> Result<()> { + let ctx = issue_23317_context()?; + assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name())); + + let plan = ctx + .sql(ISSUE_23668_ORDER_BY_QUERY) + .await? + .into_optimized_plan()?; + let dialect = DuckDBDialect::new(); + let unparser = Unparser::new(&dialect); + let sql = unparser.plan_to_sql(&plan)?.to_string(); + + assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?; + + assert!( + sql.contains(r#"ORDER BY round(sum("total_revenue"), 2)"#), + "ORDER BY aggregate should resolve against the derived projection output: {sql}", + ); + assert!( + !sql.contains(r#"sum("cs"."total_revenue")"#), + "ORDER BY must not reference out-of-scope alias cs: {sql}", + ); + + Ok(()) +} + +#[tokio::test] +async fn optimized_duckdb_unparse_top_level_sort_over_agg_uses_select_alias() -> Result<()> +{ + let ctx = issue_23317_context()?; + assert!(ctx.remove_optimizer_rule(SingleDistinctToGroupBy::new().name())); + + let plan = ctx + .sql(ISSUE_23668_TOP_LEVEL_SORT_QUERY) + .await? + .into_optimized_plan()?; + let dialect = DuckDBDialect::new(); + let unparser = Unparser::new(&dialect); + let sql = unparser.plan_to_sql(&plan)?.to_string(); + + assert_issue_23317_unparsed_sql_plans(&ctx, &sql).await?; + + assert!( + sql.contains(r#"ORDER BY "customers""#), + "top-level ORDER BY should resolve to the select alias: {sql}", + ); + assert!( + !sql.contains(r#""cs"."customer_id") AS "customers""#), + "aggregate output must not reference out-of-scope alias cs: {sql}", + ); + + Ok(()) +} + /// The outcome of running a single roundtrip test. /// /// A successful test produces [`TestCaseResult::Success`]. diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 5eef9b82d975e..9fe97a8291b6a 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -101,6 +101,69 @@ pub fn plan_to_sql(plan: &LogicalPlan) -> Result { unparser.plan_to_sql(plan) } +/// Aggregate-expression scope for one rendered SELECT block. +/// +/// When an aggregate's input is itself emitted as a derived subquery (a +/// projection sits between the aggregate and its relation), the input columns +/// are only reachable by that derived table's output names. Base-table +/// qualifiers like `t.col` name a relation that is out of scope above the +/// boundary, so emitting them produces SQL a strict engine rejects. +/// +/// Every clause that renders an aggregate expression (SELECT / GROUP BY / +/// HAVING / QUALIFY / ORDER BY) has to apply the same rule. Detect the +/// boundary once here and reuse it, so the clauses can't drift apart (which is +/// how earlier fixes left some clauses correct and others not). +struct UnparserAggScope<'a> { + agg: &'a Aggregate, + /// `agg.input` renders as a derived projection, so out-of-scope qualifiers + /// must be stripped from expressions in this scope. + input_is_derived_projection: bool, +} + +impl<'a> UnparserAggScope<'a> { + fn new(agg: &'a Aggregate) -> Self { + Self { + agg, + input_is_derived_projection: Unparser::contains_projection_before_relation( + agg.input.as_ref(), + ), + } + } + + /// Prepare a projected column or predicate that still references the + /// aggregate by its output columns: unproject it back onto the aggregate + /// (and `windows`) expressions, then normalize it for this scope. + fn prepare(&self, expr: Expr, windows: Option<&[&Window]>) -> Result { + self.normalize(unproject_agg_exprs(expr, self.agg, windows)?) + } + + /// Normalize an expression that is already in aggregate form (group / aggr + /// exprs, or an unprojected sort expr): strip the qualifiers that fall out + /// of scope once the input is a derived projection. No-op otherwise. + fn normalize(&self, expr: Expr) -> Result { + if self.input_is_derived_projection { + Unparser::strip_column_qualifiers_for_schema( + expr, + self.agg.input.schema().as_ref(), + ) + } else { + Ok(expr) + } + } + + /// Unproject a sort expression onto this aggregate, then normalize it so + /// ORDER BY uses the same scope as the other clauses. + fn prepare_sort_expr( + &self, + sort_expr: SortExpr, + input: &LogicalPlan, + ) -> Result { + let mut sort_expr = unproject_sort_expr(sort_expr, Some(self.agg), input)?; + sort_expr.expr = self.normalize(sort_expr.expr)?; + Ok(sort_expr) + } +} + impl Unparser<'_> { pub fn plan_to_sql(&self, plan: &LogicalPlan) -> Result { let mut plan = normalize_union_schema(plan)?; @@ -312,17 +375,12 @@ impl Unparser<'_> { match (agg, window) { (Some(agg), window) => { let window_option = window.as_deref(); - let agg_input_has_derived_projection = - Self::contains_projection_before_relation(agg.input.as_ref()); + let unparser_agg_scope = UnparserAggScope::new(agg); let items = exprs .into_iter() .map(|proj_expr| { - let unproj = unproject_agg_exprs(proj_expr, agg, window_option)?; - let unproj = Self::normalize_agg_input_columns( - unproj, - agg, - agg_input_has_derived_projection, - )?; + let unproj = + unparser_agg_scope.prepare(proj_expr, window_option)?; self.select_item_to_sql(&unproj) }) .collect::>>()?; @@ -333,12 +391,7 @@ impl Unparser<'_> { .iter() .cloned() .map(|expr| { - let expr = Self::normalize_agg_input_columns( - expr, - agg, - agg_input_has_derived_projection, - )?; - self.expr_to_sql(&expr) + self.expr_to_sql(&unparser_agg_scope.normalize(expr)?) }) .collect::>>()?, vec![], @@ -379,18 +432,6 @@ impl Unparser<'_> { } } - fn normalize_agg_input_columns( - expr: Expr, - agg: &Aggregate, - input_has_derived_projection: bool, - ) -> Result { - if input_has_derived_projection { - Self::strip_column_qualifiers_for_schema(expr, agg.input.schema().as_ref()) - } else { - Ok(expr) - } - } - fn contains_projection_before_relation(plan: &LogicalPlan) -> bool { match plan { LogicalPlan::Projection(_) => true, @@ -429,6 +470,19 @@ impl Unparser<'_> { } } + /// Unproject a sort expression; normalize it when the sort is above an + /// aggregate, otherwise just unproject (no scope to normalize against). + fn unproject_sort_expr_in_scope( + sort_expr: SortExpr, + agg: Option<&Aggregate>, + input: &LogicalPlan, + ) -> Result { + match agg { + Some(agg) => UnparserAggScope::new(agg).prepare_sort_expr(sort_expr, input), + None => unproject_sort_expr(sort_expr, None, input), + } + } + fn derive( &self, plan: &LogicalPlan, @@ -592,6 +646,9 @@ impl Unparser<'_> { window_expr .iter() .map(|expr| { + // No normalization: this agg branch is only reachable from a + // hand-built plan. SQL wraps windows in a projection, which + // reconstruct_select_statement handles (and normalizes). let expr = if let Some(agg) = agg { unproject_agg_exprs(expr.clone(), agg, None)? } else { @@ -977,7 +1034,7 @@ impl Unparser<'_> { sort.expr .iter() .map(|sort_expr| { - unproject_sort_expr( + Self::unproject_sort_expr_in_scope( sort_expr.clone(), agg, sort.input.as_ref(), @@ -1028,23 +1085,14 @@ impl Unparser<'_> { let mut unprojected = unproject_window_exprs(filter.predicate.clone(), window)?; if let Some(agg) = agg { - unprojected = unproject_agg_exprs(unprojected, agg, None)?; - unprojected = Self::normalize_agg_input_columns( - unprojected, - agg, - Self::contains_projection_before_relation(agg.input.as_ref()), - )?; + unprojected = + UnparserAggScope::new(agg).prepare(unprojected, None)?; } let filter_expr = self.expr_to_sql(&unprojected)?; select.qualify(Some(filter_expr)); } else if let Some(agg) = agg { - let unprojected = - unproject_agg_exprs(filter.predicate.clone(), agg, None)?; - let unprojected = Self::normalize_agg_input_columns( - unprojected, - agg, - Self::contains_projection_before_relation(agg.input.as_ref()), - )?; + let unprojected = UnparserAggScope::new(agg) + .prepare(filter.predicate.clone(), None)?; let filter_expr = self.expr_to_sql(&unprojected)?; select.having(Some(filter_expr)); } else { @@ -1130,7 +1178,11 @@ impl Unparser<'_> { .expr .iter() .map(|sort_expr| { - unproject_sort_expr(sort_expr.clone(), agg, sort.input.as_ref()) + Self::unproject_sort_expr_in_scope( + sort_expr.clone(), + agg, + sort.input.as_ref(), + ) }) .collect::>>()?; @@ -1146,8 +1198,7 @@ impl Unparser<'_> { LogicalPlan::Aggregate(agg) => { // Aggregation can be already handled in the projection case if !select.already_projected() { - let agg_input_has_derived_projection = - Self::contains_projection_before_relation(agg.input.as_ref()); + let unparser_agg_scope = UnparserAggScope::new(agg); // The query returns aggregate and group expressions. If that weren't the case, // the aggregate would have been placed inside a projection, making the check above^ false let exprs: Vec<_> = agg @@ -1156,12 +1207,7 @@ impl Unparser<'_> { .chain(agg.group_expr.iter()) .cloned() .map(|expr| { - let expr = Self::normalize_agg_input_columns( - expr, - agg, - agg_input_has_derived_projection, - )?; - self.select_item_to_sql(&expr) + self.select_item_to_sql(&unparser_agg_scope.normalize(expr)?) }) .collect::>>()?; select.projection(exprs); @@ -1171,12 +1217,7 @@ impl Unparser<'_> { .iter() .cloned() .map(|expr| { - let expr = Self::normalize_agg_input_columns( - expr, - agg, - agg_input_has_derived_projection, - )?; - self.expr_to_sql(&expr) + self.expr_to_sql(&unparser_agg_scope.normalize(expr)?) }) .collect::>>()?, vec![], From f1ab86dad406a189e43ac19d24965b3fbf9dbba9 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 25 Jul 2026 18:39:02 +0800 Subject: [PATCH 022/109] Add FixedSizeList support for recursive struct schema adaptation (#22980) ## Which issue does this PR close? * Part of #20835 ## Rationale for this change `FixedSizeList` containing `Struct` values was not handled by the existing recursive nested adaptation logic used for schema evolution. As a result, planner-time compatibility checks, nested cast detection, and runtime casting did not support additive struct evolution within `FixedSizeList` containers. This change adds `FixedSizeList` support and verifies planner/runtime parity so that planning allows exactly the cases runtime can adapt while continuing to reject incompatible schema changes. ## What changes are included in this PR? * Extend `cast_column` to support recursive casting of `FixedSizeList` values when source and target list sizes match. * Add `FixedSizeList` handling to: * `requires_nested_struct_cast` * `validate_data_type_compatibility` * Implement recursive casting of nested `Struct` values contained in `FixedSizeList`. * Preserve planner/runtime parity by validating child type compatibility before runtime fallback logic is applied. * Add handling for null-parent `FixedSizeList` entries by masking hidden child values before retrying casts, avoiding failures caused by semantically inaccessible child data. * Refactor list and list-view casting helpers to use Arrow `AsArray` accessors. ## Are these changes tested? Yes. The following tests were added: * `test_cast_fixed_size_list_struct` * `test_validate_fixed_size_list_struct_compatibility` * `test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected` * `test_validate_fixed_size_list_struct_size_mismatch_rejected` * `test_cast_fixed_size_list_struct_all_null` * `test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type` * `test_cast_fixed_size_list_struct_missing_non_nullable_field_runtime_rejected` * `test_cast_fixed_size_list_struct_ignores_hidden_child_values_for_null_parent` Existing coverage in `test_requires_nested_struct_cast` was also extended to include `FixedSizeList` cases. These tests cover: * Additive nullable nested-field evolution * All-null and partially null list cases * Incompatible nested type changes * Non-nullable field addition rejection * Planner/runtime parity validation ## Are there any user-facing changes? No user-facing changes. This is an internal enhancement to nested schema adaptation and casting behavior for `FixedSizeList` types. ## LLM-generated code disclosure This PR includes LLM-generated code and comments. All LLM-generated content has been manually reviewed. --------- Co-authored-by: Andrew Lamb --- datafusion/common/src/nested_struct.rs | 424 ++++++++++++++++-- datafusion/core/tests/parquet/expr_adapter.rs | 124 ++++- 2 files changed, 496 insertions(+), 52 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index cdd6215d08e2f..e915b91b911cc 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -18,9 +18,10 @@ use crate::error::{_plan_err, Result}; use arrow::{ array::{ - Array, ArrayRef, DictionaryArray, GenericListArray, GenericListViewArray, - StructArray, downcast_integer, new_null_array, + Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray, + GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array, }, + buffer::NullBuffer, compute::{CastOptions, can_cast_types, cast_with_options}, datatypes::{DataType, DataType::Struct, Field, FieldRef}, }; @@ -58,9 +59,7 @@ fn cast_struct_column( target_fields: &[Arc], cast_options: &CastOptions, ) -> Result { - if source_col.data_type() == &DataType::Null - || (!source_col.is_empty() && source_col.null_count() == source_col.len()) - { + if source_col.data_type() == &DataType::Null { return Ok(new_null_array( &Struct(target_fields.to_vec().into()), source_col.len(), @@ -70,6 +69,14 @@ fn cast_struct_column( if let Some(source_struct) = source_col.as_any().downcast_ref::() { let source_fields = source_struct.fields(); validate_struct_compatibility(source_fields, target_fields)?; + + if !source_col.is_empty() && source_col.null_count() == source_col.len() { + return Ok(new_null_array( + &Struct(target_fields.to_vec().into()), + source_col.len(), + )); + } + let mut fields: Vec> = Vec::with_capacity(target_fields.len()); let mut arrays: Vec = Vec::with_capacity(target_fields.len()); let num_rows = source_col.len(); @@ -183,6 +190,15 @@ pub fn cast_column( (DataType::LargeList(_), DataType::LargeList(target_inner)) => { cast_list_column::(source_col, target_inner, cast_options) } + ( + DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(target_inner, target_list_size), + ) if source_list_size == target_list_size => cast_fixed_size_list_column( + source_col, + target_inner, + *target_list_size, + cast_options, + ), (DataType::ListView(_), DataType::ListView(target_inner)) => { cast_list_view_column::(source_col, target_inner, cast_options) } @@ -208,15 +224,7 @@ fn cast_list_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list array but got {}", - source_col.data_type() - )) - })?; + let source_list = source_col.as_list::(); let cast_values = cast_column( source_list.values(), @@ -238,15 +246,7 @@ fn cast_list_view_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list view array but got {}", - source_col.data_type() - )) - })?; + let source_list = source_col.as_list_view::(); let cast_values = cast_column( source_list.values(), @@ -264,6 +264,82 @@ fn cast_list_view_column( Ok(Arc::new(result)) } +fn cast_fixed_size_list_column( + source_col: &ArrayRef, + target_inner_field: &FieldRef, + target_list_size: i32, + cast_options: &CastOptions, +) -> Result { + let source_list = source_col.as_fixed_size_list(); + + let source_values = source_list.values(); + let target_type = target_inner_field.data_type(); + + let cast_values = match cast_column(source_values, target_type, cast_options) { + Ok(cast_values) => cast_values, + Err(error) => match cast_fixed_size_list_values_with_parent_nulls( + source_values, + target_type, + cast_options, + source_list.nulls(), + target_list_size, + ) { + Some(masked_cast) => masked_cast?, + None => return Err(error), + }, + }; + + Ok(Arc::new(FixedSizeListArray::try_new( + Arc::clone(target_inner_field), + target_list_size, + cast_values, + source_list.nulls().cloned(), + )?)) +} + +fn cast_fixed_size_list_values_with_parent_nulls( + source_values: &ArrayRef, + target_type: &DataType, + cast_options: &CastOptions, + parent_nulls: Option<&NullBuffer>, + list_size: i32, +) -> Option> { + let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; + + // FixedSizeList stores child slots for null parent lists. Those child + // values are semantically hidden, but recursive casts still inspect them. + let hidden_child_nulls = parent_nulls.expand(list_size as usize); + let masked_values = mask_array_values(source_values, &hidden_child_nulls); + Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) +} + +fn mask_array_values( + values: &ArrayRef, + additional_nulls: &NullBuffer, +) -> Result { + let nulls = NullBuffer::union(values.nulls(), Some(additional_nulls)); + + if let Some(struct_array) = values.as_any().downcast_ref::() { + let struct_nulls = nulls + .as_ref() + .expect("additional nulls always produce nulls"); + let arrays = struct_array + .columns() + .iter() + .map(|child| mask_array_values(child, struct_nulls)) + .collect::>>()?; + return Ok(Arc::new(StructArray::new( + struct_array.fields().clone(), + arrays, + nulls, + ))); + } + + Ok(make_array( + values.to_data().into_builder().nulls(nulls).build()?, + )) +} + fn cast_dictionary_column( source_col: &ArrayRef, source_key_type: &DataType, @@ -425,6 +501,12 @@ pub fn validate_data_type_compatibility( (Struct(source_nested), Struct(target_nested)) => { validate_struct_compatibility(source_nested, target_nested)?; } + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + validate_field_compatibility(s, t)?; + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) @@ -460,8 +542,8 @@ pub fn validate_data_type_compatibility( /// name-based nested struct casting logic, rather than Arrow's standard cast. /// /// This is the case when both types are struct types, or both are the same -/// container type (List, LargeList, ListView, LargeListView, Dictionary) wrapping -/// types that recursively contain structs. +/// container type (List, LargeList, equal-width FixedSizeList, ListView, +/// LargeListView, Dictionary) wrapping types that recursively contain structs. /// /// Use this predicate at both planning time (to decide whether to apply struct /// compatibility validation) and execution time (to decide whether to route @@ -472,6 +554,12 @@ pub fn requires_nested_struct_cast( ) -> bool { match (source_type, target_type) { (Struct(_), Struct(_)) => true, + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + requires_nested_struct_cast(s.data_type(), t.data_type()) + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) @@ -508,8 +596,9 @@ mod tests { use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS}; use arrow::{ array::{ - BinaryArray, Int32Array, Int32Builder, Int64Array, ListArray, ListViewArray, - MapArray, MapBuilder, NullArray, StringArray, StringBuilder, + BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array, + ListArray, ListViewArray, MapArray, MapBuilder, NullArray, StringArray, + StringBuilder, }, buffer::{NullBuffer, ScalarBuffer}, datatypes::{DataType, Field, FieldRef, Int32Type}, @@ -1307,6 +1396,275 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } + fn fixed_size_list_struct_field(fields: Vec<(&str, DataType)>) -> FieldRef { + arc_field( + "item", + struct_type( + fields + .into_iter() + .map(|(name, data_type)| field(name, data_type)) + .collect(), + ), + ) + } + + fn create_fixed_size_list_test_fields( + source_struct_fields: Vec<(&str, DataType)>, + target_struct_fields: Vec<(&str, DataType)>, + ) -> (FieldRef, FieldRef) { + ( + fixed_size_list_struct_field(source_struct_fields), + fixed_size_list_struct_field(target_struct_fields), + ) + } + + fn fixed_size_list_struct_values( + array: &ArrayRef, + ) -> (&FixedSizeListArray, &StructArray) { + let list = array.as_any().downcast_ref::().unwrap(); + let values = list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + (list, values) + } + + #[test] + fn test_cast_fixed_size_list_struct() { + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + )]); + + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false])), + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert_eq!(result_list.len(), 2); + assert!(result_list.is_valid(0)); + assert!(result_list.is_null(1)); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + assert_eq!(a_col.values(), &[1, 2, 3, 4]); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_validate_fixed_size_list_struct_compatibility() { + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList(target_field, 2); + + assert!(requires_nested_struct_cast(&source, &target)); + assert!(validate_data_type_compatibility("col", &source, &target).is_ok()); + } + + #[test] + fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() { + let (source_field, _) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList( + arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ), + 2, + ); + + let error = validate_data_type_compatibility("col", &source, &target) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_fixed_size_list_struct_size_mismatch_rejected() { + let source_field = fixed_size_list_struct_field(vec![("a", DataType::Int32)]); + let target_field = Arc::clone(&source_field); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 3); + + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'col'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!( + runtime_error, + "cannot cast fixed-size-list to fixed-size-list with different size" + ); + } + + #[test] + fn test_cast_fixed_size_list_struct_all_null() { + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 2)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert_eq!(result_list.null_count(), 2); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(a_col.iter().all(|v| v.is_none())); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Binary)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 2); + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'a'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Binary), + Arc::new(BinaryArray::from(vec![ + Some(b"x".as_ref()), + Some(b"y".as_ref()), + ])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(runtime_error, "Cannot cast struct field 'a'"); + } + + #[test] + fn test_cast_fixed_size_list_struct_missing_non_nullable_field_runtime_rejected() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let target_field = arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 1)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_cast_fixed_size_list_returns_error_for_non_nullable_child() { + let source_field = Arc::new(Field::new("item", DataType::Int32, true)); + let target_field = Arc::new(Field::new("item", DataType::Int32, false)); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(Int32Array::from(vec![None, Some(1)])), + None, + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(error, "Found unmasked nulls for non-nullable"); + } + + #[test] + fn test_cast_sliced_fixed_size_list_struct_ignores_hidden_child_values() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Utf8)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Utf8), + Arc::new(StringArray::from(vec![ + "0", "0", "not_int", "also_bad", "1", "2", + ])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new( + FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false, true])), + ) + .slice(1, 2), + ); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert!(result_list.is_null(0)); + assert!(result_list.is_valid(1)); + let a_col = get_column_as!(&struct_values, "a", Int32Array); + assert!(a_col.is_null(0)); + assert!(a_col.is_null(1)); + assert_eq!(a_col.value(2), 1); + assert_eq!(a_col.value(3), 2); + } + #[test] fn test_requires_nested_struct_cast() { let s1 = struct_type(vec![field("a", DataType::Int32)]); @@ -1322,8 +1680,12 @@ mod tests { &DataType::Dictionary(Box::new(DataType::Int32), Box::new(s2.clone())), )); assert!(requires_nested_struct_cast( - &DataType::ListView(arc_field("item", s1)), - &DataType::ListView(arc_field("item", s2)), + &DataType::ListView(arc_field("item", s1.clone())), + &DataType::ListView(arc_field("item", s2.clone())), + )); + assert!(requires_nested_struct_cast( + &DataType::FixedSizeList(arc_field("item", s1), 2), + &DataType::FixedSizeList(arc_field("item", s2), 2), )); // Non-struct types should return false. @@ -1335,5 +1697,9 @@ mod tests { &DataType::List(arc_field("item", DataType::Int32)), &DataType::List(arc_field("item", DataType::Int64)), )); + assert!(!requires_nested_struct_cast( + &DataType::FixedSizeList(arc_field("item", DataType::Int32), 2), + &DataType::FixedSizeList(arc_field("item", DataType::Int64), 2), + )); } } diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rs index fd70d74a9140c..535828fa29c2f 100644 --- a/datafusion/core/tests/parquet/expr_adapter.rs +++ b/datafusion/core/tests/parquet/expr_adapter.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use arrow::array::{ - Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray, - RecordBatch, StringArray, StructArray, record_batch, + Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array, + LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch, }; use arrow::buffer::OffsetBuffer; use arrow::compute::concat_batches; @@ -60,13 +60,19 @@ async fn write_parquet(batch: RecordBatch, store: Arc, path: &s enum NestedListKind { List, LargeList, + FixedSizeList, } +const FIXED_SIZE_LIST_LEN: usize = 2; + impl NestedListKind { fn field_data_type(self, item_field: Arc) -> DataType { match self { Self::List => DataType::List(item_field), Self::LargeList => DataType::LargeList(item_field), + Self::FixedSizeList => { + DataType::FixedSizeList(item_field, FIXED_SIZE_LIST_LEN as i32) + } } } @@ -89,6 +95,19 @@ impl NestedListKind { values, None, )), + Self::FixedSizeList => { + assert_eq!( + lengths.as_slice(), + &[FIXED_SIZE_LIST_LEN], + "FixedSizeList fixtures must contain exactly {FIXED_SIZE_LIST_LEN} elements per row" + ); + Arc::new(FixedSizeListArray::new( + item_field, + FIXED_SIZE_LIST_LEN as i32, + values, + None, + )) + } } } @@ -96,6 +115,7 @@ impl NestedListKind { match self { Self::List => "list", Self::LargeList => "large_list", + Self::FixedSizeList => "fixed_size_list", } } } @@ -277,7 +297,8 @@ fn nested_list_table_schema( } // Helper to extract message values from a nested list column. -// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray. +// Returns the values at indices 0 and 1 from either a ListArray, LargeListArray, +// or FixedSizeListArray. fn extract_nested_list_values( kind: NestedListKind, column: &ArrayRef, @@ -297,7 +318,50 @@ fn extract_nested_list_values( .expect("messages should be a LargeListArray"); (list.value(0), list.value(1)) } + NestedListKind::FixedSizeList => { + let list = column + .as_any() + .downcast_ref::() + .expect("messages should be a FixedSizeListArray"); + (list.value(0), list.value(1)) + } + } +} + +fn evolved_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 30, + name: "gamma", + chain: Some("eth"), + ignored: Some(99), + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 40, + name: "delta", + chain: Some("doge"), + ignored: Some(100), + }); + } + messages +} + +fn error_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 10, + name: "alpha", + chain: Some("eth"), + ignored: None, + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 20, + name: "beta", + chain: Some("doge"), + ignored: None, + }); } + messages } // Helper to set up a nested list test fixture. @@ -352,15 +416,11 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res ); // new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`. + let new_messages = evolved_messages(kind); let new_batch = nested_messages_batch( kind, 2, - &[NestedMessageRow { - id: 30, - name: "gamma", - chain: Some("eth"), - ignored: Some(99), - }], + &new_messages, &message_fields(DataType::Utf8, true, true, true), ); @@ -429,7 +489,12 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res .as_any() .downcast_ref::() .unwrap(); - assert_eq!(new_chain.iter().collect::>(), vec![Some("eth")]); + let expected_new_chain = if matches!(kind, NestedListKind::FixedSizeList) { + vec![Some("eth"), Some("doge")] + } else { + vec![Some("eth")] + }; + assert_eq!(new_chain.iter().collect::>(), expected_new_chain); let projected = ctx .sql( @@ -863,12 +928,12 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> { Ok(()) } -/// Macro to generate paired test functions for List and LargeList variants. -/// Expands to two `#[tokio::test]` functions with the specified names. -macro_rules! test_struct_schema_evolution_pair { +/// Macro to generate schema evolution tests for list-like variants. +macro_rules! test_struct_schema_evolution_variants { ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn: $assertion_fn:path $(, args: $($arg:expr),+)? ) => { #[tokio::test] @@ -880,10 +945,16 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() { $assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await; } + + #[tokio::test] + async fn $fixed_size_list_test() { + $assertion_fn(NestedListKind::FixedSizeList $(, $($arg),+)?).await; + } }; ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn_result: $assertion_fn:path ) => { #[tokio::test] @@ -895,31 +966,34 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() -> Result<()> { $assertion_fn(NestedListKind::LargeList).await } + + #[tokio::test] + async fn $fixed_size_list_test() -> Result<()> { + $assertion_fn(NestedListKind::FixedSizeList).await + } }; } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_end_to_end, large_list: test_large_list_struct_schema_evolution_end_to_end, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_end_to_end, fn_result: assert_nested_list_struct_schema_evolution ); async fn assert_nested_list_struct_schema_evolution_errors( kind: NestedListKind, + source_includes_chain: bool, chain_type: DataType, chain_nullable: bool, expected_error: &str, ) { + let messages = error_messages(kind); let batch = nested_messages_batch( kind, 1, - &[NestedMessageRow { - id: 10, - name: "alpha", - chain: Some("eth"), - ignored: None, - }], - &message_fields(DataType::Utf8, true, true, false), + &messages, + &message_fields(DataType::Utf8, true, source_includes_chain, false), ); let table_schema = @@ -949,6 +1023,7 @@ async fn assert_nested_list_struct_schema_evolution_errors( async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + false, DataType::Utf8, false, "non-nullable", @@ -959,6 +1034,7 @@ async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { async fn assert_incompatible_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + true, incompatible_chain_type(), true, "Cannot cast struct field 'chain'", @@ -970,15 +1046,17 @@ fn incompatible_chain_type() -> DataType { DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into()) } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_non_nullable_missing_field_fails, large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_non_nullable_missing_field_fails, fn: assert_non_nullable_missing_chain_field_fails ); -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_incompatible_field_fails, large_list: test_large_list_struct_schema_evolution_incompatible_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_incompatible_field_fails, fn: assert_incompatible_chain_field_fails ); From 8393fd32b7152779886c2b0176fab12af0cbbe0c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 25 Jul 2026 08:17:41 -0600 Subject: [PATCH 023/109] refactor: share hex encoding across datafusion-common, functions, and spark (#23766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? N/A ## Rationale for this change Hex encoding was implemented six times across the workspace, at two different levels of optimization. Three copies used a fast byte-pair lookup table: - `datafusion/spark/src/function/math/hex.rs` - `datafusion/functions/src/string/to_hex.rs` - `datafusion/functions/src/encoding/inner.rs` (via the `hex` crate) The other three used a slower nibble-at-a-time loop pushing one character at a time: - `datafusion/functions/src/crypto/md5.rs` - `datafusion/spark/src/function/hash/sha1.rs` - `datafusion/spark/src/function/hash/sha2.rs` Beyond the duplication, this split meant the digest functions were paying for a slower encoder than the one already sitting elsewhere in the tree. Consolidating on a single implementation removes the duplication and moves `md5`, `sha1`, and `sha2` onto the fast path. ## What changes are included in this PR? A new `datafusion_common::utils::hex` module holds the only hex encoder in the workspace: ```rust pub enum HexCase { Lower, Upper } pub fn encode_bytes_into(bytes: &[u8], case: HexCase, out: &mut Vec); pub fn encode_bytes(bytes: &[u8], case: HexCase) -> String; pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]); pub fn encode_u64(v: u64, case: HexCase, buf: &mut [u8; 16]) -> &[u8]; ``` Four entry points rather than one because the call sites have genuinely different output needs, and forcing them through a single shape would cost an allocation somewhere: `to_hex` writes straight into a `StringArray` values buffer, Spark's `hex` appends to a reused scratch `Vec`, `encode` writes into a pre-sized slice, and the digest functions want an owned `String`. Migrated call sites, all producing bit-identical output: | File | Change | | --- | --- | | `functions/src/string/to_hex.rs` | local table and two write helpers deleted; eight trait impls collapsed to two macros | | `spark/src/function/math/hex.rs` | two nibble tables, two lookup tables, `build_hex_lookup` and `hex_int64` deleted | | `functions/src/crypto/md5.rs` | local table and `hex_encode` deleted | | `spark/src/function/hash/sha1.rs` | local table and inline nibble loop deleted | | `spark/src/function/hash/sha2.rs` | local table and `hex_encode` deleted; eight call sites migrated | | `functions/src/encoding/inner.rs` | both encode sites moved off the `hex` crate | Deliberately left alone: - `hex::decode` / `hex::decode_to_slice` in `encoding/inner.rs`, and Spark's `unhex`. The decode direction has different semantics — Spark's `unhex` left-pads odd-length input, the `hex` crate does not — so unifying it is a separate question. The `hex` dependency stays for those. - `ScalarValue`'s binary `Display` impl in `common/src/scalar/mod.rs`, which writes to a `fmt::Formatter` rather than a byte buffer, and is a cold path. Three details worth a reviewer's attention: - The two ancestors of `encode_u64` disagreed on zero: `to_hex` wrote `'0'` into the caller's buffer, Spark's `hex_int64` returned a `'static` `b"0"` that never touched it. The shared version always writes into the buffer and returns a subslice of it, so the lifetime is uniform. Both callers still produce `"0"`. - Spark's `hex_encode_bytes` guards large binary input with `checked_mul(2)` + `try_reserve`, returning a `DataFusionError` rather than aborting on allocation failure. That guard stays at the call site; `encode_bytes_into` performs no reservation of its own, so behaviour is unchanged. - The encoders are `#[inline(always)]`, not `#[inline]`. They are called once per row from two other crates, and plain `#[inline]` left them out-of-line across the crate boundary. Measured cost of that: +3% on Spark's byte paths and up to +18% on `to_hex`'s i32 path, i.e. the refactor was a net regression on those benchmarks until the attribute changed. ## Are these changes tested? Every migrated function keeps its existing unit and sqllogictest coverage, which is what pins bit-identical output — in particular `spark/hash/sha1.slt` and `sha2.slt` assert concrete digest strings for all four SHA-2 bit lengths across both the scalar and array paths, and `expr.slt` covers `to_hex` and `md5`. New unit tests in `common/src/utils/hex.rs` cover zero, `u64::MAX`, single-nibble values, the odd/even digit-count boundary, two's complement of negative input, empty input, all 256 byte values in both cases, appending into a non-empty buffer, and that a reused scratch buffer never leaks stale digits between calls. Tests cross-check against `format!("{:x}")` rather than restating the implementation. Two Spark tests changed. `test_hex_int64` now drives `hex_encode_int64` instead of the deleted private `hex_int64`, keeping all ten cases including `i64::MIN`, `-1`, and the uppercase expectations. `test_hex_lookup_table_covers_all_bytes` was deleted — it only cross-checked the raw lookup tables, and `encode_bytes_covers_every_byte_value` now does that exhaustively through the public API. A test was added for Spark's lowercase byte path, which previously had no coverage. ### Benchmarks Criterion, `apache/main` @ `eef101769` as baseline. Median of the reported change interval. `datafusion/functions/benches/to_hex.rs`: | Benchmark | 1024 | 4096 | 8192 | | --- | --- | --- | --- | | `i32_random` | −13.8% | −12.8% | −10.2% | | `i64_random` | −8.8% | −9.5% | −8.1% | | `i64_large_values` | −9.9% | −9.8% | −7.9% | `scalar_i32` −2.3%, `scalar_i64` −1.8%. `datafusion/spark/benches/sha2.rs`: | Benchmark | 1024 | 4096 | 8192 | | --- | --- | --- | --- | | `array_binary_256` | −20.7% | −18.2% | −19.1% | | `array_scalar_binary_256` | −14.6% | −13.2% | −13.1% | `scalar/size=1` −3.3%. `datafusion/spark/benches/hex.rs`: | Benchmark | 1024 | 4096 | 8192 | | --- | --- | --- | --- | | `hex_int64` | −2.9% | −3.6% | −3.5% | | `hex_int64_dict` | −3.2% | −1.5% | −0.2% | | `hex_utf8` | −0.2% | −0.3% | +0.7% | | `hex_binary` | −0.4% | −0.6% | −0.3% | The `hex_utf8` and `hex_binary` paths already used the byte-pair table before this PR, so they are expected to be flat; they are. `datafusion/functions/benches/crypto.rs`: `md5_array` −4.0%, `md5_scalar` −3.7%. The `sha224` / `sha256` / `sha384` / `sha512` cases in that file range from −0.1% to +2.2%, but they exercise `crypto/basic.rs`, which this PR does not modify and which contains no hex encoding — those numbers are run-to-run variance, not an effect of this change. No number is quoted for Spark `sha1`: it has no benchmark, and its change is the same substitution applied to `md5` and `sha2`. ## Are there any user-facing changes? No behaviour change — all migrated functions produce byte-identical output. `datafusion_common::utils::hex` is new public API on `datafusion-common`. --------- Co-authored-by: Andrew Lamb Co-authored-by: Jeffrey Vo --- datafusion/common/src/utils/hex.rs | 357 +++++++++++++++++++++ datafusion/common/src/utils/mod.rs | 1 + datafusion/functions/src/crypto/md5.rs | 29 +- datafusion/functions/src/encoding/inner.rs | 13 +- datafusion/functions/src/string/to_hex.rs | 126 +++----- datafusion/spark/src/function/hash/sha1.rs | 12 +- datafusion/spark/src/function/hash/sha2.rs | 33 +- datafusion/spark/src/function/math/hex.rs | 111 ++----- 8 files changed, 445 insertions(+), 237 deletions(-) create mode 100644 datafusion/common/src/utils/hex.rs diff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rs new file mode 100644 index 0000000000000..6d0811350eaae --- /dev/null +++ b/datafusion/common/src/utils/hex.rs @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Hex encoding of bytes and integers. +//! +//! [`encode_bytes`] and [`encode_bytes_into`] encode a byte slice into an +//! owned `String` or an appended `Vec`, respectively; [`encode_bytes_to_slice`] +//! writes into a caller-provided, pre-sized buffer. [`encode_u64`] encodes an +//! integer, trimming leading zeros. All four take a [`HexCase`] to choose +//! between lowercase and uppercase digits. + +use crate::Result; +use crate::error::_internal_err; + +/// Case of the emitted hex digits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HexCase { + /// Digits `0123456789abcdef`. + Lower, + /// Digits `0123456789ABCDEF`. + Upper, +} + +const LOWER_DIGITS: &[u8; 16] = b"0123456789abcdef"; +const UPPER_DIGITS: &[u8; 16] = b"0123456789ABCDEF"; + +/// Maps a full byte to its two hex digits, so encoding advances a whole byte +/// per iteration instead of a nibble. +const LOOKUP_LOWER: [[u8; 2]; 256] = build_lookup(LOWER_DIGITS); +const LOOKUP_UPPER: [[u8; 2]; 256] = build_lookup(UPPER_DIGITS); + +const fn build_lookup(digits: &[u8; 16]) -> [[u8; 2]; 256] { + let mut table = [[0u8; 2]; 256]; + let mut i = 0; + while i < 256 { + table[i][0] = digits[i >> 4]; + table[i][1] = digits[i & 0xF]; + i += 1; + } + table +} + +impl HexCase { + #[inline] + const fn lookup(self) -> &'static [[u8; 2]; 256] { + match self { + HexCase::Lower => &LOOKUP_LOWER, + HexCase::Upper => &LOOKUP_UPPER, + } + } + + #[inline] + const fn digits(self) -> &'static [u8; 16] { + match self { + HexCase::Lower => LOWER_DIGITS, + HexCase::Upper => UPPER_DIGITS, + } + } +} + +/// Appends the hex encoding of `bytes` to `out`. +/// +/// Allocates only through `out`'s own growth. Callers that must bound or guard +/// that growth should reserve capacity in `out` before calling. +#[inline(always)] +pub fn encode_bytes_into(bytes: &[u8], case: HexCase, out: &mut Vec) { + let lookup = case.lookup(); + for &byte in bytes { + out.extend_from_slice(&lookup[byte as usize]); + } +} + +/// Writes the hex encoding of `bytes` into `out`. +/// +/// This is for callers that already own a pre-sized buffer (for example a +/// slice of a larger, pre-allocated output array) and want to write directly +/// into it rather than appending to a `Vec`. +/// +/// Returns an internal error if `out` is not exactly `2 * bytes.len()` bytes +/// long, without filling any of the `out` buffer. +/// +/// # Example +/// +/// ``` +/// use datafusion_common::utils::hex::{HexCase, encode_bytes_to_slice}; +/// +/// let mut out = [0u8; 8]; +/// encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?; +/// assert_eq!(&out, b"deadbeef"); +/// # Ok::<(), datafusion_common::DataFusionError>(()) +/// ``` +#[inline(always)] +pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Result<()> { + let expected = bytes.len() * 2; + if out.len() != expected { + return _internal_err!( + "hex output buffer is {} bytes, expected {expected}", + out.len() + ); + } + let lookup = case.lookup(); + for (&b, chunk) in bytes.iter().zip(out.chunks_exact_mut(2)) { + chunk.copy_from_slice(&lookup[b as usize]); + } + Ok(()) +} + +/// Returns the hex encoding of `bytes` as an owned `String`. +/// +/// # Example +/// +/// ``` +/// use datafusion_common::utils::hex::{HexCase, encode_bytes}; +/// +/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower), "deadbeef"); +/// assert_eq!(encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper), "DEADBEEF"); +/// ``` +#[inline] +pub fn encode_bytes(bytes: &[u8], case: HexCase) -> String { + let mut out = Vec::with_capacity(bytes.len() * 2); + encode_bytes_into(bytes, case, &mut out); + // SAFETY: `out` holds only ASCII hex digits, which are valid UTF-8. + unsafe { String::from_utf8_unchecked(out) } +} + +/// Writes `v` as hex into `buf` and returns the written subslice. +/// +/// Digits are written right-aligned with leading zeros trimmed, so the result +/// borrows the tail of `buf`. Zero encodes as `"0"`. +/// +/// Signed values should be cast with `as u64`, which yields the two's +/// complement representation that both `to_hex` and Spark's `hex` produce for +/// negative input. +/// +/// # Example +/// +/// The caller owns the buffer and can reuse it across calls; each call +/// returns a fresh subslice of it, borrowed for as long as `buf` is: +/// +/// ``` +/// use datafusion_common::utils::hex::{HexCase, encode_u64}; +/// +/// let mut buf = [0u8; 16]; +/// assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab"); +/// assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0"); +/// ``` +#[inline(always)] +pub fn encode_u64(v: u64, case: HexCase, buf: &mut [u8; 16]) -> &[u8] { + let start = write_digits(v, case, buf); + &buf[start..] +} + +/// Writes the digits of `v` right-aligned in `buf`, returning the index of the +/// first digit. +/// +/// Split out from [`encode_u64`] so the mutable borrow of `buf` ends before the +/// returned slice reborrows it. +#[inline(always)] +fn write_digits(v: u64, case: HexCase, buf: &mut [u8; 16]) -> usize { + if v == 0 { + buf[15] = b'0'; + return 15; + } + + // Consume two nibbles (one full byte) per iteration. + let lookup = case.lookup(); + let mut pos = 16; + let mut rest = v; + while rest >= 0x10 { + pos -= 2; + let pair = lookup[(rest & 0xFF) as usize]; + buf[pos] = pair[0]; + buf[pos + 1] = pair[1]; + rest >>= 8; + } + if rest > 0 { + // A single high nibble (0x1..=0xF) remains. + pos -= 1; + buf[pos] = case.digits()[rest as usize]; + } + + pos +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hex_u64(v: u64, case: HexCase) -> String { + let mut buf = [0u8; 16]; + String::from_utf8(encode_u64(v, case, &mut buf).to_vec()).unwrap() + } + + #[test] + fn encode_u64_zero() { + assert_eq!(hex_u64(0, HexCase::Lower), "0"); + assert_eq!(hex_u64(0, HexCase::Upper), "0"); + } + + #[test] + fn encode_u64_single_nibble() { + for v in 1..=0xFu64 { + assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}")); + assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}")); + } + } + + #[test] + fn encode_u64_digit_count_boundaries() { + // Straddle each odd/even digit-count boundary: the two-nibbles-per + // iteration loop plus the trailing single-nibble fixup. + for v in [ + 0x10u64, + 0xFF, + 0x100, + 0xFFF, + 0x1000, + 0xFFFFF, + 0xFFFF_FFFF, + 0x1_0000_0000, + ] { + assert_eq!(hex_u64(v, HexCase::Lower), format!("{v:x}")); + assert_eq!(hex_u64(v, HexCase::Upper), format!("{v:X}")); + } + } + + #[test] + fn encode_u64_max() { + assert_eq!(hex_u64(u64::MAX, HexCase::Lower), "ffffffffffffffff"); + assert_eq!(hex_u64(u64::MAX, HexCase::Upper), "FFFFFFFFFFFFFFFF"); + } + + #[test] + fn encode_u64_signed_is_twos_complement() { + // Callers cast signed values with `as u64`; this is the behaviour both + // `to_hex` and Spark `hex` rely on for negative input. + assert_eq!(hex_u64(-1i64 as u64, HexCase::Lower), "ffffffffffffffff"); + assert_eq!(hex_u64(i64::MIN as u64, HexCase::Upper), "8000000000000000"); + } + + #[test] + fn encode_bytes_empty() { + assert_eq!(encode_bytes(&[], HexCase::Lower), ""); + assert_eq!(encode_bytes(&[], HexCase::Upper), ""); + } + + #[test] + fn encode_bytes_examples() { + assert_eq!(encode_bytes(&[0x00], HexCase::Lower), "00"); + assert_eq!(encode_bytes(&[0xAB], HexCase::Lower), "ab"); + assert_eq!(encode_bytes(&[0xAB], HexCase::Upper), "AB"); + assert_eq!( + encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower), + "deadbeef" + ); + assert_eq!( + encode_bytes(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper), + "DEADBEEF" + ); + } + + #[test] + fn encode_bytes_covers_every_byte_value() { + let bytes: Vec = (0..=255u8).collect(); + + let expected: String = bytes.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(encode_bytes(&bytes, HexCase::Lower), expected); + + let expected: String = bytes.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(encode_bytes(&bytes, HexCase::Upper), expected); + } + + #[test] + fn encode_bytes_into_appends_without_clearing() { + let mut out = b"prefix-".to_vec(); + encode_bytes_into(&[0x01, 0x02], HexCase::Lower, &mut out); + assert_eq!(out, b"prefix-0102"); + } + + #[test] + fn encode_u64_reused_buffer_leaks_no_stale_digits() { + let mut buf = [0u8; 16]; + assert_eq!( + encode_u64(u64::MAX, HexCase::Lower, &mut buf), + b"ffffffffffffffff" + ); + assert_eq!(encode_u64(0, HexCase::Lower, &mut buf), b"0"); + assert_eq!(encode_u64(0xAB, HexCase::Lower, &mut buf), b"ab"); + } + + #[test] + fn encode_bytes_to_slice_empty() -> Result<()> { + let mut out: [u8; 0] = []; + encode_bytes_to_slice(&[], HexCase::Lower, &mut out)?; + assert_eq!(out, [] as [u8; 0]); + Ok(()) + } + + #[test] + fn encode_bytes_to_slice_examples() -> Result<()> { + let mut out = [0u8; 8]; + encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut out)?; + assert_eq!(&out, b"deadbeef"); + + let mut out = [0u8; 8]; + encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Upper, &mut out)?; + assert_eq!(&out, b"DEADBEEF"); + Ok(()) + } + + #[test] + fn encode_bytes_to_slice_agrees_with_encode_bytes() -> Result<()> { + let bytes: Vec = (0..=255u8).collect(); + for case in [HexCase::Lower, HexCase::Upper] { + let mut out = vec![0u8; bytes.len() * 2]; + encode_bytes_to_slice(&bytes, case, &mut out)?; + assert_eq!(String::from_utf8(out).unwrap(), encode_bytes(&bytes, case)); + } + Ok(()) + } + + #[test] + fn encode_bytes_to_slice_rejects_wrong_length() { + // Too short: the old `debug_assert` let release builds silently drop + // the remaining input. + let mut short = [0u8; 6]; + let err = + encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut short) + .unwrap_err(); + assert!( + err.message() + .contains("hex output buffer is 6 bytes, expected 8"), + "unexpected message: {err}" + ); + + // Too long: would have left stale bytes at the tail. + let mut long = [0u8; 10]; + assert!( + encode_bytes_to_slice(&[0xde, 0xad, 0xbe, 0xef], HexCase::Lower, &mut long) + .is_err() + ); + } +} diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs index 94bbb91a7fa8b..73772b319351c 100644 --- a/datafusion/common/src/utils/mod.rs +++ b/datafusion/common/src/utils/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod aggregate; pub mod expr; +pub mod hex; pub mod memory; pub mod proxy; pub mod string_utils; diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs index 178aebf0fbd41..b1206d2e423cc 100644 --- a/datafusion/functions/src/crypto/md5.rs +++ b/datafusion/functions/src/crypto/md5.rs @@ -21,6 +21,7 @@ use datafusion_common::{ cast::as_binary_array, internal_err, types::{logical_binary, logical_string}, + utils::hex::{HexCase, encode_bytes}, utils::take_function_args, }; use datafusion_expr::{ @@ -98,22 +99,6 @@ impl ScalarUDFImpl for Md5Func { } } -/// Hex encoding lookup table for fast byte-to-hex conversion -const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef"; - -/// Fast hex encoding using a lookup table instead of format strings. -/// This is significantly faster than using `write!("{:02x}")` for each byte. -#[inline] -fn hex_encode(data: impl AsRef<[u8]>) -> String { - let bytes = data.as_ref(); - let mut s = String::with_capacity(bytes.len() * 2); - for &b in bytes { - s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char); - s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char); - } - s -} - fn md5(args: &[ColumnarValue]) -> Result { let [data] = take_function_args("md5", args)?; let value = digest_process(data, DigestAlgorithm::Md5)?; @@ -122,13 +107,15 @@ fn md5(args: &[ColumnarValue]) -> Result { Ok(match value { ColumnarValue::Array(array) => { let binary_array = as_binary_array(&array)?; - let string_array: StringViewArray = - binary_array.iter().map(|opt| opt.map(hex_encode)).collect(); + let string_array: StringViewArray = binary_array + .iter() + .map(|opt| opt.map(|b| encode_bytes(b, HexCase::Lower))) + .collect(); ColumnarValue::Array(Arc::new(string_array)) } - ColumnarValue::Scalar(ScalarValue::Binary(opt)) => { - ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(hex_encode))) - } + ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar( + ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))), + ), _ => return internal_err!("Impossibly got invalid results from digest"), }) } diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index 5d4740d80b94c..850e312abdb40 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -33,7 +33,10 @@ use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, internal_err, not_impl_err, plan_err, types::{NativeType, logical_string}, - utils::take_function_args, + utils::{ + hex::{HexCase, encode_bytes as encode_hex, encode_bytes_to_slice}, + take_function_args, + }, }; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -370,7 +373,7 @@ impl Encoding { match self { Self::Base64 => BASE64_ENGINE.encode(value), Self::Base64Padded => BASE64_ENGINE_PADDED.encode(value), - Self::Hex => hex::encode(value), + Self::Hex => encode_hex(value, HexCase::Lower), } } @@ -477,11 +480,7 @@ where for v in array.iter() { if let Some(v) = v { let out_len = v.len() * 2; - // The slice is sized to exactly `2 * v.len()`, which is the only - // condition under which `encode_to_slice` can fail, so this cannot - // error. - hex::encode_to_slice(v, &mut values[pos..pos + out_len]) - .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?; + encode_bytes_to_slice(v, HexCase::Lower, &mut values[pos..pos + out_len])?; pos += out_len; } offsets.push(OutputOffset::usize_as(pos)); diff --git a/datafusion/functions/src/string/to_hex.rs b/datafusion/functions/src/string/to_hex.rs index 497a0a1206922..a6bcd179664df 100644 --- a/datafusion/functions/src/string/to_hex.rs +++ b/datafusion/functions/src/string/to_hex.rs @@ -24,6 +24,7 @@ use arrow::datatypes::{ Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type, }; use datafusion_common::cast::as_primitive_array; +use datafusion_common::utils::hex::{HexCase, encode_u64}; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -31,9 +32,6 @@ use datafusion_expr::{ }; use datafusion_macros::user_doc; -/// Hex lookup table for fast conversion -const HEX_CHARS: &[u8; 16] = b"0123456789abcdef"; - /// Converts the number to its equivalent hexadecimal representation. /// to_hex(2147483647) = '7fffffff' fn to_hex_array(array: &ArrayRef) -> Result @@ -59,8 +57,7 @@ where // Process all values directly (including null slots - we write empty strings for nulls) // The null bitmap will mark which entries are actually null for value in integer_array.values() { - let hex_len = value.write_hex_to_buffer(&mut hex_buffer); - values.extend_from_slice(&hex_buffer[16 - hex_len..]); + values.extend_from_slice(value.write_hex(&mut hex_buffer)); offsets.push(values.len() as i32); } @@ -79,101 +76,50 @@ where #[inline] fn to_hex_scalar(value: T) -> String { let mut hex_buffer = [0u8; 16]; - let hex_len = value.write_hex_to_buffer(&mut hex_buffer); - // SAFETY: hex_buffer is ASCII hex digits - unsafe { std::str::from_utf8_unchecked(&hex_buffer[16 - hex_len..]).to_string() } + let hex = value.write_hex(&mut hex_buffer); + // SAFETY: hex holds only ASCII hex digits. + unsafe { std::str::from_utf8_unchecked(hex).to_string() } } /// Trait for converting integer types to hexadecimal in a buffer trait ToHex: ArrowNativeType { - /// Write hex representation to buffer and return the number of hex digits written. - /// The hex digits are written right-aligned in the buffer (starting from position 16 - len). - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize; -} - -/// Write unsigned value to hex buffer and return the number of digits written. -/// Digits are written right-aligned in the buffer. -#[inline] -fn write_unsigned_hex_to_buffer(value: u64, buffer: &mut [u8; 16]) -> usize { - if value == 0 { - buffer[15] = b'0'; - return 1; - } - - // Write hex digits from right to left - let mut pos = 16; - let mut v = value; - while v > 0 { - pos -= 1; - buffer[pos] = HEX_CHARS[(v & 0xf) as usize]; - v >>= 4; - } - - 16 - pos -} - -/// Write signed value to hex buffer (two's complement for negative) and return digit count -#[inline] -fn write_signed_hex_to_buffer(value: i64, buffer: &mut [u8; 16]) -> usize { - // For negative values, use two's complement representation (same as casting to u64) - write_unsigned_hex_to_buffer(value as u64, buffer) -} - -impl ToHex for i8 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_signed_hex_to_buffer(self as i64, buffer) - } -} - -impl ToHex for i16 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_signed_hex_to_buffer(self as i64, buffer) - } -} - -impl ToHex for i32 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_signed_hex_to_buffer(self as i64, buffer) - } -} - -impl ToHex for i64 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_signed_hex_to_buffer(self, buffer) - } + /// Writes the hex representation into `buf` and returns the written + /// subslice. Digits are right-aligned in `buf` with leading zeros trimmed. + fn write_hex(self, buf: &mut [u8; 16]) -> &[u8]; } -impl ToHex for u8 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_unsigned_hex_to_buffer(self as u64, buffer) - } -} - -impl ToHex for u16 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_unsigned_hex_to_buffer(self as u64, buffer) - } +/// Signed values use their two's complement representation, matching a cast to +/// the corresponding unsigned type. +macro_rules! impl_to_hex_signed { + ($ty:ty) => { + impl ToHex for $ty { + #[inline] + fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] { + encode_u64(self as i64 as u64, HexCase::Lower, buf) + } + } + }; } -impl ToHex for u32 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_unsigned_hex_to_buffer(self as u64, buffer) - } +macro_rules! impl_to_hex_unsigned { + ($ty:ty) => { + impl ToHex for $ty { + #[inline] + fn write_hex(self, buf: &mut [u8; 16]) -> &[u8] { + encode_u64(self as u64, HexCase::Lower, buf) + } + } + }; } -impl ToHex for u64 { - #[inline] - fn write_hex_to_buffer(self, buffer: &mut [u8; 16]) -> usize { - write_unsigned_hex_to_buffer(self, buffer) - } -} +impl_to_hex_signed!(i8); +impl_to_hex_signed!(i16); +impl_to_hex_signed!(i32); +impl_to_hex_signed!(i64); +impl_to_hex_unsigned!(u8); +impl_to_hex_unsigned!(u16); +impl_to_hex_unsigned!(u32); +impl_to_hex_unsigned!(u64); #[user_doc( doc_section(label = "String Functions"), diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rs index dd9009eb8233f..05a224f33f25a 100644 --- a/datafusion/spark/src/function/hash/sha1.rs +++ b/datafusion/spark/src/function/hash/sha1.rs @@ -24,6 +24,7 @@ use datafusion_common::cast::{ as_large_binary_array, }; use datafusion_common::types::{NativeType, logical_string}; +use datafusion_common::utils::hex::{HexCase, encode_bytes}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -89,18 +90,9 @@ impl ScalarUDFImpl for SparkSha1 { } } -/// Hex encoding lookup table for fast byte-to-hex conversion -const HEX_CHARS_LOWER: &[u8; 16] = b"0123456789abcdef"; - #[inline] fn spark_sha1_digest(value: &[u8]) -> String { - let result = Sha1::digest(value); - let mut s = String::with_capacity(result.len() * 2); - for &b in result.as_slice() { - s.push(HEX_CHARS_LOWER[(b >> 4) as usize] as char); - s.push(HEX_CHARS_LOWER[(b & 0x0f) as usize] as char); - } - s + encode_bytes(&Sha1::digest(value), HexCase::Lower) } fn spark_sha1_impl<'a>(input: impl Iterator>) -> ArrayRef { diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs index 38fa0cc643751..541df2957669e 100644 --- a/datafusion/spark/src/function/hash/sha2.rs +++ b/datafusion/spark/src/function/hash/sha2.rs @@ -20,6 +20,7 @@ use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::types::{ NativeType, logical_binary, logical_int32, logical_string, }; +use datafusion_common::utils::hex::{HexCase, encode_bytes}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ @@ -112,22 +113,22 @@ impl ScalarUDFImpl for SparkSha2 { 224 => { let mut digest = sha2::Sha224::default(); digest.update(bytes); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } 0 | 256 => { let mut digest = sha2::Sha256::default(); digest.update(bytes); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } 384 => { let mut digest = sha2::Sha384::default(); digest.update(bytes); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } 512 => { let mut digest = sha2::Sha512::default(); digest.update(bytes); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } _ => None, }; @@ -222,22 +223,22 @@ where (Some(value), Some(224)) => { let mut digest = sha2::Sha224::default(); digest.update(value); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } (Some(value), Some(0 | 256)) => { let mut digest = sha2::Sha256::default(); digest.update(value); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } (Some(value), Some(384)) => { let mut digest = sha2::Sha384::default(); digest.update(value); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } (Some(value), Some(512)) => { let mut digest = sha2::Sha512::default(); digest.update(value); - Some(hex_encode(digest.finalize())) + Some(encode_bytes(&digest.finalize(), HexCase::Lower)) } // Unknown bit-lengths go to null, same as in Spark _ => None, @@ -245,19 +246,3 @@ where .collect::(); Arc::new(array) } - -const HEX_CHARS: [u8; 16] = *b"0123456789abcdef"; - -#[inline] -fn hex_encode>(data: T) -> String { - let bytes = data.as_ref(); - let mut out = Vec::with_capacity(bytes.len() * 2); - for &b in bytes { - let hi = b >> 4; - let lo = b & 0x0F; - out.push(HEX_CHARS[hi as usize]); - out.push(HEX_CHARS[lo as usize]); - } - // SAFETY: out contains only ASCII - unsafe { String::from_utf8_unchecked(out) } -} diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rs index a283bd8fa7de6..1f505fda21b6f 100644 --- a/datafusion/spark/src/function/math/hex.rs +++ b/datafusion/spark/src/function/math/hex.rs @@ -28,6 +28,7 @@ use arrow::{ use datafusion_common::cast::as_large_binary_array; use datafusion_common::cast::as_string_view_array; use datafusion_common::types::{NativeType, logical_int64, logical_string}; +use datafusion_common::utils::hex::{HexCase, encode_bytes_into, encode_u64}; use datafusion_common::utils::take_function_args; use datafusion_common::{ DataFusionError, @@ -110,54 +111,6 @@ impl ScalarUDFImpl for SparkHex { } } -/// Hex encoding lookup tables for fast byte-to-hex conversion. -/// -/// Each entry maps a full byte to its two-character hex encoding so the -/// hot loop becomes one load + one two-byte extend per input byte instead -/// of two nibble lookups and two pushes. -const HEX_CHARS_UPPER_NIBBLES: &[u8; 16] = b"0123456789ABCDEF"; -const HEX_CHARS_LOWER_NIBBLES: &[u8; 16] = b"0123456789abcdef"; - -const HEX_LOOKUP_UPPER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_UPPER_NIBBLES); -const HEX_LOOKUP_LOWER: [[u8; 2]; 256] = build_hex_lookup(HEX_CHARS_LOWER_NIBBLES); - -const fn build_hex_lookup(nibbles: &[u8; 16]) -> [[u8; 2]; 256] { - let mut table = [[0u8; 2]; 256]; - let mut i = 0; - while i < 256 { - table[i][0] = nibbles[(i >> 4) & 0xF]; - table[i][1] = nibbles[i & 0xF]; - i += 1; - } - table -} - -#[inline] -fn hex_int64(num: i64, buffer: &mut [u8; 16]) -> &[u8] { - if num == 0 { - return b"0"; - } - - // Walk the value two nibbles (one full byte) at a time. The buffer is - // filled from the right so the high-order nibbles end up first; the - // returned slice trims leading zeros automatically. - let mut n = num as u64; - let mut i = 16; - while n >= 0x10 { - i -= 2; - let pair = HEX_LOOKUP_UPPER[(n & 0xFF) as usize]; - buffer[i] = pair[0]; - buffer[i + 1] = pair[1]; - n >>= 8; - } - if n > 0 { - // Single remaining high nibble (value 0x1..=0xF). - i -= 1; - buffer[i] = HEX_CHARS_UPPER_NIBBLES[n as usize]; - } - &buffer[i..] -} - /// Generic hex encoding for byte array types fn hex_encode_bytes<'a, I, T>( iter: I, @@ -168,10 +121,10 @@ where I: Iterator>, T: AsRef<[u8]> + 'a, { - let lookup = if lowercase { - &HEX_LOOKUP_LOWER + let case = if lowercase { + HexCase::Lower } else { - &HEX_LOOKUP_UPPER + HexCase::Upper }; // Write hex digits directly into one growing value buffer, tracking offsets @@ -195,9 +148,7 @@ where "failed to reserve {additional} bytes for hex output: {e}" ) })?; - for &byte in bytes { - values.extend_from_slice(&lookup[byte as usize]); - } + encode_bytes_into(bytes, case, &mut values); nulls.append_non_null(); } else { nulls.append_null(); @@ -233,7 +184,7 @@ fn hex_encode_int64( for v in iter { if let Some(num) = v { let mut temp = [0u8; 16]; - let slice = hex_int64(num, &mut temp); + let slice = encode_u64(num as u64, HexCase::Upper, &mut temp); // SAFETY: slice contains only ASCII hex digests, which are valid UTF-8 unsafe { builder.append_value(from_utf8_unchecked(slice)); @@ -381,11 +332,10 @@ pub fn compute_hex( #[cfg(test)] mod test { - use std::str::from_utf8_unchecked; use std::sync::Arc; use arrow::array::{ - BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray, + Array, BinaryArray, DictionaryArray, Int32Array, Int64Array, StringArray, }; use arrow::{ array::{ @@ -486,7 +436,7 @@ mod test { #[test] fn test_hex_int64() { - let test_cases = vec![ + let cases = vec![ (0_i64, "0"), (1, "1"), (15, "F"), @@ -499,37 +449,28 @@ mod test { (-1, "FFFFFFFFFFFFFFFF"), ]; - for (num, expected) in test_cases { - let mut cache = [0u8; 16]; - let slice = super::hex_int64(num, &mut cache); - - unsafe { - let result = from_utf8_unchecked(slice); - assert_eq!(expected, result, "hex_int64({num}) mismatch"); - } + let arr = + super::hex_encode_int64(cases.iter().map(|(n, _)| Some(*n)), cases.len()) + .unwrap(); + let arr = as_string_array(&arr); + for (i, (num, expected)) in cases.iter().enumerate() { + assert_eq!(*expected, arr.value(i), "hex({num})"); } } #[test] - fn test_hex_lookup_table_covers_all_bytes() { - // Cross-check the precomputed table against an independent encoder - // for every possible byte value and both casings. - for byte in 0u8..=255 { - let upper = format!("{byte:02X}"); - let lower = format!("{byte:02x}"); - let upper_pair = super::HEX_LOOKUP_UPPER[byte as usize]; - let lower_pair = super::HEX_LOOKUP_LOWER[byte as usize]; - assert_eq!( - upper.as_bytes(), - &upper_pair, - "upper encoding mismatch for byte 0x{byte:02X}" - ); - assert_eq!( - lower.as_bytes(), - &lower_pair, - "lower encoding mismatch for byte 0x{byte:02X}" - ); - } + fn test_hex_encode_bytes_lowercase() { + // Every in-repo caller of `hex_encode_bytes` goes through `spark_hex`, + // which always passes `lowercase = false`. The `lowercase = true` path + // is reachable only via `spark_sha2_hex`, which has no in-workspace + // caller, so it otherwise has no coverage. Drive it directly here. + let input = StringArray::from(vec![Some("hi"), Some("bye"), None, Some("rust")]); + let result = super::hex_encode_bytes(input.iter(), true, input.len()).unwrap(); + let result = as_string_array(&result); + + let expected = + StringArray::from(vec![Some("6869"), Some("627965"), None, Some("72757374")]); + assert_eq!(result, &expected); } #[test] From 68a676e1c1e2719aa8b985616b121c1b1b288ef2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 25 Jul 2026 14:53:22 -0400 Subject: [PATCH 024/109] test: cover `array_agg(DISTINCT)` on dictionaries and bounded `retract_batch` memory (#23873) ## Which issue does this PR close? - related to https://github.com/apache/datafusion/pull/23716 ## Rationale for this change It adds test coverage for two gaps found while reviewing https://github.com/apache/datafusion/pull/23716. ## What changes are included in this PR? Tests only, no functional change. - Dictionary inputs - memory usage on retract (make sure memory is released) Note I moved `array_agg` cases out `aggregate.slt` as it is already more than 9k lines long ## Are these changes tested? They are only tests ## Are there any user-facing changes? No. Tests only, no public API changes. --------- Co-authored-by: Claude Opus 5 (1M context) --- .../functions-aggregate/src/array_agg.rs | 49 ++ .../sqllogictest/test_files/aggregate.slt | 510 -------------- .../sqllogictest/test_files/array_agg.slt | 620 ++++++++++++++++++ 3 files changed, 669 insertions(+), 510 deletions(-) create mode 100644 datafusion/sqllogictest/test_files/array_agg.slt diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index b563a6389ec7e..4b0a9d3ddadca 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -2708,4 +2708,53 @@ mod tests { assert_eq!(values, vec![100i32, 200, 300, 400]); Ok(()) } + + #[test] + fn distinct_retract_memory_is_bounded() -> Result<()> { + use arrow::array::Int64Array; + + // Emulates `ROWS BETWEEN CURRENT ROW AND CURRENT ROW`: every row enters + // the frame and immediately leaves it again. Only `CARDINALITY` distinct + // values are ever seen and the live set never holds more than one of + // them + const CARDINALITY: i64 = 10; + const WARMUP_ROWS: i64 = 1_000; + const EXTRA_ROWS: i64 = 20_000; + + let mut acc = + DistinctArrayAggAccumulator::try_new(&DataType::Int64, None, false)?; + + let slide = |acc: &mut DistinctArrayAggAccumulator, rows: i64| -> Result<()> { + for i in 0..rows { + let value: ArrayRef = Arc::new(Int64Array::from(vec![i % CARDINALITY])); + acc.update_batch(std::slice::from_ref(&value))?; + acc.retract_batch(std::slice::from_ref(&value))?; + } + Ok(()) + }; + + // Let every buffer reach its steady state before taking a baseline. + slide(&mut acc, WARMUP_ROWS)?; + let baseline = acc.size(); + + slide(&mut acc, EXTRA_ROWS)?; + let grown = acc.size(); + + assert!( + grown <= 2 * baseline, + "size() must not grow with the number of retracted rows: \ + {baseline} bytes after {WARMUP_ROWS} rows, \ + {grown} bytes after {} rows", + WARMUP_ROWS + EXTRA_ROWS + ); + + // Everything was retracted, so nothing is left in the frame. + let result = acc.evaluate()?; + assert!( + matches!(&result, ScalarValue::List(arr) if arr.is_null(0)), + "expected null list after retracting every row, got {result:?}" + ); + + Ok(()) + } } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 1515e17e3fdff..8e7a9639f55f8 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -212,228 +212,6 @@ WITHIN GROUP (ORDER BY c3) OVER (ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) FROM aggregate_test_100 -# array agg can use order by -query ? -SELECT array_agg(c13 ORDER BY c13) -FROM - (SELECT * - FROM aggregate_test_100 - ORDER BY c13 - LIMIT 5) as t1 ----- -[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8] - -# array agg can use order by with distinct -query ? -SELECT array_agg(DISTINCT c13 ORDER BY c13) -FROM - (SELECT * - FROM aggregate_test_100 - ORDER BY c13 - LIMIT 5) as t1 ----- -[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8] - -query error Execution error: In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list -SELECT array_agg(DISTINCT c13 ORDER BY c12) -FROM aggregate_test_100 - -query error Execution error: In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list -SELECT array_agg(DISTINCT c13 ORDER BY c13, c12) -FROM aggregate_test_100 - -query ?? rowsort -with tbl as (SELECT * FROM (VALUES ('xxx', 'yyy'), ('xxx', 'yyy'), ('xxx2', 'yyy2')) AS t(x, y)) -select - array_agg(x order by x) as x_agg, - array_agg(y order by y) as y_agg -from tbl -group by all ----- -[xxx, xxx, xxx2] [yyy, yyy, yyy2] - -query ?? -SELECT - (SELECT array_agg(c12 ORDER BY c12) FROM aggregate_test_100), - (SELECT array_agg(c13 ORDER BY c13) FROM aggregate_test_100) ----- -[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] - -query ?? -SELECT - array_agg(c12 ORDER BY c12), - array_agg(c13 ORDER BY c13) -FROM aggregate_test_100 ----- -[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] - -query ?? rowsort -with tbl as (SELECT * FROM (VALUES ('xxx', 'yyy'), ('xxx', 'yyy'), ('xxx2', 'yyy2')) AS t(x, y)) -select - array_agg(distinct x order by x) as x_agg, - array_agg(distinct y order by y) as y_agg -from tbl -group by all ----- -[xxx, xxx2] [yyy, yyy2] - -query ?? -SELECT - (SELECT array_agg(DISTINCT c12 ORDER BY c12) FROM aggregate_test_100), - (SELECT array_agg(DISTINCT c13 ORDER BY c13) FROM aggregate_test_100) ----- -[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] - -query ?? -SELECT - array_agg(DISTINCT c12 ORDER BY c12), - array_agg(DISTINCT c13 ORDER BY c13) -FROM aggregate_test_100 ----- -[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] - -statement ok -CREATE EXTERNAL TABLE agg_order ( -c1 INT NOT NULL, -c2 INT NOT NULL, -c3 INT NOT NULL -) -STORED AS CSV -LOCATION '../core/tests/data/aggregate_agg_multi_order.csv' -OPTIONS ('format.has_header' 'true'); - -# test array_agg with order by multiple columns -query ? -select array_agg(c1 order by c2 desc, c3) from agg_order; ----- -[5, 6, 7, 8, 9, 1, 2, 3, 4, 10] - -query TT -explain select array_agg(c1 order by c2 desc, c3) from agg_order; ----- -logical_plan -01)Aggregate: groupBy=[[]], aggr=[[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]]] -02)--TableScan: agg_order projection=[c1, c2, c3] -physical_plan -01)AggregateExec: mode=Final, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]] -02)--CoalescePartitionsExec -03)----AggregateExec: mode=Partial, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]] -04)------SortExec: expr=[c2@1 DESC, c3@2 ASC NULLS LAST], preserve_partitioning=[true] -05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_agg_multi_order.csv]]}, projection=[c1, c2, c3], file_type=csv, has_header=true - -# Regression test: ARRAY_AGG with conflicting ASC/DESC ORDER BY in the same query. -# get_finer_aggregate_exprs_requirement picks ASC as the common requirement and -# reverses the DESC aggregate (is_reversed=true, ordering_req=[ASC]). -# The optimizer then sets is_input_pre_ordered=true on both. Without the fix, -# state() emits values reversed to DESC but ordering keys still in ASC order, -# causing merge_batch to pair each value with the wrong key (silent wrong results). -query TT -explain select array_agg(c1 order by c1), array_agg(c1 order by c1 desc) from agg_order; ----- -logical_plan -01)Aggregate: groupBy=[[]], aggr=[[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]]] -02)--TableScan: agg_order projection=[c1] -physical_plan -01)AggregateExec: mode=Final, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]] -02)--CoalescePartitionsExec -03)----AggregateExec: mode=Partial, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]] -04)------SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[true] -05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_agg_multi_order.csv]]}, projection=[c1], file_type=csv, has_header=true - -query ?? -select array_agg(c1 order by c1), array_agg(c1 order by c1 desc) from agg_order; ----- -[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] - -# test array_agg_order with list data type -statement ok -CREATE TABLE array_agg_order_list_table AS VALUES - ('w', 2, [1,2,3], 10), - ('w', 1, [9,5,2], 20), - ('w', 1, [3,2,5], 30), - ('b', 2, [4,5,6], 20), - ('b', 1, [7,8,9], 30) -; - -query T? rowsort -select column1, array_agg(column3 order by column2, column4 desc) from array_agg_order_list_table group by column1; ----- -b [[7, 8, 9], [4, 5, 6]] -w [[3, 2, 5], [9, 5, 2], [1, 2, 3]] - -query T?? rowsort -select column1, first_value(column3 order by column2, column4 desc), last_value(column3 order by column2, column4 desc) from array_agg_order_list_table group by column1; ----- -b [7, 8, 9] [4, 5, 6] -w [3, 2, 5] [1, 2, 3] - -query T? rowsort -select column1, nth_value(column3, 2 order by column2, column4 desc) from array_agg_order_list_table group by column1; ----- -b [4, 5, 6] -w [9, 5, 2] - -query ? -select array_agg(DISTINCT column2 order by column2) from array_agg_order_list_table; ----- -[1, 2] - -query ? -select array_agg(DISTINCT column2 order by column2 desc) from array_agg_order_list_table; ----- -[2, 1] - -query ? -select array_agg(DISTINCT column2 + 1 order by column2 + 1 desc) from array_agg_order_list_table; ----- -[3, 2] - -query ? -select array_agg(DISTINCT column2 order by column2) from array_agg_order_list_table GROUP BY column1; ----- -[1, 2] -[1, 2] - -statement error In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list -select array_agg(DISTINCT column2 order by column1) from array_agg_order_list_table; - -statement ok -drop table array_agg_order_list_table; - -# test array_agg_distinct with list data type -statement ok -CREATE TABLE array_agg_distinct_list_table AS VALUES - ('w', [0,1]), - ('w', [0,1]), - ('w', [1,0]), - ('b', [1,0]), - ('b', [1,0]), - ('b', [1,0]), - ('b', [0,1]), - (NULL, [0,1]), - ('b', NULL) -; - -# Apply array_sort to have deterministic result, higher dimension nested array also works but not for array sort, -# so they are covered in `datafusion/functions-aggregate/src/array_agg.rs` -query ?? -select array_sort(c1), array_sort(c2) from ( - select array_agg(distinct column1) as c1, array_agg(distinct column2) ignore nulls as c2 from array_agg_distinct_list_table -); ----- -[NULL, b, w] [[0, 1], [1, 0]] - -statement ok -drop table array_agg_distinct_list_table; - -# Test array_agg with DISTINCT and IGNORE NULLS (regression test for issue #19735) -query ? -SELECT array_sort(ARRAY_AGG(DISTINCT x IGNORE NULLS)) as result -FROM (VALUES (1), (2), (NULL), (2), (NULL), (1)) AS t(x); ----- -[1, 2] # Test that non-DISTINCT aggregates also preserve IGNORE NULLS when mixed with DISTINCT # This tests the two-phase aggregation rewrite in SingleDistinctToGroupBy @@ -481,75 +259,6 @@ FROM (VALUES ---- 2 [40, 30, 20, 10] -statement error This feature is not implemented: Calling array_agg: LIMIT not supported in function arguments: 1 -SELECT array_agg(c13 LIMIT 1) FROM aggregate_test_100 - - -# Test distinct aggregate function with merge batch -query II -with A as ( - select 1 as id, 2 as foo - UNION ALL - select 1, null - UNION ALL - select 1, null - UNION ALL - select 1, 3 - UNION ALL - select 1, 2 - ---- The order is non-deterministic, verify with length -) select array_length(array_agg(distinct a.foo)), sum(distinct 1) from A a group by a.id; ----- -3 1 - -# It has only AggregateExec with FinalPartitioned mode, so `merge_batch` is used -# If the plan is changed, whether the `merge_batch` is used should be verified to ensure the test coverage -query TT -explain with A as ( - select 1 as id, 2 as foo - UNION ALL - select 1, null - UNION ALL - select 1, null - UNION ALL - select 1, 3 - UNION ALL - select 1, 2 -) select array_length(array_agg(distinct a.foo)), sum(distinct 1) from A a group by a.id; ----- -logical_plan -01)Projection: array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1)) -02)--Aggregate: groupBy=[[a.id]], aggr=[[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]] -03)----SubqueryAlias: a -04)------SubqueryAlias: a -05)--------Union -06)----------Projection: Int64(1) AS id, Int64(2) AS foo -07)------------EmptyRelation: rows=1 -08)----------Projection: Int64(1) AS id, Int64(NULL) AS foo -09)------------EmptyRelation: rows=1 -10)----------Projection: Int64(1) AS id, Int64(NULL) AS foo -11)------------EmptyRelation: rows=1 -12)----------Projection: Int64(1) AS id, Int64(3) AS foo -13)------------EmptyRelation: rows=1 -14)----------Projection: Int64(1) AS id, Int64(2) AS foo -15)------------EmptyRelation: rows=1 -physical_plan -01)ProjectionExec: expr=[array_length(array_agg(DISTINCT a.foo)@1) as array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))] -02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted -03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5 -04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted -05)--------UnionExec -06)----------ProjectionExec: expr=[1 as id, 2 as foo] -07)------------PlaceholderRowExec -08)----------ProjectionExec: expr=[1 as id, NULL as foo] -09)------------PlaceholderRowExec -10)----------ProjectionExec: expr=[1 as id, NULL as foo] -11)------------PlaceholderRowExec -12)----------ProjectionExec: expr=[1 as id, 3 as foo] -13)------------PlaceholderRowExec -14)----------ProjectionExec: expr=[1 as id, 2 as foo] -15)------------PlaceholderRowExec - # FIX: custom absolute values # csv_query_avg_multi_batch @@ -2327,7 +2036,6 @@ statement ok DROP TABLE approx_distinct_interval_test; - ## This test executes the APPROX_PERCENTILE_CONT aggregation against the test ## data, asserting the estimated quantiles are ±5% their actual values. ## @@ -2875,7 +2583,6 @@ d 2.444444444444 25.444444444444 e 3 40.333333333333 - query TR SELECT c1, approx_percentile_cont(0.95) WITHIN GROUP (ORDER BY c3 DESC) AS c3_p95 FROM aggregate_test_100 GROUP BY 1 ORDER BY 1 ---- @@ -3143,23 +2850,6 @@ SELECT count(1 + 1) ---- 1 -# csv_query_array_agg -query ? -SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 ORDER BY c13 LIMIT 2) test ----- -[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB] - -# csv_query_array_agg_empty -query ? -SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 LIMIT 0) test ----- -NULL - -# csv_query_array_agg_one -query ? -SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 ORDER BY c13 LIMIT 1) test ----- -[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm] # csv_query_array_agg_with_overflow query IIRIII @@ -3212,12 +2902,6 @@ NULL 4 29 1.260869565217 123 -117 23 NULL 5 -194 -13.857142857143 118 -101 14 NULL NULL 781 7.81 125 -117 100 -# select with count to forces array_agg_distinct function, since single distinct expression is converted to group by by optimizer -# csv_query_array_agg_distinct -query ?I -SELECT array_sort(array_agg(distinct c2)), count(1) FROM aggregate_test_100 ----- -[1, 2, 3, 4, 5] 100 # aggregate_time_min_and_max query TT @@ -3380,7 +3064,6 @@ SELECT max(c1) FROM test; 3 - # count_basic statement ok create table t (c int) as values (1), (2), (null), (3), (null), (4), (5); @@ -4742,177 +4425,6 @@ SELECT percentile_cont(0.75) WITHIN GROUP (ORDER BY v DESC) FROM (VALUES (1), (2 ---- 2.75 -# array_agg_zero -query ? -SELECT ARRAY_AGG([]) ----- -[[]] - -# array_agg_one -query ? -SELECT ARRAY_AGG([1]) ----- -[[1]] - -# test array_agg with no row qualified -statement ok -create table t(a int, b float, c bigint) as values (1, 1.2, 2); - -# returns NULL, follows DuckDB's behaviour -query ? -select array_agg(a) from t where a > 2; ----- -NULL - -query ? -select array_agg(b) from t where b > 3.1; ----- -NULL - -query ? -select array_agg(c) from t where c > 3; ----- -NULL - -query ?I -select array_agg(c), count(1) from t where c > 3; ----- -NULL 0 - -# returns 0 rows if group by is applied, follows DuckDB's behaviour -query ? -select array_agg(a) from t where a > 3 group by a; ----- - -query ?I -select array_agg(a), count(1) from t where a > 3 group by a; ----- - -# returns NULL, follows DuckDB's behaviour -query ? -select array_agg(distinct a) from t where a > 3; ----- -NULL - -query ?I -select array_agg(distinct a), count(1) from t where a > 3; ----- -NULL 0 - -# returns 0 rows if group by is applied, follows DuckDB's behaviour -query ? -select array_agg(distinct a) from t where a > 3 group by a; ----- - -query ?I -select array_agg(distinct a), count(1) from t where a > 3 group by a; ----- - -# test order sensitive array agg -query ? -select array_agg(a order by a) from t where a > 3; ----- -NULL - -query ? -select array_agg(a order by a) from t where a > 3 group by a; ----- - -query ?I -select array_agg(a order by a), count(1) from t where a > 3 group by a; ----- - -statement ok -drop table t; - -# test with no values -statement ok -create table t(a int, b float, c bigint); - -query ? -select array_agg(a) from t; ----- -NULL - -query ? -select array_agg(b) from t; ----- -NULL - -query ? -select array_agg(c) from t; ----- -NULL - -query ?I -select array_agg(distinct a), count(1) from t; ----- -NULL 0 - -query ?I -select array_agg(distinct b), count(1) from t; ----- -NULL 0 - -query ?I -select array_agg(distinct b), count(1) from t; ----- -NULL 0 - -statement ok -drop table t; - - -# array_agg_i32 -statement ok -create table t (c1 int) as values (1), (2), (3), (4), (5); - -query ? -select array_agg(c1) from t; ----- -[1, 2, 3, 4, 5] - -statement ok -drop table t; - -# array_agg_nested -statement ok -create table t as values (make_array([1, 2, 3], [4, 5])), (make_array([6], [7, 8])), (make_array([9])); - -query ? -select array_agg(column1) from t; ----- -[[[1, 2, 3], [4, 5]], [[6], [7, 8]], [[9]]] - -statement ok -drop table t; - -# array_agg_ignore_nulls -statement ok -create table t as values (NULL, ''), (1, 'c'), (2, 'a'), (NULL, 'b'), (4, NULL), (NULL, NULL), (5, 'a'); - -query ? -select array_agg(column1) ignore nulls as c1 from t; ----- -[1, 2, 4, 5] - -query II -select count(*), array_length(array_agg(distinct column2) ignore nulls) from t; ----- -7 4 - -query ? -select array_agg(column2 order by column1) ignore nulls from t; ----- -[c, a, a, , b] - -query ? -select array_agg(DISTINCT column2 order by column2) ignore nulls from t; ----- -[, a, b, c] - -statement ok -drop table t; # variance_single_value query RRRR @@ -4927,7 +4439,6 @@ select var(sq.column1), var_pop(sq.column1), stddev(sq.column1), stddev_pop(sq.c 2 1 1.414213562373 1 - # aggregates on empty tables statement ok CREATE TABLE empty (column1 bigint, column2 int); @@ -5765,7 +5276,6 @@ DROP TABLE min_bool; ################# - ################# # min_max on strings/binary with null values and groups ################# @@ -6475,7 +5985,6 @@ ORDER BY tag 426172 426172 1 426172 426172 1 - statement ok drop table t_source; @@ -7441,7 +6950,6 @@ statement error select regr_sxy(NULL, 'bar'); - # regr_*() NULL results query RRIRRRRRR select regr_slope(1,1), regr_intercept(1,1), regr_count(1,1), regr_r2(1,1), regr_avgx(1,1), regr_avgy(1,1), regr_sxx(1,1), regr_syy(1,1), regr_sxy(1,1); @@ -7469,7 +6977,6 @@ select regr_slope(column2, column1), regr_intercept(column2, column1), regr_coun NULL NULL 3 NULL 1 4 0 8 0 - # regr_*() basic tests query RRIRRRRRR select @@ -7574,7 +7081,6 @@ b 3 0 2 1 2 6 2 18 6 c NULL NULL 1 NULL 1 10 0 0 0 - # regr_*() testing merge_batch() from RegrAccumulator's internal implementation statement ok set datafusion.execution.batch_size = 1; @@ -7634,7 +7140,6 @@ statement ok set datafusion.execution.batch_size = 8192; - # regr_*() testing retract_batch() from RegrAccumulator's internal implementation query RRIRRRRRR SELECT @@ -8104,13 +7609,11 @@ statement ok drop table distinct_count_large_binary_table; - ## Cleanup from distinct count tests statement ok drop table distinct_count_string_table; - # rule `aggregate_statistics` should not optimize MIN/MAX to wrong values on empty relation statement ok @@ -9133,19 +8636,6 @@ VALUES ---- x 1 -query error Error during planning: WITHIN GROUP is only supported for ordered-set aggregate functions -SELECT array_agg(a_varchar) WITHIN GROUP (ORDER BY a_varchar) -FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); - - -query error Error during planning: WITHIN GROUP is only supported for ordered-set aggregate functions -SELECT array_agg(DISTINCT a_varchar) WITHIN GROUP (ORDER BY a_varchar) -FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); - - -query error Error during planning: ORDER BY and WITHIN GROUP clauses cannot be used together in the same aggregate function -SELECT array_agg(a_varchar order by a_varchar) WITHIN GROUP (ORDER BY a_varchar) -FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); # distinct average statement ok diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.slt new file mode 100644 index 0000000000000..f44e7f7d02e9c --- /dev/null +++ b/datafusion/sqllogictest/test_files/array_agg.slt @@ -0,0 +1,620 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +####### +# Tests for the array_agg aggregate function. +# +# Sliding (bounded) window frames, which exercise `retract_batch`, live in +# `array_agg_sliding_window.slt`. +####### + +####### +# Setup test data table +####### +statement ok +CREATE EXTERNAL TABLE aggregate_test_100 ( + c1 VARCHAR NOT NULL, + c2 TINYINT NOT NULL, + c3 SMALLINT NOT NULL, + c4 SMALLINT, + c5 INT, + c6 BIGINT NOT NULL, + c7 SMALLINT NOT NULL, + c8 INT NOT NULL, + c9 INT UNSIGNED NOT NULL, + c10 BIGINT UNSIGNED NOT NULL, + c11 FLOAT NOT NULL, + c12 DOUBLE NOT NULL, + c13 VARCHAR NOT NULL, + c14 DATE NOT NULL, + c15 TIMESTAMP NOT NULL, +) +STORED AS CSV +LOCATION '../../testing/data/csv/aggregate_test_100_with_dates.csv' +OPTIONS ('format.has_header' 'true'); + +####### +# Basic array_agg +####### + +# csv_query_array_agg +query ? +SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 ORDER BY c13 LIMIT 2) test +---- +[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB] + +# csv_query_array_agg_empty +query ? +SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 LIMIT 0) test +---- +NULL + +# csv_query_array_agg_one +query ? +SELECT array_agg(c13) FROM (SELECT * FROM aggregate_test_100 ORDER BY c13 LIMIT 1) test +---- +[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm] + +# array_agg_zero +query ? +SELECT ARRAY_AGG([]) +---- +[[]] + +# array_agg_one +query ? +SELECT ARRAY_AGG([1]) +---- +[[1]] + +# test array_agg with no row qualified +statement ok +create table t(a int, b float, c bigint) as values (1, 1.2, 2); + +# returns NULL, follows DuckDB's behaviour +query ? +select array_agg(a) from t where a > 2; +---- +NULL + +query ? +select array_agg(b) from t where b > 3.1; +---- +NULL + +query ? +select array_agg(c) from t where c > 3; +---- +NULL + +query ?I +select array_agg(c), count(1) from t where c > 3; +---- +NULL 0 + +# returns 0 rows if group by is applied, follows DuckDB's behaviour +query ? +select array_agg(a) from t where a > 3 group by a; +---- + +query ?I +select array_agg(a), count(1) from t where a > 3 group by a; +---- + +# returns NULL, follows DuckDB's behaviour +query ? +select array_agg(distinct a) from t where a > 3; +---- +NULL + +query ?I +select array_agg(distinct a), count(1) from t where a > 3; +---- +NULL 0 + +# returns 0 rows if group by is applied, follows DuckDB's behaviour +query ? +select array_agg(distinct a) from t where a > 3 group by a; +---- + +query ?I +select array_agg(distinct a), count(1) from t where a > 3 group by a; +---- + +# test order sensitive array agg +query ? +select array_agg(a order by a) from t where a > 3; +---- +NULL + +query ? +select array_agg(a order by a) from t where a > 3 group by a; +---- + +query ?I +select array_agg(a order by a), count(1) from t where a > 3 group by a; +---- + +statement ok +drop table t; + +# test with no values +statement ok +create table t(a int, b float, c bigint); + +query ? +select array_agg(a) from t; +---- +NULL + +query ? +select array_agg(b) from t; +---- +NULL + +query ? +select array_agg(c) from t; +---- +NULL + +query ?I +select array_agg(distinct a), count(1) from t; +---- +NULL 0 + +query ?I +select array_agg(distinct b), count(1) from t; +---- +NULL 0 + +query ?I +select array_agg(distinct b), count(1) from t; +---- +NULL 0 + +statement ok +drop table t; + + +# array_agg_i32 +statement ok +create table t (c1 int) as values (1), (2), (3), (4), (5); + +query ? +select array_agg(c1) from t; +---- +[1, 2, 3, 4, 5] + +statement ok +drop table t; + +# array_agg_nested +statement ok +create table t as values (make_array([1, 2, 3], [4, 5])), (make_array([6], [7, 8])), (make_array([9])); + +query ? +select array_agg(column1) from t; +---- +[[[1, 2, 3], [4, 5]], [[6], [7, 8]], [[9]]] + +statement ok +drop table t; + +# array_agg_ignore_nulls +statement ok +create table t as values (NULL, ''), (1, 'c'), (2, 'a'), (NULL, 'b'), (4, NULL), (NULL, NULL), (5, 'a'); + +query ? +select array_agg(column1) ignore nulls as c1 from t; +---- +[1, 2, 4, 5] + +query II +select count(*), array_length(array_agg(distinct column2) ignore nulls) from t; +---- +7 4 + +query ? +select array_agg(column2 order by column1) ignore nulls from t; +---- +[c, a, a, , b] + +query ? +select array_agg(DISTINCT column2 order by column2) ignore nulls from t; +---- +[, a, b, c] + +statement ok +drop table t; + +####### +# array_agg with ORDER BY +####### + +# array agg can use order by +query ? +SELECT array_agg(c13 ORDER BY c13) +FROM + (SELECT * + FROM aggregate_test_100 + ORDER BY c13 + LIMIT 5) as t1 +---- +[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8] + +# array agg can use order by with distinct +query ? +SELECT array_agg(DISTINCT c13 ORDER BY c13) +FROM + (SELECT * + FROM aggregate_test_100 + ORDER BY c13 + LIMIT 5) as t1 +---- +[0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8] + +query error Execution error: In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list +SELECT array_agg(DISTINCT c13 ORDER BY c12) +FROM aggregate_test_100 + +query error Execution error: In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list +SELECT array_agg(DISTINCT c13 ORDER BY c13, c12) +FROM aggregate_test_100 + +query ?? rowsort +with tbl as (SELECT * FROM (VALUES ('xxx', 'yyy'), ('xxx', 'yyy'), ('xxx2', 'yyy2')) AS t(x, y)) +select + array_agg(x order by x) as x_agg, + array_agg(y order by y) as y_agg +from tbl +group by all +---- +[xxx, xxx, xxx2] [yyy, yyy, yyy2] + +query ?? +SELECT + (SELECT array_agg(c12 ORDER BY c12) FROM aggregate_test_100), + (SELECT array_agg(c13 ORDER BY c13) FROM aggregate_test_100) +---- +[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] + +query ?? +SELECT + array_agg(c12 ORDER BY c12), + array_agg(c13 ORDER BY c13) +FROM aggregate_test_100 +---- +[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] + +query ?? rowsort +with tbl as (SELECT * FROM (VALUES ('xxx', 'yyy'), ('xxx', 'yyy'), ('xxx2', 'yyy2')) AS t(x, y)) +select + array_agg(distinct x order by x) as x_agg, + array_agg(distinct y order by y) as y_agg +from tbl +group by all +---- +[xxx, xxx2] [yyy, yyy2] + +query ?? +SELECT + (SELECT array_agg(DISTINCT c12 ORDER BY c12) FROM aggregate_test_100), + (SELECT array_agg(DISTINCT c13 ORDER BY c13) FROM aggregate_test_100) +---- +[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] + +query ?? +SELECT + array_agg(DISTINCT c12 ORDER BY c12), + array_agg(DISTINCT c13 ORDER BY c13) +FROM aggregate_test_100 +---- +[0.01479305307777301, 0.02182578039211991, 0.03968347085780355, 0.04429073092078406, 0.047343434291126085, 0.04893135681998029, 0.0494924465469434, 0.05573662213439634, 0.05636955101974106, 0.061029375346466685, 0.07260475960924484, 0.09465635123783445, 0.12357539988406441, 0.152498292971736, 0.16301110515739792, 0.1640882545084913, 0.1754261586710173, 0.17592486905979987, 0.17909035118828576, 0.18628859265874176, 0.19113293583306745, 0.2145232647388039, 0.21535402343780985, 0.24899794314659673, 0.2537253407987472, 0.2667177795079635, 0.27159190516490006, 0.2739938529235548, 0.28534428578703896, 0.2944158618048994, 0.296036538664718, 0.3051364088814128, 0.30585375151301186, 0.3114712539863804, 0.3231750610081745, 0.32869374687050157, 0.33639590659276175, 0.3600766362333053, 0.36936304600612724, 0.38870280983958583, 0.39144436569161134, 0.40342283197779727, 0.4094218353587008, 0.40975383525297016, 0.42073125331890115, 0.4273123318932347, 0.42950521730777025, 0.4830878559436823, 0.5081765563442366, 0.5437595540422571, 0.5590205548347534, 0.5593249815276734, 0.5603062368164834, 0.560333188635217, 0.5614503754617461, 0.565352842229935, 0.574210838214554, 0.5759450483859969, 0.5773498217058918, 0.5991138115095911, 0.6009475544728957, 0.6108938307533, 0.6316565296547284, 0.6404495093354053, 0.6405262429561641, 0.6425694115212065, 0.658671129040488, 0.6668423897406515, 0.6864391962767343, 0.7035635283169166, 0.7325106678655877, 0.7328050041291218, 0.7614304100703713, 0.7631239070049998, 0.7670021786149205, 0.7697753383420857, 0.7764360990307122, 0.7784918983501654, 0.7973920072996036, 0.819715865079681, 0.8506721053047003, 0.8813167497816289, 0.8824879447595726, 0.9185813970744787, 0.9231889896940375, 0.9237877978193884, 0.9255031346434324, 0.9293883502480845, 0.9294097332465232, 0.9463098243875633, 0.946325164889271, 0.9491397432856566, 0.9567595541247681, 0.9706712283358269, 0.9723580396501548, 0.9748360509016578, 0.9800193410444061, 0.980809631269599, 0.991517828651004, 0.9965400387585364] [0VVIHzxWtNOFLtnhjHEKjXaJOSLJfm, 0keZ5G8BffGwgF2RwQD59TFzMStxCB, 0og6hSkhbX8AC1ktFS4kounvTzy8Vo, 1aOcrEGd0cOqZe2I5XBOm0nDcwtBZO, 2T3wSlHdEmASmO0xcXHnndkKEt6bz8, 3BEOHQsMEFZ58VcNTOJYShTBpAPzbt, 4HX6feIvmNXBN7XGqgO4YVBkhu8GDI, 4JznSdBajNWhu4hRQwjV1FjTTxY68i, 52mKlRE3aHCBZtjECq6sY9OqVf8Dze, 56MZa5O1hVtX4c5sbnCfxuX5kDChqI, 6FPJlLAcaQ5uokyOWZ9HGdLZObFvOZ, 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW, 6oIXZuIPIqEoPBvFmbt2Nxy3tryGUE, 6x93sxYioWuq5c9Kkk8oTAAORM7cH0, 802bgTGl6Bk5TlkPYYTxp5JkKyaYUA, 8LIh0b6jmDGm87BmIyjdxNIpX4ugjD, 90gAtmGEeIqUTbo1ZrxCvWtsseukXC, 9UbObCsVkmYpJGcGrgfK90qOnwb2Lj, AFGCj7OWlEB5QfniEFgonMq90Tq5uH, ALuRhobVWbnQTTWZdSOk0iVe8oYFhW, Amn2K87Db5Es3dFQO9cw9cvpAM6h35, AyYVExXK6AR2qUTxNZ7qRHQOVGMLcz, BJqx5WokrmrrezZA0dUbleMYkG5U2O, BPtQMxnuSPpxMExYV9YkDa6cAN7GP3, BsM5ZAYifRh5Lw3Y8X1r53I0cTJnfE, C2GT5KVyOPZpgKVl110TyZO0NcJ434, DuJNG8tufSqW0ZstHqWj3aGvFLMg4A, EcCuckwsF3gV1Ecgmh5v4KM8g1ozif, ErJFw6hzZ5fmI5r8bhE4JzlscnhKZU, F7NSTjWvQJyBburN7CXRUlbgp2dIrA, Fi4rJeTQq4eXj8Lxg3Hja5hBVTVV5u, H5j5ZHy1FGesOAHjkQEDYCucbpKWRu, HKSMQ9nTnwXCJIte1JrM1dtYnDtJ8g, IWl0G3ZlMNf7WT8yjIB49cx7MmYOmr, IZTkHMLvIKuiLjhDjYMmIHxh166we4, Ig1QcuKsjHXkproePdERo2w0mYzIqd, JHNgc2UCaiXOdmkxwDDyGhRlO0mnBQ, JN0VclewmjwYlSl8386MlWv5rEhWCz, JafwVLSVk5AVoXFuzclesQ000EE2k1, KJFcmTVjdkCMv94wYCtfHMFhzyRsmH, Ktb7GQ0N1DrxwkCkEUsTaIXk0xYinn, Ld2ej8NEv5zNcqU60FwpHeZKBhfpiV, LiEBxds3X0Uw0lxiYjDqrkAaAwoiIW, MXhhH1Var3OzzJCtI9VNyYvA0q8UyJ, MeSTAXq8gVxVjbEjgkvU9YLte0X9uE, NEhyk8uIx4kEULJGa8qIyFjjBcP2G6, O66j6PaYuZhEUtqV6fuU7TyjM2WxC5, OF7fQ37GzaZ5ikA2oMyvleKtgnLjXh, OPwBqCEK5PWTjWaiOyL45u2NLTaDWv, Oq6J4Rx6nde0YlhOIJkFsX2MsSvAQ0, Ow5PGpfTm4dXCfTDsXAOTatXRoAydR, QEHVvcP8gxI6EMJIrvcnIhgzPNjIvv, QJYm7YRA3YetcBHI5wkMZeLXVmfuNy, QYlaIAnJA6r8rlAb6f59wcxvcPcWFf, RilTlL1tKkPOUFuzmLydHAVZwv1OGl, Sfx0vxv1skzZWT1PqVdoRDdO6Sb6xH, TTQUwpMNSXZqVBKAFvXu7OlWvKXJKX, TtDKUZxzVxsq758G6AWPSYuZgVgbcl, VDhtJkYjAYPykCgOU9x3v7v3t4SO1a, VY0zXmXeksCT8BzvpzpPLbmU9Kp9Y4, Vp3gmWunM5A7wOC9YW2JroFqTWjvTi, WHmjWk2AY4c6m7DA4GitUx6nmb1yYS, XemNcT1xp61xcM1Qz3wZ1VECCnq06O, Z2sWcQr0qyCJRMHDpRy3aQr7PkHtkK, aDxBtor7Icd9C5hnTvvw5NrIre740e, akiiY5N0I44CMwEnBL6RTBk7BRkxEj, b3b9esRhTzFEawbs6XhpKnD9ojutHB, bgK1r6v3BCTh0aejJUhkA1Hn6idXGp, cBGc0kSm32ylBDnxogG727C0uhZEYZ, cq4WSAIFwx3wwTUS5bp1wCe71R6U5I, dVdvo6nUD5FgCgsbOZLds28RyGTpnx, e2Gh6Ov8XkXoFdJWhl0EjwEHlMDYyG, f9ALCzwDAKmdu7Rk2msJaB1wxe5IBX, fuyvs0w7WsKSlXqJ1e6HFSoLmx03AG, gTpyQnEODMcpsPnJMZC66gh33i3m0b, gpo8K5qtYePve6jyPt6xgJx4YOVjms, gxfHWUF8XgY2KdFxigxvNEXe2V2XMl, i6RQVXKUh7MzuGMDaNclUYnFUAireU, ioEncce3mPOXD2hWhpZpCPWGATG6GU, jQimhdepw3GKmioWUlVSWeBVRKFkY3, l7uwDoTepWwnAP0ufqtHJS3CRi7RfP, lqhzgLsXZ8JhtpeeUWWNbMz8PHI705, m6jD0LBIQWaMfenwRCTANI9eOdyyto, mhjME0zBHbrK6NMkytMTQzOssOa1gF, mzbkwXKrPeZnxg2Kn1LRF5hYSsmksS, nYVJnVicpGRqKZibHyBAmtmzBXAFfT, oHJMNvWuunsIMIWFnYG31RCfkOo2V7, oLZ21P2JEDooxV1pU31cIxQHEeeoLu, okOkcWflkNXIy4R8LzmySyY1EC3sYd, pLk3i59bZwd5KBZrI1FiweYTd5hteG, pTeu0WMjBRTaNRT15rLCuEh3tBJVc5, qnPOOmslCJaT45buUisMRnM0rc77EK, t6fQUjJejPcjc04wHvHTPe55S65B4V, ukOiFGGFnQJDHFgZxHMpvhD3zybF0M, ukyD7b0Efj7tNlFSRmzZ0IqkEzg2a8, waIGbOGl1PM6gnzZ4uuZt4E2yDWRHs, wwXqSGKLyBQyPkonlzBNYUJTCo4LRS, xipQ93429ksjNcXPX5326VSg1xJZcW, y7C453hRWd4E7ImjNDWlpexB8nUqjh, ydkwycaISlYSlEq3TlkS2m15I2pcp8] + +statement ok +CREATE EXTERNAL TABLE agg_order ( +c1 INT NOT NULL, +c2 INT NOT NULL, +c3 INT NOT NULL +) +STORED AS CSV +LOCATION '../core/tests/data/aggregate_agg_multi_order.csv' +OPTIONS ('format.has_header' 'true'); + +# test array_agg with order by multiple columns +query ? +select array_agg(c1 order by c2 desc, c3) from agg_order; +---- +[5, 6, 7, 8, 9, 1, 2, 3, 4, 10] + +query TT +explain select array_agg(c1 order by c2 desc, c3) from agg_order; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]]] +02)--TableScan: agg_order projection=[c1, c2, c3] +physical_plan +01)AggregateExec: mode=Final, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]] +02)--CoalescePartitionsExec +03)----AggregateExec: mode=Partial, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c2 DESC NULLS FIRST, agg_order.c3 ASC NULLS LAST]] +04)------SortExec: expr=[c2@1 DESC, c3@2 ASC NULLS LAST], preserve_partitioning=[true] +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_agg_multi_order.csv]]}, projection=[c1, c2, c3], file_type=csv, has_header=true + +# Regression test: ARRAY_AGG with conflicting ASC/DESC ORDER BY in the same query. +# get_finer_aggregate_exprs_requirement picks ASC as the common requirement and +# reverses the DESC aggregate (is_reversed=true, ordering_req=[ASC]). +# The optimizer then sets is_input_pre_ordered=true on both. Without the fix, +# state() emits values reversed to DESC but ordering keys still in ASC order, +# causing merge_batch to pair each value with the wrong key (silent wrong results). +query TT +explain select array_agg(c1 order by c1), array_agg(c1 order by c1 desc) from agg_order; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]]] +02)--TableScan: agg_order projection=[c1] +physical_plan +01)AggregateExec: mode=Final, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]] +02)--CoalescePartitionsExec +03)----AggregateExec: mode=Partial, gby=[], aggr=[array_agg(agg_order.c1) ORDER BY [agg_order.c1 ASC NULLS LAST], array_agg(agg_order.c1) ORDER BY [agg_order.c1 DESC NULLS FIRST]] +04)------SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[true] +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_agg_multi_order.csv]]}, projection=[c1], file_type=csv, has_header=true + +query ?? +select array_agg(c1 order by c1), array_agg(c1 order by c1 desc) from agg_order; +---- +[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + +# test array_agg_order with list data type +statement ok +CREATE TABLE array_agg_order_list_table AS VALUES + ('w', 2, [1,2,3], 10), + ('w', 1, [9,5,2], 20), + ('w', 1, [3,2,5], 30), + ('b', 2, [4,5,6], 20), + ('b', 1, [7,8,9], 30) +; + +query T? rowsort +select column1, array_agg(column3 order by column2, column4 desc) from array_agg_order_list_table group by column1; +---- +b [[7, 8, 9], [4, 5, 6]] +w [[3, 2, 5], [9, 5, 2], [1, 2, 3]] + +query T?? rowsort +select column1, first_value(column3 order by column2, column4 desc), last_value(column3 order by column2, column4 desc) from array_agg_order_list_table group by column1; +---- +b [7, 8, 9] [4, 5, 6] +w [3, 2, 5] [1, 2, 3] + +query T? rowsort +select column1, nth_value(column3, 2 order by column2, column4 desc) from array_agg_order_list_table group by column1; +---- +b [4, 5, 6] +w [9, 5, 2] + +query ? +select array_agg(DISTINCT column2 order by column2) from array_agg_order_list_table; +---- +[1, 2] + +query ? +select array_agg(DISTINCT column2 order by column2 desc) from array_agg_order_list_table; +---- +[2, 1] + +query ? +select array_agg(DISTINCT column2 + 1 order by column2 + 1 desc) from array_agg_order_list_table; +---- +[3, 2] + +query ? +select array_agg(DISTINCT column2 order by column2) from array_agg_order_list_table GROUP BY column1; +---- +[1, 2] +[1, 2] + +statement error In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list +select array_agg(DISTINCT column2 order by column1) from array_agg_order_list_table; + +statement ok +drop table array_agg_order_list_table; + +####### +# array_agg with DISTINCT +####### + +# select with count to forces array_agg_distinct function, since single distinct expression is converted to group by by optimizer +# csv_query_array_agg_distinct +query ?I +SELECT array_sort(array_agg(distinct c2)), count(1) FROM aggregate_test_100 +---- +[1, 2, 3, 4, 5] 100 + +# test array_agg_distinct with list data type +statement ok +CREATE TABLE array_agg_distinct_list_table AS VALUES + ('w', [0,1]), + ('w', [0,1]), + ('w', [1,0]), + ('b', [1,0]), + ('b', [1,0]), + ('b', [1,0]), + ('b', [0,1]), + (NULL, [0,1]), + ('b', NULL) +; + +# Apply array_sort to have deterministic result, higher dimension nested array also works but not for array sort, +# so they are covered in `datafusion/functions-aggregate/src/array_agg.rs` +query ?? +select array_sort(c1), array_sort(c2) from ( + select array_agg(distinct column1) as c1, array_agg(distinct column2) ignore nulls as c2 from array_agg_distinct_list_table +); +---- +[NULL, b, w] [[0, 1], [1, 0]] + +statement ok +drop table array_agg_distinct_list_table; + + +# Test array_agg with DISTINCT and IGNORE NULLS (regression test for issue #19735) +query ? +SELECT array_sort(ARRAY_AGG(DISTINCT x IGNORE NULLS)) as result +FROM (VALUES (1), (2), (NULL), (2), (NULL), (1)) AS t(x); +---- +[1, 2] + +# Test distinct aggregate function with merge batch +query II +with A as ( + select 1 as id, 2 as foo + UNION ALL + select 1, null + UNION ALL + select 1, null + UNION ALL + select 1, 3 + UNION ALL + select 1, 2 + ---- The order is non-deterministic, verify with length +) select array_length(array_agg(distinct a.foo)), sum(distinct 1) from A a group by a.id; +---- +3 1 + +# It has only AggregateExec with FinalPartitioned mode, so `merge_batch` is used +# If the plan is changed, whether the `merge_batch` is used should be verified to ensure the test coverage +query TT +explain with A as ( + select 1 as id, 2 as foo + UNION ALL + select 1, null + UNION ALL + select 1, null + UNION ALL + select 1, 3 + UNION ALL + select 1, 2 +) select array_length(array_agg(distinct a.foo)), sum(distinct 1) from A a group by a.id; +---- +logical_plan +01)Projection: array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1)) +02)--Aggregate: groupBy=[[a.id]], aggr=[[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))]] +03)----SubqueryAlias: a +04)------SubqueryAlias: a +05)--------Union +06)----------Projection: Int64(1) AS id, Int64(2) AS foo +07)------------EmptyRelation: rows=1 +08)----------Projection: Int64(1) AS id, Int64(NULL) AS foo +09)------------EmptyRelation: rows=1 +10)----------Projection: Int64(1) AS id, Int64(NULL) AS foo +11)------------EmptyRelation: rows=1 +12)----------Projection: Int64(1) AS id, Int64(3) AS foo +13)------------EmptyRelation: rows=1 +14)----------Projection: Int64(1) AS id, Int64(2) AS foo +15)------------EmptyRelation: rows=1 +physical_plan +01)ProjectionExec: expr=[array_length(array_agg(DISTINCT a.foo)@1) as array_length(array_agg(DISTINCT a.foo)), sum(DISTINCT Int64(1))@2 as sum(DISTINCT Int64(1))] +02)--AggregateExec: mode=FinalPartitioned, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted +03)----RepartitionExec: partitioning=Hash([id@0], 4), input_partitions=5 +04)------AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[array_agg(DISTINCT a.foo), sum(DISTINCT Int64(1))], ordering_mode=Sorted +05)--------UnionExec +06)----------ProjectionExec: expr=[1 as id, 2 as foo] +07)------------PlaceholderRowExec +08)----------ProjectionExec: expr=[1 as id, NULL as foo] +09)------------PlaceholderRowExec +10)----------ProjectionExec: expr=[1 as id, NULL as foo] +11)------------PlaceholderRowExec +12)----------ProjectionExec: expr=[1 as id, 3 as foo] +13)------------PlaceholderRowExec +14)----------ProjectionExec: expr=[1 as id, 2 as foo] +15)------------PlaceholderRowExec + +####### +# Unsupported syntax +####### + +statement error This feature is not implemented: Calling array_agg: LIMIT not supported in function arguments: 1 +SELECT array_agg(c13 LIMIT 1) FROM aggregate_test_100 + +query error Error during planning: WITHIN GROUP is only supported for ordered-set aggregate functions +SELECT array_agg(a_varchar) WITHIN GROUP (ORDER BY a_varchar) +FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); + + +query error Error during planning: WITHIN GROUP is only supported for ordered-set aggregate functions +SELECT array_agg(DISTINCT a_varchar) WITHIN GROUP (ORDER BY a_varchar) +FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); + + +query error Error during planning: ORDER BY and WITHIN GROUP clauses cannot be used together in the same aggregate function +SELECT array_agg(a_varchar order by a_varchar) WITHIN GROUP (ORDER BY a_varchar) +FROM (VALUES ('a'), ('d'), ('c'), ('a')) t(a_varchar); + +# test array_agg_distinct with dictionary encoded data +statement ok +CREATE TABLE array_agg_distinct_dict_table AS VALUES + ('w', 1), + ('w', 1), + ('b', 2), + ('b', 1), + (NULL, 2) +; + +# Apply array_sort to have deterministic result +query ?? +select array_sort(c1), array_sort(c2) from ( + select array_agg(distinct arrow_cast(column1, 'Dictionary(Int32, Utf8)')) as c1, + array_agg(distinct arrow_cast(column2, 'Dictionary(Int8, Int64)')) ignore nulls as c2 + from array_agg_distinct_dict_table +); +---- +[NULL, b, w] [1, 2] + +# The element type of the returned list must stay dictionary encoded, otherwise the +# aggregate output does not match the schema it declared +query T +select arrow_typeof(array_agg(distinct arrow_cast(column1, 'Dictionary(Int32, Utf8)'))) +from array_agg_distinct_dict_table; +---- +List(Dictionary(Int32, Utf8)) + +# ... including when the dictionary is nested inside another type +query ? +select array_sort(c) from ( + select array_agg(distinct struct(arrow_cast(column1, 'Dictionary(Int32, Utf8)') as f)) as c + from array_agg_distinct_dict_table +); +---- +[{f: NULL}, {f: b}, {f: w}] + +# ... and when no rows are aggregated at all +query ? +select array_agg(distinct arrow_cast(column1, 'Dictionary(Int32, Utf8)')) +from array_agg_distinct_dict_table where column2 > 100; +---- +NULL + +query T +select arrow_typeof(array_agg(distinct arrow_cast(column1, 'Dictionary(Int32, Utf8)'))) +from array_agg_distinct_dict_table where column2 > 100; +---- +List(Dictionary(Int32, Utf8)) + +statement ok +drop table array_agg_distinct_dict_table; From 36c417b6c14244526c81ee5dc4210740ab7b44f8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 25 Jul 2026 13:09:56 -0600 Subject: [PATCH 025/109] perf: avoid per-row String allocation in Spark bin and char (#23881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? N/A ## Rationale for this change Two Spark functions allocated a `String` per row purely to render a small, bounded amount of text: - `bin` called `format!("{value:b}")` for every row. - `char` called `ch.to_string()` for every row — a heap allocation for a single character. In both cases the output has a known upper bound (64 binary digits for an `i64`, 4 bytes for a UTF-8 character), so the rendering fits in a stack buffer and the result can be appended straight to a pre-sized builder. ## What changes are included in this PR? `math/bin.rs`: - `spark_bin` now writes digits right-aligned into a caller-supplied `[u8; 64]` and returns a `&str` borrowed from it, instead of returning an owned `String`. - The `collect::()` becomes an explicit loop over a `StringBuilder::with_capacity`, sized at 8 digits per row. - Negative values still render as their two's-complement bit pattern, matching `{:b}`. The digit loop is a `loop`, not a `while`, so zero renders as `"0"` rather than the empty string. `string/char.rs`: - `ch.to_string()` becomes `ch.encode_utf8(&mut encoded)` against a `[u8; 4]` hoisted out of the loop. Output is unchanged in both cases. ## Are these changes tested? Existing coverage pins the behaviour. `spark/math/bin.slt` asserts concrete output for the cases the rewrite had to get right: zero, negative values, `i64::MIN` (`-9223372036854775808`), `i64::MAX`, and `-2147483648` / `-32768` widened from narrower integer types. `spark/string/char.slt` covers the negative-input empty string, the null path, and characters on both sides of the ASCII boundary (`char(256)` and above wrap via `% 256`). All 119 `spark/math` and `spark/string` sqllogictest files pass, along with the 258 `datafusion-spark` unit tests. The `bin` benchmark used below, `datafusion/spark/benches/bin.rs`, is added separately in #23882 so the baseline can be measured on `main` before this change lands. It covers 1024 and 8192 rows with 20% nulls over two value distributions: small values that render to a handful of digits, and full-range values that render to the maximum 64. `char` already had `datafusion/spark/benches/char.rs` on `main`, so no benchmark change is needed for it. ### Benchmarks Criterion, `apache/main` @ `f1ab86dad` as baseline. Median of the reported change interval. | Benchmark | 1024 | 8192 | | --- | --- | --- | | `bin/small` | −75.7% | −73.7% | | `bin/wide` | −47.8% | −49.8% | `char` (1024 rows): −76.1%. ## Are there any user-facing changes? No. Both functions produce byte-identical output; this is purely an allocation change. --- datafusion/spark/src/function/math/bin.rs | 44 ++++++++++++++++---- datafusion/spark/src/function/string/char.rs | 4 +- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/datafusion/spark/src/function/math/bin.rs b/datafusion/spark/src/function/math/bin.rs index 82afd48e8dc9f..e6a0e1a7359ef 100644 --- a/datafusion/spark/src/function/math/bin.rs +++ b/datafusion/spark/src/function/math/bin.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{ArrayRef, AsArray, StringArray}; +use arrow::array::{Array, ArrayRef, AsArray, StringBuilder}; use arrow::datatypes::{DataType, Field, FieldRef, Int64Type}; use datafusion_common::types::{NativeType, logical_int64}; use datafusion_common::utils::take_function_args; @@ -88,12 +88,20 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result { let [array] = take_function_args("bin", arg)?; match &array.data_type() { DataType::Int64 => { - let result: StringArray = array - .as_primitive::() - .iter() - .map(|opt| opt.map(spark_bin)) - .collect(); - Ok(Arc::new(result)) + let array = array.as_primitive::(); + let len = array.len(); + // Most values are small, so 8 digits per row is a reasonable estimate; + // the buffer grows on its own for wider ones. + let mut builder = StringBuilder::with_capacity(len, len * 8); + // Digits are rendered into this stack buffer, so no row allocates. + let mut digits = [0u8; MAX_BIN_DIGITS]; + for value in array.iter() { + match value { + Some(value) => builder.append_value(spark_bin(value, &mut digits)), + None => builder.append_null(), + } + } + Ok(Arc::new(builder.finish())) } data_type => { internal_err!("bin does not support: {data_type}") @@ -101,6 +109,24 @@ fn spark_bin_inner(arg: &[ArrayRef]) -> Result { } } -fn spark_bin(value: i64) -> String { - format!("{value:b}") +/// An `i64` renders as at most 64 binary digits. +const MAX_BIN_DIGITS: usize = 64; + +/// Renders `value` as binary, right-aligned in `digits`, and returns the digits written. +/// +/// Negative values render as their two's-complement bit pattern, matching `{:b}`. +fn spark_bin(value: i64, digits: &mut [u8; MAX_BIN_DIGITS]) -> &str { + let mut pos = MAX_BIN_DIGITS; + let mut remaining = value as u64; + // `while` alone would produce an empty string for zero. + loop { + pos -= 1; + digits[pos] = b'0' + (remaining & 1) as u8; + remaining >>= 1; + if remaining == 0 { + break; + } + } + // SAFETY: every byte written above is an ASCII '0' or '1'. + unsafe { std::str::from_utf8_unchecked(&digits[pos..]) } } diff --git a/datafusion/spark/src/function/string/char.rs b/datafusion/spark/src/function/string/char.rs index 15b00ee98f5c7..5d6de3ae368e3 100644 --- a/datafusion/spark/src/function/string/char.rs +++ b/datafusion/spark/src/function/string/char.rs @@ -112,6 +112,8 @@ fn chr(args: &[ArrayRef]) -> Result { integer_array.len(), ); + // Each character encodes into this stack buffer, so no row allocates a `String`. + let mut encoded = [0u8; 4]; for integer_opt in integer_array { match integer_opt { Some(integer) => { @@ -119,7 +121,7 @@ fn chr(args: &[ArrayRef]) -> Result { builder.append_value(""); // empty string for negative numbers. } else { match core::char::from_u32((integer % 256) as u32) { - Some(ch) => builder.append_value(ch.to_string()), + Some(ch) => builder.append_value(ch.encode_utf8(&mut encoded)), None => { return exec_err!( "requested character not compatible for encoding." From 6d764823764bb4063d95c82ce0dac385a6cb1fc5 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Sat, 25 Jul 2026 22:54:46 +0100 Subject: [PATCH 026/109] fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848) ## Which issue does this PR close? - Closes #23847 . ## Rationale for this change Fixes a correctness bug, not sure when that config used/desirable. This is another thing I ran into during https://github.com/apache/datafusion/pull/21585 ## What changes are included in this PR? 1. New SLT test 2. Fix for bug in `datafusion-optimizer` ## Are these changes tested? Existing tests, and a new SLT test verifying the config change doesn't affect the query result and basic unit test. ## Are there any user-facing changes? None --------- Signed-off-by: Adam Gutglick --- .../optimizer/src/filter_null_join_keys.rs | 47 +++++++++++++++++++ .../test_files/null_aware_anti_join.slt | 14 ++++++ 2 files changed, 61 insertions(+) diff --git a/datafusion/optimizer/src/filter_null_join_keys.rs b/datafusion/optimizer/src/filter_null_join_keys.rs index c8f419d3e543e..e3de8048a879d 100644 --- a/datafusion/optimizer/src/filter_null_join_keys.rs +++ b/datafusion/optimizer/src/filter_null_join_keys.rs @@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys { match plan { LogicalPlan::Join(mut join) if !join.on.is_empty() + && !join.null_aware && join.null_equality == NullEquality::NullEqualsNothing => { let (left_preserved, right_preserved) = @@ -359,4 +360,50 @@ mod tests { let t2 = table_scan(Some("t2"), &schema, None)?.build()?; Ok((t1, t2)) } + + #[test] + fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> { + let (t1, t2) = test_tables()?; + let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?; + + assert_optimized_plan_equal!(plan, @r" + LeftMark Join: t1.id = t2.optional_id null_aware + TableScan: t1 + TableScan: t2 + ") + } + + #[test] + fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> { + let (t1, t2) = test_tables()?; + let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?; + + assert_optimized_plan_equal!(plan, @r" + LeftAnti Join: t1.id = t2.optional_id null_aware + TableScan: t1 + TableScan: t2 + ") + } + + /// A join whose nullable right key would get an `IS NOT NULL` filter if it + /// were not null-aware. + fn build_null_aware_plan( + left_table: LogicalPlan, + right_table: LogicalPlan, + join_type: JoinType, + ) -> Result { + LogicalPlanBuilder::from(left_table) + .join_detailed_with_options( + right_table, + join_type, + ( + vec![Column::from_qualified_name("t1.id")], + vec![Column::from_qualified_name("t2.optional_id")], + ), + None, + NullEquality::NullEqualsNothing, + true, + )? + .build() + } } diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index 1d12fc33c9a29..6061458b13dc2 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -70,6 +70,20 @@ query IT rowsort SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null); ---- +# Regression test + +statement ok +set datafusion.optimizer.filter_null_join_keys = true; + +# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or +# FALSE, so the result stays empty. +query IT rowsort +SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null); +---- + +statement ok +reset datafusion.optimizer.filter_null_join_keys; + # Verify the result is empty even though there are rows in outer_table # that don't match the non-NULL value (2) in the subquery. # This is correct null-aware behavior: if subquery contains NULL, result is unknown. From 0fcf6284fd81deddf77b04b818e26f8fa6023bd3 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sun, 26 Jul 2026 08:06:23 +0800 Subject: [PATCH 027/109] fix: don't infer join predicates for null-aware joins in push_down_filter (#23901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23900. ## Rationale for this change `push_down_filter` infers equi-key predicates across a join's ON keys and pushes them to the opposite side. For a null-aware join (the `LeftAnti` join produced by `NOT IN` with a nullable subquery), an outer predicate on the left key like `outer.id > 5` is rewritten to `sub.id > 5` and pushed onto the subquery input. Since the inferred predicate must be null-rejecting to be pushed, this drops the subquery's NULL rows and breaks the three-valued `NOT IN` semantics — a NULL in the subquery key must reach the join so the result is empty. Same class of bug as #23848, in a different rule. ## What changes are included in this PR? - Skip predicate inference in `infer_join_predicates` when `join.null_aware` is set (mirrors the #23848 guard on `FilterNullJoinKeys`). - A `push_down_filter` unit test asserting no predicate is inferred onto the subquery side of a null-aware `LeftAnti` join. - SLT coverage for the failing query, plus a `prefer_hash_join = false` / multi-partition variant. ## Are these changes tested? Yes — new unit test (verified it fails without the guard) and SLT cases. The full optimizer lib suite passes. ## Are there any user-facing changes? No, aside from the correctness fix. --- datafusion/optimizer/src/push_down_filter.rs | 56 +++++++++++++++++++ .../test_files/null_aware_anti_join.slt | 45 +++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index f30b1187b7bca..cf54ae254746d 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -576,6 +576,17 @@ fn infer_join_predicates( predicates: &[Expr], on_filters: &[Expr], ) -> Result> { + // Null-aware joins (e.g. `NOT IN` with a nullable subquery) rely on SQL + // three-valued logic: a NULL join key on the right/subquery side makes the + // predicate UNKNOWN and empties the result, so those NULLs must reach the + // join. Inferring an equi-key predicate here would rewrite a left-side + // predicate onto the right side and, because the inferred predicate must be + // null-rejecting, drop the subquery's NULL rows and produce wrong results. + // Skip inference entirely for null-aware joins. + if join.null_aware { + return Ok(vec![]); + } + // Only allow both side key is column. let join_col_keys = join .on @@ -3826,6 +3837,51 @@ mod tests { ) } + /// Regression test: for a null-aware LeftAnti join (the shape produced by + /// `NOT IN` with a nullable subquery), a right-side predicate must NOT be + /// inferred onto the join. Inference would push a null-rejecting predicate + /// to the subquery side, dropping its NULL rows and breaking the + /// three-valued `NOT IN` semantics. + #[test] + fn null_aware_left_anti_join_no_inferred_pushdown() -> Result<()> { + let table_scan = test_table_scan_with_name("test1")?; + let left = LogicalPlanBuilder::from(table_scan) + .project(vec![col("a"), col("b")])? + .build()?; + let right_table_scan = test_table_scan_with_name("test2")?; + let right = LogicalPlanBuilder::from(right_table_scan) + .project(vec![col("a"), col("b")])? + .build()?; + let plan = LogicalPlanBuilder::from(left) + .join_detailed_with_options( + right, + JoinType::LeftAnti, + ( + vec![Column::from_qualified_name("test1.a")], + vec![Column::from_qualified_name("test2.a")], + ), + None, + datafusion_common::NullEquality::NullEqualsNothing, + true, + )? + .filter(col("test1.a").gt(lit(2u32)))? + .build()?; + + // The left-side filter is pushed to the left input, but — unlike the + // non-null-aware `left_anti_join` test — no `test2.a > 2` predicate is + // inferred onto the right/subquery side. + assert_optimized_plan_equal!( + plan, + @r" + LeftAnti Join: test1.a = test2.a null_aware + Projection: test1.a, test1.b + TableScan: test1, full_filters=[test1.a > UInt32(2)] + Projection: test2.a, test2.b + TableScan: test2 + " + ) + } + #[test] fn left_anti_join_with_filters() -> Result<()> { let table_scan = test_table_scan_with_name("test1")?; diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index 6061458b13dc2..bdb56cf22045a 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -530,3 +530,48 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok RESET datafusion.optimizer.enable_join_dynamic_filter_pushdown; + +############# +## Regression: null-aware NOT IN with an outer predicate on the join key +## +## `push_down_filter` used to infer the outer predicate `id > 5` onto the +## subquery side (as `eid > 5`), dropping the subquery's NULL row and wrongly +## returning outer rows. The subquery NULL must reach the join so that +## `NOT IN` stays UNKNOWN for every row. +############# + +statement ok +CREATE TABLE nai_outer(id INT) AS VALUES (3), (7); + +statement ok +CREATE TABLE nai_inner(id INT) AS VALUES (NULL); + +# Expected: zero rows (subquery contains NULL => NOT IN is UNKNOWN for all). +query I +SELECT id FROM nai_outer WHERE id > 5 AND id NOT IN (SELECT id FROM nai_inner) ORDER BY id; +---- + +# Same query under SortMergeJoin + multiple partitions: null-aware joins must +# be planned as a CollectLeft HashJoin, not a plain anti SortMergeJoin. +statement ok +SET datafusion.optimizer.prefer_hash_join = false; + +statement ok +SET datafusion.execution.target_partitions = 4; + +query I +SELECT id FROM nai_outer WHERE id NOT IN (SELECT id FROM nai_inner) ORDER BY id; +---- + +statement ok +SET datafusion.optimizer.prefer_hash_join = true; + +# The SLT runner sets target_partitions to 4, so restore that value explicitly. +statement ok +SET datafusion.execution.target_partitions = 4; + +statement ok +DROP TABLE nai_outer; + +statement ok +DROP TABLE nai_inner; From e3e2cb227928ffa498c2845db6ce2aa86ee174b4 Mon Sep 17 00:00:00 2001 From: Geoffrey Claude Date: Sun, 26 Jul 2026 03:29:53 +0200 Subject: [PATCH 028/109] IN LIST: add branchless filter for small primitive lists (#23014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #19241. - Stacked on [#23311](https://github.com/apache/datafusion/pull/23311). - Next in stack: #23015. - Extracted from #19390. ## Rationale for this change For very small `IN` lists, building or probing a hash table can be more work than just comparing the input value with each constant. For example, for `x IN (10, 20, 30)`, the fast path can behave like: ```text x == 10 OR x == 20 OR x == 30 ``` Because the list is tiny, those comparisons are cheap. The implementation stores the constants in a fixed-size array and checks them with a compact comparison chain. “Branchless” here means the comparisons are combined without stopping at the first match. That can be faster for these small fixed-width lists because the CPU gets a predictable sequence of simple operations instead of hash-table setup and probe logic. For primitive values that are not already plain unsigned integers, this PR keeps the logical Arrow type explicit and uses a matching same-width comparison representation only inside the branchless filter. For example, `Float16` uses `UInt16` storage, `Float32` uses `UInt32` storage, and `TimestampNanosecond` uses `UInt64` storage. `Decimal128` and `IntervalMonthDayNano` use their own 16-byte native representation. This preserves bit-pattern equality while relying on Arrow's native primitive compatibility rules: timestamp timezone metadata and Decimal128 precision/scale metadata may differ, while incompatible primitive representations remain rejected. ## What changes are included in this PR? - Adds a const-generic `BranchlessFilter` for small primitive `IN` lists. - Adds thresholds for when this path is used: - up to 16 values for 1-byte types - up to 8 values for 2-byte types - up to 32 values for 4-byte types - up to 16 values for 8-byte types - up to 4 values for 16-byte types - Keeps dispatch concrete and explicit in `strategy.rs`. - Maps each optimized logical type to the comparison representation used by the branchless filter: - `Int8` -> `UInt8` - `Int16`, `Float16` -> `UInt16` - `Int32`, `Float32`, `Date32`, `Time32` -> `UInt32` - `Int64`, `Float64`, `Date64`, `Time64`, `Timestamp`, `Duration` -> `UInt64` - `Decimal128`, `IntervalMonthDayNano` -> their native 16-byte representation - Leaves larger 1-byte and 2-byte lists on the existing bitmap filters. - Leaves larger 4-byte and 8-byte lists on the existing hash/generic paths. - Leaves wider primitive types such as `Decimal256` and unsupported complex types on the generic path. - Keeps the same `IN` / `NOT IN` null behavior as the rest of the stack. - Adds focused coverage for branchless null handling, signed boundary values, slices, Float16/Float32/Float64 bit patterns, compatible timestamp/Decimal128 metadata, incompatible timestamp units, IntervalMonthDayNano values, and same-width wrong-type probe rejection. ## Are these changes tested? Yes. - `cargo fmt --all -- --check` - `cargo test -p datafusion-physical-expr expressions::in_list --lib` - `cargo test -p datafusion-physical-expr --bench in_list_strategy --no-run` - `cargo clippy --all-targets --all-features -- -D warnings` ## Are there any user-facing changes? No. This is an internal performance optimization only. ## Local benchmark snapshot Built and run with `release-nonlto`, filtered to the relevant small primitive-list rows: ```bash cargo bench -p datafusion-physical-expr --profile release-nonlto --bench in_list_strategy -- --save-baseline ``` Filters used: `narrow_integer`, `primitive/i32/small_list`, `primitive/i64/small_list`, `f32/small_list`, `timestamp_ns/small_list`, and `interval_month_day_nano/small_list`. Method: directly compared Criterion's raw sample minima (`min(time / iterations)`) from `sample.json`. Lower is better; changes within +/-5% are treated as noise. Compared baselines: [#23311](https://github.com/apache/datafusion/pull/23311) -> [#23014](https://github.com/apache/datafusion/pull/23014) Relevant scope: small primitive-list rows. Summary: 39 relevant rows, 28 faster, 0 slower, 11 within +/-5%. Largest relevant deltas: | Benchmark | Before | After | Change | |---|---:|---:|---:| | `timestamp_ns/small_list/list=4/match=50%` | 46.55 us | 3.17 us | -93.2% (14.69x faster) | | `f32/small_list/list=4/match=50%` | 33.93 us | 3.04 us | -91.0% (11.15x faster) | | `primitive/i32/small_list/list=4/match=50%` | 32.63 us | 3.08 us | -90.5% (10.58x faster) | | `primitive/i64/small_list/list=4/match=50%` | 33.55 us | 3.18 us | -90.5% (10.54x faster) | | `timestamp_ns/small_list/list=4/match=0%` | 19.57 us | 3.18 us | -83.8% (6.16x faster) | | `f32/small_list/list=4/match=0%` | 18.14 us | 3.05 us | -83.2% (5.95x faster) | | `primitive/i32/small_list/list=4/match=0%` | 17.00 us | 3.04 us | -82.1% (5.59x faster) | | `primitive/i64/small_list/list=4/match=0%` | 17.12 us | 3.22 us | -81.2% (5.31x faster) | | `primitive/i32/small_list/list=16/match=50%/NOT_IN` | 31.98 us | 7.26 us | -77.3% (4.41x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=20%` | 29.35 us | 7.32 us | -75.1% (4.01x faster) | | `timestamp_ns/small_list/list=16/match=50%` | 45.32 us | 11.79 us | -74.0% (3.84x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=50%` | 25.89 us | 7.31 us | -71.8% (3.54x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=20%/NOT_IN` | 26.05 us | 7.42 us | -71.5% (3.51x faster) | | `interval_month_day_nano/small_list/list=4/match=50%` | 52.94 us | 15.52 us | -70.7% (3.41x faster) | | `f32/small_list/list=32/match=50%` | 38.78 us | 13.27 us | -65.8% (2.92x faster) | | `primitive/i64/small_list/list=16/match=50%` | 29.46 us | 11.76 us | -60.1% (2.50x faster) |
Full relevant table (39 rows) | Benchmark | Before | After | Change | |---|---:|---:|---:| | `narrow_integer/u8/list=4/match=0%` | 3.86 us | 2.79 us | -27.8% (1.38x faster) | | `narrow_integer/u8/list=4/match=50%` | 3.84 us | 2.78 us | -27.7% (1.38x faster) | | `narrow_integer/u8/list=16/match=0%` | 3.88 us | 3.85 us | -0.8% (within noise) | | `narrow_integer/u8/list=16/match=50%` | 3.84 us | 3.86 us | +0.5% (within noise) | | `narrow_integer/i16/list=4/match=0%` | 3.93 us | 3.18 us | -19.1% (1.24x faster) | | `narrow_integer/i16/list=4/match=50%` | 3.92 us | 3.16 us | -19.5% (1.24x faster) | | `narrow_integer/i16/list=64/match=0%` | 3.96 us | 3.82 us | -3.5% (within noise) | | `narrow_integer/i16/list=64/match=50%` | 3.91 us | 3.80 us | -2.9% (within noise) | | `narrow_integer/i16/list=256/match=0%` | 3.90 us | 3.81 us | -2.5% (within noise) | | `narrow_integer/i16/list=256/match=50%` | 3.97 us | 3.81 us | -4.1% (within noise) | | `narrow_integer/f16/list=4/match=0%` | 3.87 us | 3.16 us | -18.5% (1.23x faster) | | `narrow_integer/f16/list=4/match=50%` | 3.94 us | 3.15 us | -20.2% (1.25x faster) | | `narrow_integer/f16/list=64/match=0%` | 3.87 us | 3.84 us | -0.6% (within noise) | | `narrow_integer/f16/list=64/match=50%` | 3.93 us | 3.85 us | -1.9% (within noise) | | `narrow_integer/f16/list=256/match=0%` | 3.90 us | 3.84 us | -1.5% (within noise) | | `narrow_integer/f16/list=256/match=50%` | 3.87 us | 3.91 us | +1.2% (within noise) | | `nulls/narrow_integer/u8/list=16/match=50%/nulls=20%` | 3.92 us | 4.02 us | +2.5% (within noise) | | `primitive/i32/small_list/list=4/match=0%` | 17.00 us | 3.04 us | -82.1% (5.59x faster) | | `primitive/i32/small_list/list=4/match=50%` | 32.63 us | 3.08 us | -90.5% (10.58x faster) | | `primitive/i32/small_list/list=32/match=0%` | 16.34 us | 13.33 us | -18.5% (1.23x faster) | | `primitive/i32/small_list/list=32/match=50%` | 31.17 us | 13.31 us | -57.3% (2.34x faster) | | `primitive/i32/small_list/list=16/match=50%/NOT_IN` | 31.98 us | 7.26 us | -77.3% (4.41x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=20%` | 29.35 us | 7.32 us | -75.1% (4.01x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=20%/NOT_IN` | 26.05 us | 7.42 us | -71.5% (3.51x faster) | | `nulls/primitive/i32/small_list/list=16/match=50%/nulls=50%` | 25.89 us | 7.31 us | -71.8% (3.54x faster) | | `primitive/i64/small_list/list=4/match=0%` | 17.12 us | 3.22 us | -81.2% (5.31x faster) | | `primitive/i64/small_list/list=4/match=50%` | 33.55 us | 3.18 us | -90.5% (10.54x faster) | | `primitive/i64/small_list/list=16/match=0%` | 16.34 us | 11.93 us | -27.0% (1.37x faster) | | `primitive/i64/small_list/list=16/match=50%` | 29.46 us | 11.76 us | -60.1% (2.50x faster) | | `f32/small_list/list=4/match=0%` | 18.14 us | 3.05 us | -83.2% (5.95x faster) | | `f32/small_list/list=4/match=50%` | 33.93 us | 3.04 us | -91.0% (11.15x faster) | | `f32/small_list/list=32/match=0%` | 22.05 us | 13.43 us | -39.1% (1.64x faster) | | `f32/small_list/list=32/match=50%` | 38.78 us | 13.27 us | -65.8% (2.92x faster) | | `timestamp_ns/small_list/list=4/match=0%` | 19.57 us | 3.18 us | -83.8% (6.16x faster) | | `timestamp_ns/small_list/list=4/match=50%` | 46.55 us | 3.17 us | -93.2% (14.69x faster) | | `timestamp_ns/small_list/list=16/match=0%` | 19.73 us | 12.07 us | -38.8% (1.63x faster) | | `timestamp_ns/small_list/list=16/match=50%` | 45.32 us | 11.79 us | -74.0% (3.84x faster) | | `interval_month_day_nano/small_list/list=4/match=0%` | 20.12 us | 13.20 us | -34.4% (1.52x faster) | | `interval_month_day_nano/small_list/list=4/match=50%` | 52.94 us | 15.52 us | -70.7% (3.41x faster) |
--------- Co-authored-by: Andrew Lamb --- .../physical-expr/benches/in_list_strategy.rs | 28 +- .../physical-expr/src/expressions/in_list.rs | 18 +- .../expressions/in_list/primitive_filter.rs | 483 +++++++++++++++++- .../src/expressions/in_list/strategy.rs | 163 +++++- 4 files changed, 661 insertions(+), 31 deletions(-) diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs index c70f6da2a40d9..c69af192b9cdd 100644 --- a/datafusion/physical-expr/benches/in_list_strategy.rs +++ b/datafusion/physical-expr/benches/in_list_strategy.rs @@ -37,6 +37,7 @@ //! | Narrow integer cases | Int16, Float16 | larger value domain | 4, 64, 256 | //! | 32-bit primitive cases | Int32, Float32 | small and large lists | 4, 32, 64, 256 | //! | 64-bit primitive cases | Int64, TimestampNs | small and large lists | 4, 16, 32, 128 | +//! | 128-bit interval cases | IntervalMonthDayNano | small lists | 4 | //! | Utf8 short-string cases | Utf8 | 8-byte strings | 4, 64, 256 | //! | Utf8 long-string cases | Utf8 | 24-byte strings | 4, 64, 256 | //! | Utf8View short-string cases | Utf8View | 8-byte strings | 4, 16, 64, 256 | @@ -45,8 +46,9 @@ //! | Shared-prefix string cases | Utf8, Utf8View | same prefix, different suffix | 16, 32, 64 | //! | Fixed-size binary cases | FixedSizeBinary(16) | fixed-width binary values | 4, 64, 256, 10000 | +use arrow::array::types::IntervalMonthDayNano; use arrow::array::*; -use arrow::datatypes::{Field, Int32Type, Schema}; +use arrow::datatypes::{Field, Int32Type, IntervalMonthDayNanoType, Schema}; use arrow::record_batch::RecordBatch; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use datafusion_common::ScalarValue; @@ -528,6 +530,28 @@ fn bench_timestamp_ns(c: &mut Criterion) { } } +fn bench_interval_month_day_nano(c: &mut Criterion) { + for match_pct in MATCH_RATES { + bench_numeric::( + c, + "interval_month_day_nano", + &format!("small_list/list=4/match={match_pct}%"), + &NumericBenchConfig::new( + 4, + match_pct as f64 / 100.0, + |rng| { + IntervalMonthDayNanoType::make_value( + rng.random_range(-120..=120), + rng.random_range(-31..=31), + rng.random_range(-1_000_000_000..=1_000_000_000), + ) + }, + |v| ScalarValue::IntervalMonthDayNano(Some(v)), + ), + ); + } +} + // ============================================================================= // UTF8 STRING CASE BENCHMARKS // ============================================================================= @@ -1049,7 +1073,7 @@ fn bench_fixed_size_binary(c: &mut Criterion) { criterion_group! { name = benches; config = Criterion::default(); - targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary + targets = bench_narrow_integer, bench_primitive, bench_f32, bench_timestamp_ns, bench_interval_month_day_nano, bench_utf8, bench_utf8view, bench_dictionary, bench_nulls, bench_fixed_size_binary } criterion_main!(benches); diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 2764083f31b09..e4ec72285cd3f 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -2900,10 +2900,9 @@ mod tests { #[test] fn test_in_list_esoteric_types() -> Result<()> { - // Test esoteric/less common types to validate the transform and mapping flow. - // These types are reinterpreted to base primitive types (e.g., Timestamp -> UInt64, - // Interval -> Decimal128, Float16 -> UInt16). We just need to verify basic - // functionality works - no need for comprehensive null handling tests. + // Test less common types covered by IN-list evaluation. Some of these + // use specialized filters, and others fall back to the generic path; + // this keeps the end-to-end behavior covered either way. // Helper: simple IN test that expects [Some(true), Some(false)] let test_type = |data_type: DataType, @@ -2926,7 +2925,7 @@ mod tests { Ok(()) }; - // Timestamp types (all units map to Int64 -> UInt64) + // Timestamp types test_type( DataType::Timestamp(TimeUnit::Second, None), Arc::new(TimestampSecondArray::from(vec![Some(1000), Some(2000)])), @@ -2960,7 +2959,7 @@ mod tests { ], )?; - // Time32 and Time64 (map to Int32 -> UInt32 and Int64 -> UInt64 respectively) + // Time32 and Time64 test_type( DataType::Time32(TimeUnit::Second), Arc::new(Time32SecondArray::from(vec![Some(3600), Some(7200)])), @@ -3006,7 +3005,7 @@ mod tests { ], )?; - // Duration types (map to Int64 -> UInt64) + // Duration types test_type( DataType::Duration(TimeUnit::Second), Arc::new(DurationSecondArray::from(vec![Some(86400), Some(172800)])), @@ -3052,7 +3051,7 @@ mod tests { ], )?; - // Interval types (map to 16-byte Decimal128Type) + // Interval types test_type( DataType::Interval(IntervalUnit::YearMonth), Arc::new(IntervalYearMonthArray::from(vec![Some(12), Some(24)])), @@ -3114,8 +3113,7 @@ mod tests { ], )?; - // Decimal256 (maps to Decimal128Type for 16-byte width) - // Need to use with_precision_and_scale() to set the metadata + // Decimal256. Need to use with_precision_and_scale() to set the metadata. let precision = 38; let scale = 10; test_type( diff --git a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs index 8f8d9bad04afa..e802e1d024012 100644 --- a/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs +++ b/datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs @@ -19,14 +19,15 @@ //! //! This module provides membership tests for Arrow primitive types. -use arrow::array::{Array, ArrayRef, AsArray, BooleanArray}; -use arrow::buffer::{BooleanBuffer, NullBuffer}; +use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, PrimitiveArray}; +use arrow::buffer::{BooleanBuffer, NullBuffer, ScalarBuffer}; use arrow::datatypes::*; use arrow::util::bit_iterator::BitIndexIterator; use datafusion_common::{HashSet, Result, exec_datafusion_err}; use std::hash::{Hash, Hasher}; +use std::mem::size_of; -use super::result::build_in_list_result; +use super::result::{build_in_list_result, build_result_from_contains}; use super::static_filter::{StaticFilter, handle_dictionary}; /// Storage for the bits used by [`BitmapFilter`]. @@ -222,6 +223,276 @@ where } } +pub(super) type BranchlessNative = + <::CompareType as ArrowPrimitiveType>::Native; + +/// Maximum list size for branchless lookup on 1-byte primitives. +/// +/// Sixteen 1-byte values fit in one 128-bit SIMD vector, so this keeps the +/// branchless list small enough for a single vectorized membership check. +const BRANCHLESS_MAX_1B: usize = 16; + +/// Maximum list size for branchless lookup on 2-byte primitives. +/// +/// Eight 2-byte values fit in one 128-bit SIMD vector, so this keeps the +/// branchless list small enough for a single vectorized membership check. +const BRANCHLESS_MAX_2B: usize = 8; + +/// Maximum list size for branchless lookup on 4-byte primitives. +/// +/// Thirty-two 4-byte values keep the inline list at 128 bytes. Beyond that, +/// the comparison chain and filter footprint grow enough that the hash/generic +/// fallback is a better fit. +const BRANCHLESS_MAX_4B: usize = 32; + +/// Maximum list size for branchless lookup on 8-byte primitives. +/// +/// Sixteen 8-byte values use the same 128-byte inline-list budget as 4-byte +/// primitives. Larger lists are left to the hash/generic fallback. +const BRANCHLESS_MAX_8B: usize = 16; + +/// Maximum list size for branchless lookup on 16-byte primitives. +/// +/// These comparisons are wider, so this path is limited to four values. +/// Larger lists are left to the generic fallback. +const BRANCHLESS_MAX_16B: usize = 4; + +/// Arrow primitive types supported by [`BranchlessFilter`]. +/// +/// `T` is the logical Arrow type accepted by the filter. `CompareType` is the +/// same-width type used for the fixed comparison chain. Signed integers, +/// floats, and temporal values use an unsigned comparison type so they compare +/// by their raw bit pattern. +pub(super) trait BranchlessFilterType: + ArrowPrimitiveType + Send + Sync + 'static +{ + type CompareType: ArrowPrimitiveType + Send + Sync + 'static; + + /// Maximum number of non-null IN-list values to handle with + /// [`BranchlessFilter`] for this primitive type. + const MAX_LIST_LEN: usize; +} + +macro_rules! branchless_filter_type { + ($logical:ty, $compare:ty, $max_len:expr) => { + // The branchless filter reads the same Arrow value buffer as the + // comparison type. That is only valid when both native types have the + // same width, so catch any bad mapping here at compile time. + const _: () = assert!( + size_of::<<$logical as ArrowPrimitiveType>::Native>() + == size_of::<<$compare as ArrowPrimitiveType>::Native>(), + "BranchlessFilterType::CompareType must use the same native width" + ); + + impl BranchlessFilterType for $logical { + type CompareType = $compare; + const MAX_LIST_LEN: usize = $max_len; + } + }; +} + +branchless_filter_type!(Int8Type, UInt8Type, BRANCHLESS_MAX_1B); +branchless_filter_type!(UInt8Type, UInt8Type, BRANCHLESS_MAX_1B); +branchless_filter_type!(Int16Type, UInt16Type, BRANCHLESS_MAX_2B); +branchless_filter_type!(UInt16Type, UInt16Type, BRANCHLESS_MAX_2B); +branchless_filter_type!(Float16Type, UInt16Type, BRANCHLESS_MAX_2B); + +branchless_filter_type!(Int32Type, UInt32Type, BRANCHLESS_MAX_4B); +branchless_filter_type!(UInt32Type, UInt32Type, BRANCHLESS_MAX_4B); +branchless_filter_type!(Float32Type, UInt32Type, BRANCHLESS_MAX_4B); +branchless_filter_type!(Date32Type, UInt32Type, BRANCHLESS_MAX_4B); +branchless_filter_type!(Time32SecondType, UInt32Type, BRANCHLESS_MAX_4B); +branchless_filter_type!(Time32MillisecondType, UInt32Type, BRANCHLESS_MAX_4B); + +branchless_filter_type!(Int64Type, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(UInt64Type, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(Float64Type, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(Date64Type, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(Time64MicrosecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(Time64NanosecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(TimestampSecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(TimestampMillisecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(TimestampMicrosecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(TimestampNanosecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(DurationSecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(DurationMillisecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(DurationMicrosecondType, UInt64Type, BRANCHLESS_MAX_8B); +branchless_filter_type!(DurationNanosecondType, UInt64Type, BRANCHLESS_MAX_8B); + +branchless_filter_type!(Decimal128Type, Decimal128Type, BRANCHLESS_MAX_16B); +branchless_filter_type!( + IntervalMonthDayNanoType, + IntervalMonthDayNanoType, + BRANCHLESS_MAX_16B +); + +/// Checks each input value against the `IN`-list values. +type MembershipCheck = fn(in_list_values: &[C], input_values: &[C]) -> BooleanBuffer; + +/// A branchless filter for fixed-width primitive `IN` lists up to +/// `T::MAX_LIST_LEN` values. +/// +/// The filter stores the non-null `IN`-list values in a slice and chooses a +/// comparison function for that length. Keeping the length out of +/// `BranchlessFilter` avoids generating a full copy of the filter for every +/// supported length. +pub(super) struct BranchlessFilter { + expected_data_type: DataType, + null_count: usize, + in_list_values: Box<[BranchlessNative]>, + check_values: MembershipCheck>, +} + +impl BranchlessFilter +where + T: BranchlessFilterType, + BranchlessNative: Copy + PartialEq, +{ + pub(super) fn try_new(in_array: &ArrayRef) -> Result { + let in_array = in_array.as_primitive_opt::().ok_or_else(|| { + exec_datafusion_err!("BranchlessFilter: expected {} array", T::DATA_TYPE) + })?; + let non_null_count = in_array.len() - in_array.null_count(); + // `try_new` can be called on its own, so check the limit here too. + if non_null_count > T::MAX_LIST_LEN { + return Err(exec_datafusion_err!( + "BranchlessFilter: supports at most {} non-null values, got {non_null_count}", + T::MAX_LIST_LEN + )); + } + + let all_values = branchless_values::(in_array); + let mut in_list_values = Vec::with_capacity(non_null_count); + + match in_array.nulls() { + None => { + in_list_values.extend(all_values.iter().copied()); + } + Some(nulls) => { + for row in + BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len()) + { + in_list_values.push(all_values[row]); + } + } + } + + debug_assert_eq!(in_list_values.len(), non_null_count); + let in_list_values = in_list_values.into_boxed_slice(); + let check_values = membership_check_for_len::(in_list_values.len()); + + Ok(Self { + expected_data_type: in_array.data_type().clone(), + null_count: in_array.null_count(), + in_list_values, + check_values, + }) + } +} + +impl StaticFilter for BranchlessFilter +where + T: BranchlessFilterType, + BranchlessNative: Copy + PartialEq + Send + Sync, +{ + fn null_count(&self) -> usize { + self.null_count + } + + fn contains(&self, v: &dyn Array, negated: bool) -> Result { + handle_dictionary!(self, v, negated); + + // Arrow compatibility ignores timestamp timezone and decimal precision/scale + // while still requiring the same primitive representation. + if !PrimitiveArray::::is_compatible(v.data_type()) { + return Err(exec_datafusion_err!( + "BranchlessFilter: expected {} array, got {}", + self.expected_data_type, + v.data_type() + )); + } + + let v = v.as_primitive_opt::().ok_or_else(|| { + exec_datafusion_err!("BranchlessFilter: expected {} array", T::DATA_TYPE) + })?; + let input_values = branchless_values::(v); + let matches = + (self.check_values)(self.in_list_values.as_ref(), input_values.as_ref()); + Ok(build_result_from_contains( + v.nulls(), + self.null_count > 0, + negated, + matches, + )) + } +} + +/// Picks the comparison function for `len` non-null `IN`-list values. +/// +/// A length of zero is used when the list contains only nulls. The comparisons +/// return false, and the caller then applies the usual SQL null behavior. +fn membership_check_for_len(len: usize) -> MembershipCheck> +where + T: BranchlessFilterType, + BranchlessNative: Copy + PartialEq, +{ + macro_rules! choose { + ($($n:literal),* $(,)?) => { + match len { + $($n => check_values::, $n>,)* + _ => unreachable!("list length exceeds the configured limit"), + } + }; + } + + // Avoid creating checks for lengths a type does not support. + match T::MAX_LIST_LEN { + 4 => choose!(0, 1, 2, 3, 4), + 8 => choose!(0, 1, 2, 3, 4, 5, 6, 7, 8), + 16 => choose!(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16), + 32 => choose!( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ), + _ => unreachable!("list-size limits must be 4, 8, 16, or 32"), + } +} + +#[inline] +fn check_values( + in_list_values: &[C], + input_values: &[C], +) -> BooleanBuffer +where + C: Copy + PartialEq, +{ + let in_list_values: &[C; N] = in_list_values + .try_into() + .expect("comparison length matches IN-list values"); + + BooleanBuffer::collect_bool(input_values.len(), |i| { + // SAFETY: `collect_bool` invokes this closure for indices in + // `0..input_values.len()`. + let input_value = unsafe { *input_values.get_unchecked(i) }; + // `|` checks every list value; `||` would stop after the first match. + in_list_values + .iter() + .fold(false, |acc, &value| acc | (value == input_value)) + }) +} + +fn branchless_values(array: &PrimitiveArray) -> ScalarBuffer> +where + T: BranchlessFilterType, +{ + let data = array.to_data(); + ScalarBuffer::>::new( + data.buffers()[0].clone(), + data.offset(), + data.len(), + ) +} + /// Wrapper for f32 that implements Hash and Eq using bit comparison. /// This treats NaN values as equal to each other when they have the same bit pattern. #[derive(Clone, Copy)] @@ -430,7 +701,9 @@ mod tests { use std::sync::Arc; use arrow::array::{ - DictionaryArray, Float16Array, Int8Array, Int16Array, UInt8Array, UInt16Array, + Decimal128Array, DictionaryArray, Float16Array, Float32Array, Float64Array, + Int8Array, Int16Array, IntervalMonthDayNanoArray, TimestampMillisecondArray, + TimestampNanosecondArray, UInt8Array, UInt16Array, }; use half::f16; @@ -584,4 +857,206 @@ mod tests { Ok(()) } + + #[test] + fn branchless_filter_u8_handles_nulls() -> Result<()> { + let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![Some(1), None, Some(3)])); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = UInt8Array::from(vec![Some(1), Some(2), None, Some(3)]); + + assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?; + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), None, None, Some(false)]) + ); + + Ok(()) + } + + #[test] + fn branchless_filter_all_null_list_preserves_sql_null_semantics() -> Result<()> { + let haystack: ArrayRef = Arc::new(UInt8Array::from(vec![None, None])); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = UInt8Array::from(vec![Some(1), None]); + let expected = BooleanArray::from(vec![None, None]); + + assert_eq!(filter.contains(&needles, false)?, expected); + assert_eq!(filter.contains(&needles, true)?, expected); + + Ok(()) + } + + #[test] + fn branchless_filter_i8_handles_signed_boundaries_and_slices() -> Result<()> { + let haystack: ArrayRef = Arc::new( + Int8Array::from(vec![Some(99), Some(i8::MIN), None, Some(-1), Some(42)]) + .slice(1, 3), + ); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = + Int8Array::from(vec![Some(7), Some(i8::MIN), Some(-1), None]).slice(1, 3); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(true), Some(true), None]) + ); + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), Some(false), None]) + ); + + let wrong_type = UInt8Array::from(vec![Some(128), Some(u8::MAX)]); + let err = filter.contains(&wrong_type, false).unwrap_err().to_string(); + assert!(err.contains("expected Int8 array, got UInt8"), "{err}"); + + Ok(()) + } + + #[test] + fn branchless_filter_f16_handles_bit_patterns_and_slices() -> Result<()> { + let nan_a = f16::from_bits(0x7e01); + let nan_b = f16::from_bits(0x7e02); + let haystack: ArrayRef = Arc::new( + Float16Array::from(vec![ + Some(f16::from_f32(9.0)), + Some(f16::from_f32(-0.0)), + Some(nan_a), + None, + ]) + .slice(1, 3), + ); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = Float16Array::from(vec![ + Some(f16::from_f32(0.0)), + Some(f16::from_f32(-0.0)), + Some(nan_a), + Some(nan_b), + None, + ]); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![None, Some(true), Some(true), None, None]) + ); + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![None, Some(false), Some(false), None, None]) + ); + + let wrong_type = UInt16Array::from(vec![Some(0x8000), Some(0x7e01)]); + let err = filter.contains(&wrong_type, false).unwrap_err().to_string(); + assert!(err.contains("expected Float16 array, got UInt16"), "{err}"); + + Ok(()) + } + + #[test] + fn branchless_filter_floats_use_bit_equality() -> Result<()> { + let nan_a = f32::from_bits(0x7fc0_0001); + let nan_b = f32::from_bits(0x7fc0_0002); + let haystack: ArrayRef = + Arc::new(Float32Array::from(vec![Some(-0.0), Some(nan_a)])); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = + Float32Array::from(vec![Some(0.0), Some(-0.0), Some(nan_a), Some(nan_b)]); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(false), Some(true), Some(true), Some(false)]) + ); + + let nan_a = f64::from_bits(0x7ff8_0000_0000_0001); + let nan_b = f64::from_bits(0x7ff8_0000_0000_0002); + let haystack: ArrayRef = + Arc::new(Float64Array::from(vec![Some(-0.0), Some(nan_a)])); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = + Float64Array::from(vec![Some(0.0), Some(-0.0), Some(nan_a), Some(nan_b)]); + + assert_eq!( + filter.contains(&needles, false)?, + BooleanArray::from(vec![Some(false), Some(true), Some(true), Some(false)]) + ); + + Ok(()) + } + + #[test] + fn branchless_filter_timestamp_uses_physical_compatibility() -> Result<()> { + let haystack: ArrayRef = Arc::new( + TimestampNanosecondArray::from(vec![Some(1), Some(3)]).with_timezone("UTC"), + ); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = TimestampNanosecondArray::from(vec![Some(1), Some(2), None]) + .with_timezone("UTC"); + + assert_contains(&filter, &needles, vec![Some(true), Some(false), None])?; + + let different_timezone = TimestampNanosecondArray::from(vec![Some(1), Some(2)]) + .with_timezone("Europe/Paris"); + assert_contains(&filter, &different_timezone, vec![Some(true), Some(false)])?; + + let different_unit = TimestampMillisecondArray::from(vec![Some(1)]); + let err = filter + .contains(&different_unit, false) + .unwrap_err() + .to_string(); + assert!(err.contains("Timestamp(ns"), "{err}"); + assert!(err.contains("Timestamp(ms"), "{err}"); + + Ok(()) + } + + #[test] + fn branchless_filter_decimal128_handles_precision_scale_and_nulls() -> Result<()> { + let haystack: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(12345), None, Some(-700), Some(42)]) + .with_precision_and_scale(10, 2)?, + ); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = + Decimal128Array::from(vec![Some(12345), Some(999), None, Some(-700)]) + .with_precision_and_scale(10, 2)?; + + assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?; + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), None, None, Some(false)]) + ); + + let compatible_metadata = + Decimal128Array::from(vec![Some(12345)]).with_precision_and_scale(11, 3)?; + assert_contains(&filter, &compatible_metadata, vec![Some(true)])?; + + Ok(()) + } + + #[test] + fn branchless_filter_interval_month_day_nano_handles_nulls() -> Result<()> { + let one_month = IntervalMonthDayNanoType::make_value(1, 0, 0); + let two_days = IntervalMonthDayNanoType::make_value(0, 2, 0); + let three_nanos = IntervalMonthDayNanoType::make_value(0, 0, 3); + let absent = IntervalMonthDayNanoType::make_value(4, 5, 6); + let haystack: ArrayRef = Arc::new(IntervalMonthDayNanoArray::from(vec![ + Some(one_month), + None, + Some(two_days), + Some(three_nanos), + ])); + let filter = BranchlessFilter::::try_new(&haystack)?; + let needles = IntervalMonthDayNanoArray::from(vec![ + Some(one_month), + Some(absent), + None, + Some(three_nanos), + ]); + + assert_contains(&filter, &needles, vec![Some(true), None, None, Some(true)])?; + assert_eq!( + filter.contains(&needles, true)?, + BooleanArray::from(vec![Some(false), None, None, Some(false)]) + ); + + Ok(()) + } } diff --git a/datafusion/physical-expr/src/expressions/in_list/strategy.rs b/datafusion/physical-expr/src/expressions/in_list/strategy.rs index 9db90ea4faf13..be4dce8dfdcad 100644 --- a/datafusion/physical-expr/src/expressions/in_list/strategy.rs +++ b/datafusion/physical-expr/src/expressions/in_list/strategy.rs @@ -20,7 +20,13 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute::cast; use arrow::datatypes::{ - DataType, Float16Type, Int8Type, Int16Type, UInt8Type, UInt16Type, + DataType, Date32Type, Date64Type, Decimal128Type, DurationMicrosecondType, + DurationMillisecondType, DurationNanosecondType, DurationSecondType, Float16Type, + Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, + IntervalMonthDayNanoType, IntervalUnit, Time32MillisecondType, Time32SecondType, + Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, + UInt16Type, UInt32Type, UInt64Type, }; use datafusion_common::Result; @@ -28,24 +34,88 @@ use super::array_static_filter::ArrayStaticFilter; use super::primitive_filter::*; use super::static_filter::StaticFilter; -pub(super) fn instantiate_static_filter( - in_array: ArrayRef, -) -> Result> { +type StaticFilterRef = Arc; + +pub(super) fn instantiate_static_filter(in_array: ArrayRef) -> Result { + let in_array = flatten_dictionary_haystack(in_array)?; + + if let Some(filter) = instantiate_branchless_filter(&in_array)? { + return Ok(filter); + } + + instantiate_standard_filter(in_array) +} + +fn flatten_dictionary_haystack(in_array: ArrayRef) -> Result { // Flatten dictionary-encoded haystacks to their value type so that // specialized filters (e.g. Int32StaticFilter) are used instead of // falling through to the generic ArrayStaticFilter. - let in_array = match in_array.data_type() { - DataType::Dictionary(_, value_type) => cast(&in_array, value_type.as_ref())?, - _ => in_array, - }; match in_array.data_type() { - DataType::Int8 => Ok(Arc::new(BitmapFilter::::try_new(&in_array)?)), - DataType::UInt8 => Ok(Arc::new(BitmapFilter::::try_new(&in_array)?)), - DataType::Int16 => Ok(Arc::new(BitmapFilter::::try_new(&in_array)?)), - DataType::UInt16 => Ok(Arc::new(BitmapFilter::::try_new(&in_array)?)), - DataType::Float16 => { - Ok(Arc::new(BitmapFilter::::try_new(&in_array)?)) + DataType::Dictionary(_, value_type) => Ok(cast(&in_array, value_type.as_ref())?), + _ => Ok(in_array), + } +} + +fn instantiate_branchless_filter(in_array: &ArrayRef) -> Result> { + let non_null_count = in_array.len() - in_array.null_count(); + + macro_rules! filter { + ($arrow_type:ty) => { + branchless_filter::<$arrow_type>(in_array, non_null_count) + }; + } + + match in_array.data_type() { + DataType::Int8 => filter!(Int8Type), + DataType::UInt8 => filter!(UInt8Type), + DataType::Int16 => filter!(Int16Type), + DataType::UInt16 => filter!(UInt16Type), + DataType::Float16 => filter!(Float16Type), + DataType::Int32 => filter!(Int32Type), + DataType::UInt32 => filter!(UInt32Type), + DataType::Float32 => filter!(Float32Type), + DataType::Date32 => filter!(Date32Type), + DataType::Time32(unit) => match unit { + TimeUnit::Second => filter!(Time32SecondType), + TimeUnit::Millisecond => filter!(Time32MillisecondType), + _ => Ok(None), + }, + DataType::Int64 => filter!(Int64Type), + DataType::UInt64 => filter!(UInt64Type), + DataType::Float64 => filter!(Float64Type), + DataType::Date64 => filter!(Date64Type), + DataType::Time64(unit) => match unit { + TimeUnit::Microsecond => filter!(Time64MicrosecondType), + TimeUnit::Nanosecond => filter!(Time64NanosecondType), + _ => Ok(None), + }, + DataType::Timestamp(unit, _) => match unit { + TimeUnit::Second => filter!(TimestampSecondType), + TimeUnit::Millisecond => filter!(TimestampMillisecondType), + TimeUnit::Microsecond => filter!(TimestampMicrosecondType), + TimeUnit::Nanosecond => filter!(TimestampNanosecondType), + }, + DataType::Duration(unit) => match unit { + TimeUnit::Second => filter!(DurationSecondType), + TimeUnit::Millisecond => filter!(DurationMillisecondType), + TimeUnit::Microsecond => filter!(DurationMicrosecondType), + TimeUnit::Nanosecond => filter!(DurationNanosecondType), + }, + DataType::Decimal128(_, _) => filter!(Decimal128Type), + DataType::Interval(IntervalUnit::MonthDayNano) => { + filter!(IntervalMonthDayNanoType) } + _ => Ok(None), + } +} + +fn instantiate_standard_filter(in_array: ArrayRef) -> Result { + match in_array.data_type() { + DataType::Int8 => bitmap_filter::(&in_array), + DataType::UInt8 => bitmap_filter::(&in_array), + DataType::Int16 => bitmap_filter::(&in_array), + DataType::UInt16 => bitmap_filter::(&in_array), + DataType::Float16 => bitmap_filter::(&in_array), DataType::Int32 => Ok(Arc::new(Int32StaticFilter::try_new(&in_array)?)), DataType::Int64 => Ok(Arc::new(Int64StaticFilter::try_new(&in_array)?)), DataType::UInt32 => Ok(Arc::new(UInt32StaticFilter::try_new(&in_array)?)), @@ -54,8 +124,71 @@ pub(super) fn instantiate_static_filter( DataType::Float32 => Ok(Arc::new(Float32StaticFilter::try_new(&in_array)?)), DataType::Float64 => Ok(Arc::new(Float64StaticFilter::try_new(&in_array)?)), _ => { - /* fall through to generic implementation for unsupported types (Struct, etc.) */ + // Fall through to generic implementation for unsupported types + // (Struct, etc.). Ok(Arc::new(ArrayStaticFilter::try_new(in_array)?)) } } } + +fn bitmap_filter(in_array: &ArrayRef) -> Result +where + T: BitmapFilterType, +{ + Ok(Arc::new(BitmapFilter::::try_new(in_array)?)) +} + +fn branchless_filter( + in_array: &ArrayRef, + non_null_count: usize, +) -> Result> +where + T: BranchlessFilterType, + BranchlessNative: Copy + PartialEq + Send + Sync, +{ + // Larger lists use the standard filter. `try_new` checks the limit again. + if non_null_count > T::MAX_LIST_LEN { + return Ok(None); + } + + Ok(Some(Arc::new(BranchlessFilter::::try_new(in_array)?))) +} + +#[cfg(test)] +mod tests { + use arrow::array::UInt32Array; + use arrow::datatypes::UInt32Type; + + use super::super::primitive_filter::BranchlessFilterType; + use super::*; + + fn uint32_array(values: Vec>) -> ArrayRef { + Arc::new(UInt32Array::from(values)) + } + + #[test] + fn branchless_routing_respects_max_list_len() -> Result<()> { + let max_len = ::MAX_LIST_LEN; + + let values = (0..max_len) + .map(|value| Some(value as u32)) + .collect::>(); + assert!(instantiate_branchless_filter(&uint32_array(values))?.is_some()); + + let values = (0..=max_len) + .map(|value| Some(value as u32)) + .collect::>(); + assert!(instantiate_branchless_filter(&uint32_array(values))?.is_none()); + + Ok(()) + } + + #[test] + fn branchless_routing_handles_zero_non_null_values() -> Result<()> { + let array = uint32_array(vec![None; 3]); + + assert!(instantiate_branchless_filter(&array)?.is_some()); + + Ok(()) + } +} From e660118996630bf7173b31a67e97dee97dac1fe4 Mon Sep 17 00:00:00 2001 From: Karpagam Balasubramaniam Date: Sat, 25 Jul 2026 20:26:54 -0700 Subject: [PATCH 029/109] feat: add Spark-compatible hypot function (#23774) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23770 - Part of #15914 ## Rationale for this change Spark provides [`hypot(expr1, expr2)`](https://spark.apache.org/docs/latest/api/sql/#hypot), which returns `sqrt(expr1^2 + expr2^2)` computed without intermediate overflow or underflow. It was not yet implemented in `datafusion-spark` — only an auto-generated test stub existed at `spark/math/hypot.slt` with its query commented out. ## What changes are included in this PR? - Add `SparkHypot` (implementing `ScalarUDFImpl`) in `datafusion/spark/src/function/math/hypot.rs`, backed by Rust's `f64::hypot` — the same overflow-safe algorithm as Java/Spark's `Math.hypot`. - Register it in `datafusion/spark/src/function/math/mod.rs`. - Enable the `hypot.slt` sqllogictest. The signature is `exact(Float64, Float64) -> Float64`, following the `datafusion-spark` convention of only accepting types Spark supports. Computation uses the Arrow `binary` kernel so NULL in either argument propagates to a NULL result, matching Spark. ## Are these changes tested? Yes — `datafusion/sqllogictest/test_files/spark/math/hypot.slt` covers: - scalar Pythagorean triples (`hypot(3, 4)` → 5, `hypot(5, 12)` → 13), - double inputs, - NULL propagation when either argument is NULL, - the array path (including a NULL row), - overflow-safety: `hypot(3e200, 4e200)` stays finite, whereas a naive `sqrt(a^2 + b^2)` would overflow to `Infinity`. ## Are there any user-facing changes? Yes — adds the Spark-compatible `hypot` scalar function to `datafusion-spark`. No breaking changes to public APIs. --- datafusion/spark/src/function/math/hypot.rs | 84 +++++++++++++ datafusion/spark/src/function/math/mod.rs | 4 + .../test_files/spark/math/hypot.slt | 116 +++++++++++++++++- 3 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 datafusion/spark/src/function/math/hypot.rs diff --git a/datafusion/spark/src/function/math/hypot.rs b/datafusion/spark/src/function/math/hypot.rs new file mode 100644 index 0000000000000..a1e30a7e4abe2 --- /dev/null +++ b/datafusion/spark/src/function/math/hypot.rs @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, AsArray, Float64Array}; +use arrow::compute::kernels::arity::binary; +use arrow::datatypes::{DataType, Float64Type}; +use datafusion_common::Result; +use datafusion_common::utils::take_function_args; +use datafusion_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_functions::utils::make_scalar_function; + +/// Spark-compatible `hypot` function. +/// +/// +/// +/// Returns `sqrt(expr1^2 + expr2^2)` computed without intermediate overflow or +/// underflow, matching Spark's use of `java.lang.Math.hypot`. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkHypot { + signature: Signature, +} + +impl Default for SparkHypot { + fn default() -> Self { + Self::new() + } +} + +impl SparkHypot { + pub fn new() -> Self { + Self { + // Spark only defines hypot over doubles + signature: Signature::exact( + vec![DataType::Float64, DataType::Float64], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for SparkHypot { + fn name(&self) -> &str { + "hypot" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + make_scalar_function(spark_hypot, vec![])(&args.args) + } +} + +fn spark_hypot(args: &[ArrayRef]) -> Result { + let [x, y] = take_function_args("hypot", args)?; + + let x = x.as_primitive::(); + let y = y.as_primitive::(); + let result: Float64Array = binary(x, y, |a, b| a.hypot(b))?; + Ok(Arc::new(result)) +} diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs index 0079ef0fc97cd..fb57b536f26ec 100644 --- a/datafusion/spark/src/function/math/mod.rs +++ b/datafusion/spark/src/function/math/mod.rs @@ -22,6 +22,7 @@ pub mod expm1; pub mod factorial; pub mod floor; pub mod hex; +pub mod hypot; pub mod modulus; pub mod negative; pub mod pow; @@ -41,6 +42,7 @@ make_udf_function!(expm1::SparkExpm1, expm1); make_udf_function!(factorial::SparkFactorial, factorial); make_udf_function!(floor::SparkFloor, floor); make_udf_function!(hex::SparkHex, hex); +make_udf_function!(hypot::SparkHypot, hypot); make_udf_function!(modulus::SparkMod, modulus); make_udf_function!(modulus::SparkPmod, pmod); make_udf_function!(pow::SparkPow, pow); @@ -66,6 +68,7 @@ pub mod expr_fn { )); export_functions!((floor, "Returns floor of expr.", arg1)); export_functions!((hex, "Computes hex value of the given column.", arg1)); + export_functions!((hypot, "Returns sqrt(a^2 + b^2) without intermediate overflow or underflow.", arg1 arg2)); export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2)); export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2)); export_functions!(( @@ -107,6 +110,7 @@ pub fn functions() -> Vec> { factorial(), floor(), hex(), + hypot(), modulus(), pmod(), pow(), diff --git a/datafusion/sqllogictest/test_files/spark/math/hypot.slt b/datafusion/sqllogictest/test_files/spark/math/hypot.slt index 1349be0a95ee7..564b34add8b9f 100644 --- a/datafusion/sqllogictest/test_files/spark/math/hypot.slt +++ b/datafusion/sqllogictest/test_files/spark/math/hypot.slt @@ -21,7 +21,115 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 -## Original Query: SELECT hypot(3, 4); -## PySpark 3.5.5 Result: {'HYPOT(3, 4)': 5.0, 'typeof(HYPOT(3, 4))': 'double', 'typeof(3)': 'int', 'typeof(4)': 'int'} -#query -#SELECT hypot(3::int, 4::int); +# Scalar: classic Pythagorean triples (3-4-5, 5-12-13) +query R +SELECT hypot(3, 4); +---- +5 + +query R +SELECT hypot(5, 12); +---- +13 + +# Double inputs +query R +SELECT hypot(3.0::double, 4.0::double); +---- +5 + +# NULL if either argument is NULL +query R +SELECT hypot(NULL::double, 4.0::double); +---- +NULL + +query R +SELECT hypot(3.0::double, NULL::double); +---- +NULL + +# Array path, including a NULL row +query R +SELECT hypot(a, b) FROM (VALUES (3.0::double, 4.0::double), (6.0::double, 8.0::double), (NULL::double, 1.0::double)) AS t(a, b); +---- +5 +10 +NULL + +# Overflow-safe: naive sqrt(a*a + b*b) overflows to Infinity here; hypot stays finite (matches Spark's Math.hypot) +query B +SELECT hypot(3e200::double, 4e200::double) < 'Infinity'::double; +---- +true + +# any infinite input yields +Infinity, even when the other is NaN +query R +SELECT hypot('Infinity'::double, 4.0::double); +---- +Infinity + +query R +SELECT hypot(4.0::double, '-Infinity'::double); +---- +Infinity + +query R +SELECT hypot('Infinity'::double, 'NaN'::double); +---- +Infinity + +# NaN propagates when neither input is infinite +query R +SELECT hypot('NaN'::double, 4.0::double); +---- +NaN + +# signed zeros +query RRR +SELECT hypot(0.0::double, 0.0::double), hypot(-0.0::double, 0.0::double), hypot(3.0::double, -0.0::double); +---- +0 0 3 + +# NULL propagates even when the other input is Infinity +query R +SELECT hypot(NULL::double, 'Infinity'::double); +---- +NULL + +# negative inputs yield the positive magnitude +query RR +SELECT hypot(-3.0::double, -4.0::double), hypot(-3.0::double, 4.0::double); +---- +5 5 + +# Underflow-safe: naive sqrt(a*a + b*b) underflows to 0 for tiny inputs; hypot stays nonzero (matches Spark's Math.hypot) +query B +SELECT hypot(3e-200::double, 4e-200::double) > 0; +---- +true + +# Array path with special values (normal, +Infinity, NaN, NULL) +query R +SELECT hypot(a, b) FROM (VALUES + (3.0::double, 4.0::double), + ('Infinity'::double, 1.0::double), + ('NaN'::double, 1.0::double), + (NULL::double, 1.0::double)) AS t(a, b); +---- +5 +Infinity +NaN +NULL + +# both inputs NaN -> NaN +query R +SELECT hypot('NaN'::double, 'NaN'::double); +---- +NaN + +# both inputs infinite -> +Infinity +query R +SELECT hypot('Infinity'::double, '-Infinity'::double); +---- +Infinity \ No newline at end of file From 88365ddd62b17c1eabd20ed0b064f626f9e77686 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Sat, 25 Jul 2026 20:27:25 -0700 Subject: [PATCH 030/109] chore: adjust `size` accounting for `min_max` (#23899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #23393 . ## Rationale for this change `SlidingMinAccumulator::size` and `SlidingMaxAccumulator::size` only reported the stack size of their `ScalarValue` field plus its heap payload, ignoring the memory held by the underlying `MovingMin` / `MovingMax` sliding-window buffers. For windowed `MIN`/`MAX` over string or list data, the two per-element stacks can hold megabytes of `ScalarValue` payload that the memory pool was never told about, understating accumulator memory usage. ## What changes are included in this PR? - Add a private `heap_size(elem_heap)` method to `MovingMin` and `MovingMax` that reports the two stack buffers' capacity in bytes plus each stored element's heap payload. - Factor the shared implementation into a `moving_stacks_heap_size` free helper so the two types stay in sync. - Include the buffer bytes in `SlidingMinAccumulator::size` and `SlidingMaxAccumulator::size` via the new method. ## Are these changes tested? Yes. Two new unit tests in `datafusion/functions-aggregate/src/min_max.rs`: - `moving_min_max_heap_size_i32` — fixed-width `T`, verifies buffer-only accounting with and without pushed elements. - `moving_min_max_heap_size_counts_elems` — `T = String`, verifies each of the two slots in a `(T, T)` pair contributes independently to the heap payload (mirroring the two independent `Clone`s made by `push`). --- datafusion/functions-aggregate/src/min_max.rs | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs index f4eaaab853464..1a3179170c7b6 100644 --- a/datafusion/functions-aggregate/src/min_max.rs +++ b/datafusion/functions-aggregate/src/min_max.rs @@ -52,7 +52,7 @@ use datafusion_expr::{ use datafusion_expr::{GroupsAccumulator, StatisticsArgs}; use datafusion_macros::user_doc; use half::f16; -use std::mem::size_of_val; +use std::mem::{size_of, size_of_val}; use std::ops::Deref; fn get_min_max_result_type(input_types: &[DataType]) -> Result> { @@ -433,7 +433,9 @@ impl Accumulator for SlidingMaxAccumulator { } fn size(&self) -> usize { - size_of_val(self) - size_of_val(&self.max) + self.max.size() + size_of_val(self) - size_of_val(&self.max) + + self.max.size() + + self.moving_max.heap_size(|sv| sv.size() - size_of_val(sv)) } } @@ -721,7 +723,9 @@ impl Accumulator for SlidingMinAccumulator { } fn size(&self) -> usize { - size_of_val(self) - size_of_val(&self.min) + self.min.size() + size_of_val(self) - size_of_val(&self.min) + + self.min.size() + + self.moving_min.heap_size(|sv| sv.size() - size_of_val(sv)) } } @@ -857,6 +861,30 @@ impl MovingMin { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Heap bytes owned by the two stack buffers plus each stored `T`'s + /// heap payload as reported by `elem_heap`. Excludes `size_of::()`. + #[inline] + fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize { + moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap) + } +} + +/// Shared implementation for [`MovingMin::heap_size`] and +/// [`MovingMax::heap_size`]. Both share the same two-stack layout. +#[inline] +fn moving_stacks_heap_size( + push_stack: &Vec<(T, T)>, + pop_stack: &Vec<(T, T)>, + elem_heap: impl Fn(&T) -> usize, +) -> usize { + let buffers = (push_stack.capacity() + pop_stack.capacity()) * size_of::<(T, T)>(); + let elems: usize = push_stack + .iter() + .chain(pop_stack.iter()) + .map(|(a, b)| elem_heap(a) + elem_heap(b)) + .sum(); + buffers + elems } /// Keep track of the maximum value in a sliding window. @@ -975,6 +1003,13 @@ impl MovingMax { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Heap bytes owned by the two stack buffers plus each stored `T`'s + /// heap payload as reported by `elem_heap`. Excludes `size_of::()`. + #[inline] + fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize { + moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap) + } } make_udaf_expr_and_func!( @@ -1213,6 +1248,48 @@ mod tests { Ok(()) } + #[test] + fn moving_min_max_heap_size_i32() { + // Fixed-width `T` has no per-element heap payload, so `heap_size` + // reports exactly the two stack buffers' capacity in bytes. + let mut moving_min = MovingMin::::with_capacity(4); + let mut moving_max = MovingMax::::with_capacity(4); + let elem = |_: &i32| 0; + + // Both stacks are `with_capacity(4)`, so total slots = 8. + let buffer_only = 2 * 4 * size_of::<(i32, i32)>(); + assert_eq!(moving_min.heap_size(elem), buffer_only); + assert_eq!(moving_max.heap_size(elem), buffer_only); + + for i in 0..3 { + moving_min.push(i); + moving_max.push(i); + } + // Elements sit inside the pre-allocated buffers, so still buffer-only. + assert_eq!(moving_min.heap_size(elem), buffer_only); + assert_eq!(moving_max.heap_size(elem), buffer_only); + } + + #[test] + fn moving_min_max_heap_size_counts_elems() { + // Each buffered slot is a `(T, T)` pair, so a stored element is + // visited twice by `heap_size` — mirrors two independent `Clone`s. + let mut moving_min = MovingMin::::with_capacity(2); + let mut moving_max = MovingMax::::with_capacity(2); + let elem = |s: &String| s.capacity(); + + moving_min.push("abcdef".to_string()); + moving_max.push("abcdef".to_string()); + + // Both `push_stack` and `pop_stack` allocate `capacity` slots. + let buffers = 2 * 2 * size_of::<(String, String)>(); + // 2 slots per stored element (value + running extremum) times the + // per-element heap payload from `elem`. + let elems = 2 * 6; + assert_eq!(moving_min.heap_size(elem), buffers + elems); + assert_eq!(moving_max.heap_size(elem), buffers + elems); + } + #[test] fn test_min_max_coerce_types() { // the coerced types is same with input types From 551c592ca6b6e50ff41b62ca8bff282a28be01e4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 26 Jul 2026 13:45:55 -0400 Subject: [PATCH 031/109] Add ObjectStore-backed TempFileFactor / spill example (#23170) ## Which issue does this PR close? - Related to https://github.com/apache/datafusion/pull/21882#issuecomment-4793942224. ## Rationale for this change PR #21882 adds custom spill file support. Having an example showing how a downstream application can use this new API will help make sure the API is good enough for our needs ## What changes are included in this PR? This PR adds a small example showing how users can back spill files with an `ObjectStore`, using a local object store for a runnable example while keeping the implementation applicable to remote stores. ## Are these changes tested? Yes by CI ## Are there any user-facing changes? Yes. This adds a new example for configuring ObjectStore-backed spill files. --- datafusion-examples/Cargo.toml | 2 +- datafusion-examples/README.md | 1 + datafusion-examples/examples/data_io/main.rs | 10 +- .../examples/data_io/object_store_spill.rs | 273 ++++++++++++++++++ datafusion/execution/src/disk_manager.rs | 20 ++ 5 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 datafusion-examples/examples/data_io/object_store_spill.rs diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 5f66412e7debd..e59f7eb9483c3 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -60,7 +60,7 @@ futures = { workspace = true } insta = { workspace = true } log = { workspace = true } mimalloc = { version = "0.1", default-features = false } -object_store = { workspace = true, features = ["aws", "http"] } +object_store = { workspace = true, features = ["aws", "fs", "http"] } prost = { workspace = true } rand = { workspace = true } serde = { version = "1", features = ["derive"] } diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 4746ac9114733..86cfffe1a80e8 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -93,6 +93,7 @@ cargo run --example dataframe -- dataframe | catalog | [`data_io/catalog.rs`](examples/data_io/catalog.rs) | Register tables into a custom catalog | | in_memory_object_store | [`data_io/in_memory_object_store.rs`](examples/data_io/in_memory_object_store.rs) | Read CSV from an in-memory object store (pattern applies to JSON/Parquet) | | json_shredding | [`data_io/json_shredding.rs`](examples/data_io/json_shredding.rs) | Implement filter rewriting for JSON shredding | +| object_store_spill | [`data_io/object_store_spill.rs`](examples/data_io/object_store_spill.rs) | Use ObjectStore-backed spill files | | parquet_adv_idx | [`data_io/parquet_advanced_index.rs`](examples/data_io/parquet_advanced_index.rs) | Create a secondary index across multiple parquet files | | parquet_emb_idx | [`data_io/parquet_embedded_index.rs`](examples/data_io/parquet_embedded_index.rs) | Store a custom index inside Parquet files | | parquet_enc | [`data_io/parquet_encrypted.rs`](examples/data_io/parquet_encrypted.rs) | Read & write encrypted Parquet files | diff --git a/datafusion-examples/examples/data_io/main.rs b/datafusion-examples/examples/data_io/main.rs index 0b1c435b932e7..041308463cda9 100644 --- a/datafusion-examples/examples/data_io/main.rs +++ b/datafusion-examples/examples/data_io/main.rs @@ -21,7 +21,7 @@ //! //! ## Usage //! ```bash -//! cargo run --example data_io -- [all|catalog|in_memory_object_store|json_shredding|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog] +//! cargo run --example data_io -- [all|catalog|in_memory_object_store|json_shredding|object_store_spill|parquet_adv_idx|parquet_emb_idx|parquet_enc_with_kms|parquet_enc|parquet_exec_visitor|parquet_idx|query_http_csv|remote_catalog] //! ``` //! //! Each subcommand runs a corresponding example: @@ -36,6 +36,9 @@ //! - `json_shredding` //! (file: json_shredding.rs, desc: Implement filter rewriting for JSON shredding) //! +//! - `object_store_spill` +//! (file: object_store_spill.rs, desc: Use ObjectStore-backed spill files) +//! //! - `parquet_adv_idx` //! (file: parquet_advanced_index.rs, desc: Create a secondary index across multiple parquet files) //! @@ -66,6 +69,7 @@ mod catalog; mod in_memory_object_store; mod json_shredding; +mod object_store_spill; mod parquet_advanced_index; mod parquet_embedded_index; mod parquet_encrypted; @@ -87,6 +91,7 @@ enum ExampleKind { Catalog, InMemoryObjectStore, JsonShredding, + ObjectStoreSpill, ParquetAdvIdx, ParquetEmbIdx, ParquetEnc, @@ -118,6 +123,9 @@ impl ExampleKind { in_memory_object_store::in_memory_object_store().await? } ExampleKind::JsonShredding => json_shredding::json_shredding().await?, + ExampleKind::ObjectStoreSpill => { + object_store_spill::object_store_spill().await? + } ExampleKind::ParquetAdvIdx => { parquet_advanced_index::parquet_advanced_index().await? } diff --git a/datafusion-examples/examples/data_io/object_store_spill.rs b/datafusion-examples/examples/data_io/object_store_spill.rs new file mode 100644 index 0000000000000..d7d5392f66953 --- /dev/null +++ b/datafusion-examples/examples/data_io/object_store_spill.rs @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! See `main.rs` for how to run it. +//! +//! [`object_store_spill`] demonstrates how to use the [`TempFileFactory`] API to configure +//! DataFusion to spill intermediate results to remote storage when it exceeds +//! the configured memory limits. +//! +//! See [`datafusion::execution::memory_pool`] for more information on how +//! DataFusion decides when operators should spill, and [`SpillFile`] for the +//! spill file abstraction this example implements. +use std::future::Future; +use std::io::Write; +use std::path::Path as StdPath; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use bytes::Bytes; +use datafusion::common::Result; +use datafusion::execution::disk_manager::DiskManagerBuilder; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_common::exec_err; +use futures::{Stream, StreamExt, TryStreamExt, stream}; +use object_store::local::LocalFileSystem; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; +use tempfile::tempdir; + +/// Demonstrates configuring DataFusion with spill files backed by an ObjectStore. +pub async fn object_store_spill() -> Result<()> { + // A real system would use S3, GCS, Azure, or some other ObjectStore for + // remote spills. This example uses a local-file-backed ObjectStore for + // simplicity. + let tmp_dir = tempdir()?; + let store: Arc = + Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?); + + // Create the custom TempFileFactory that creates spill files in the ObjectStore. + let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(store)); + let disk_manager_builder = + DiskManagerBuilder::default().with_temp_file_factory(temp_file_factory.clone()); + let runtime = RuntimeEnvBuilder::new() + .with_disk_manager_builder(disk_manager_builder) // use the factory + // and set a small memory limit so the example spills + .with_memory_limit(1024 * 1024, 1.0) + .build_arc()?; + + // Configure a SessionContext for running queries; use a single partition + // and no sort spill reservation to make the example deterministic and keep + // the spill behavior easy to observe. + let config = SessionConfig::new() + .with_sort_spill_reservation_bytes(0) + .with_sort_in_place_threshold_bytes(0) + .with_target_partitions(1); + let ctx = SessionContext::new_with_config_rt(config, Arc::clone(&runtime)); + + // Run an SQL query that sorts a "large" amount of data. Given the + // SessionContext's low memory limit, the sort will spill. + let row_count = 10_000_000; + let mut stream = ctx + .sql(&format!( + "SELECT * FROM generate_series(1, {row_count}) AS t(v) ORDER BY v DESC" + )) + .await? + .execute_stream() + .await?; + + // Drive the query to completion, and verify output + let mut output_rows = 0; + while let Some(batch) = stream.next().await { + output_rows += batch?.num_rows(); + } + + assert_eq!(output_rows, row_count as usize); + assert!( + temp_file_factory.created_files() > 0, + "expected the custom TempFileFactory to be used for spilling" + ); + + Ok(()) +} + +/// Creates spill files backed by an [`ObjectStore`]. +/// +/// DataFusion calls this factory whenever an operator needs a new temporary +/// file for spilling. A remote deployment would use the same pattern with an +/// S3, GCS, Azure, or other remote ObjectStore implementation. +struct ObjectStoreTempFileFactory { + /// ObjectStore used for spill file reads and writes. + store: Arc, + /// Monotonic counter used to create unique object paths. + counter: AtomicU64, + /// Counts how many spill files DataFusion requested from this factory. + created_files: AtomicU64, +} + +impl ObjectStoreTempFileFactory { + /// Create a new spill file factory that stores spill data in `store`. + fn new(store: Arc) -> Self { + Self { + store, + counter: AtomicU64::new(0), + created_files: AtomicU64::new(0), + } + } + + /// Return the number of spill files created through this factory. + fn created_files(&self) -> u64 { + self.created_files.load(Ordering::Relaxed) + } +} + +impl TempFileFactory for ObjectStoreTempFileFactory { + /// Create one logical spill file backed by an ObjectStore path. + fn create_temp_file(&self, description: &str) -> Result> { + let id = self.counter.fetch_add(1, Ordering::Relaxed); + self.created_files.fetch_add(1, Ordering::Relaxed); + + // Convert a query-provided spill description into an ObjectStore-safe path component. + // + // For example, `"Sort Spill: partition 0"` becomes `"Sort_Spill__partition_0"`. + let cleaned_description: String = description + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + let location = Path::from(format!("spill/{cleaned_description}-{id}.bin")); + + // Return a SpillFile implementation that reads and writes this ObjectStore path. + Ok(Arc::new(ObjectStoreSpillFile { + store: Arc::clone(&self.store), + location, + size: Arc::new(AtomicU64::new(0)), + })) + } +} + +/// Logical spill file stored at an ObjectStore path. +/// +/// DataFusion writes spill data by calling [`SpillFile::open_writer`] and reads +/// it back by calling [`SpillFile::read_stream`]. +struct ObjectStoreSpillFile { + /// ObjectStore containing the spill object. + store: Arc, + /// ObjectStore path for this spill object. + location: Path, + /// Last committed object size, updated when the writer finishes. + size: Arc, +} + +impl SpillFile for ObjectStoreSpillFile { + /// Return no local filesystem path because the spill file is accessed through ObjectStore. + fn path(&self) -> Option<&StdPath> { + None // Remote ObjectStores do not have a local OS path. + } + + /// Return the size of the uploaded object + fn size(&self) -> Option { + // Return the last committed size, which this example tracks after upload. + Some(self.size.load(Ordering::Relaxed)) + } + + /// Read the spill file contents as a byte stream. + fn read_stream(&self) -> Result> + Send>>> { + let store = Arc::clone(&self.store); + let location = self.location.clone(); + + // Use `stream::once` to defer the ObjectStore read until DataFusion + // polls the returned stream. + let result_stream = + async move { store.get(&location).await.map(|r| r.into_stream()) }; + let stream = stream::once(result_stream) + .try_flatten() + .map_err(Into::into); + + Ok(Box::pin(stream)) + } + + /// Open a synchronous writer for this spill file. + fn open_writer(&self) -> Result> { + // Create a writer that buffers bytes and uploads them on finish. + Ok(Box::new(ObjectStoreSpillWriter { + store: Arc::clone(&self.store), + location: self.location.clone(), + size: Arc::clone(&self.size), + buffer: Vec::new(), + })) + } +} + +/// Adapts DataFusion's [`SpillWriter`] API to ObjectStore. +/// +/// This simple example buffers bytes in memory and uploads them in +/// [`SpillWriter::finish`]. A production remote implementation should consider +/// multipart or streaming uploads. +struct ObjectStoreSpillWriter { + /// ObjectStore to read/write bytes to. + store: Arc, + /// ObjectStore path to upload to. + location: Path, + /// Shared size field on the corresponding [`ObjectStoreSpillFile`]. + size: Arc, + /// Buffered spill bytes waiting to be uploaded. + /// + /// This simple example buffers the spill and uploads it on finish. + /// Production remote stores should consider multipart or streaming uploads. + buffer: Vec, +} + +impl Write for ObjectStoreSpillWriter { + /// Append bytes to the in-memory buffer. + fn write(&mut self, buf: &[u8]) -> std::io::Result { + // Buffer bytes written through the synchronous Write API. + self.buffer.extend_from_slice(buf); + Ok(buf.len()) + } + + /// No-op because data is committed in [`SpillWriter::finish`]. + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl SpillWriter for ObjectStoreSpillWriter { + /// Upload buffered bytes to ObjectStore and mark the spill file complete. + fn finish(&mut self) -> Result<()> { + // Move the buffered bytes into the upload future. + let store = Arc::clone(&self.store); + let location = self.location.clone(); + let data = std::mem::take(&mut self.buffer); + let size = data.len() as u64; + + // This simple example buffers the spill and uploads it on finish. + // Production remote stores should consider multipart or streaming uploads. + block_on_object_store(async move { + store + .put(&location, PutPayload::from_bytes(data.into())) + .await?; + Ok(()) + })?; + + self.size.store(size, Ordering::Relaxed); + Ok(()) + } +} + +/// Run an async ObjectStore operation. +/// +/// Adding a native async API is tracked in +fn block_on_object_store(future: impl Future>) -> Result { + if let Ok(handle) = tokio::runtime::Handle::try_current() { + tokio::task::block_in_place(|| handle.block_on(future)) + } else { + exec_err!("No current Tokio runtime available") + } +} diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs index 8534c4f4ab75e..313379f01291f 100644 --- a/datafusion/execution/src/disk_manager.rs +++ b/datafusion/execution/src/disk_manager.rs @@ -74,6 +74,26 @@ impl DiskManagerBuilder { self } + /// Configure a custom factory for creating temporary spill files. + /// + /// This sets the disk manager mode to [`DiskManagerMode::Custom`], so + /// operators that spill during query execution create files through the + /// provided [`TempFileFactory`] instead of using local temporary files. + pub fn set_temp_file_factory(&mut self, temp_file_factory: Arc) { + self.mode = DiskManagerMode::Custom(temp_file_factory); + } + + /// Configure a custom factory for creating temporary spill files. + /// + /// See details on [`Self::set_temp_file_factory`]. + pub fn with_temp_file_factory( + mut self, + temp_file_factory: Arc, + ) -> Self { + self.set_temp_file_factory(temp_file_factory); + self + } + pub fn set_max_temp_directory_size(&mut self, value: u64) { self.max_temp_directory_size = value; } From bb670fbabec111cf74e3ae3ee78d0abec65d7569 Mon Sep 17 00:00:00 2001 From: Phoenix Date: Mon, 27 Jul 2026 02:16:04 +0800 Subject: [PATCH 032/109] refactor(proto): remove legacy scan field (#23445) ## Which issue does this PR close? - Closes #N/A. ## Rationale for this change Since DataFusion doesn't typically guartantee wire format compatibility, I remove the backward compatiblity shim that introduced in PR #23189 FYI: https://github.com/apache/datafusion/pull/23189#discussion_r3554855165 ## What changes are included in this PR? DataFusion does not guarantee serialized plans across versions. Keeping `partitioned_by_file_group` therefore leaves a dead schema field and decoder path after `output_partitioning` became the source of truth. Reserve the old field number and name to prevent future reuse. ## Are these changes tested? Yes ## Are there any user-facing changes? Yes Signed-off-by: Jiawei Zhao Co-authored-by: Andrew Lamb --- .../proto-models/proto/datafusion.proto | 4 +- .../proto-models/src/generated/pbjson.rs | 18 ------ .../proto-models/src/generated/prost.rs | 2 - .../proto/src/physical_plan/from_proto.rs | 18 +----- .../proto/src/physical_plan/to_proto.rs | 3 - .../tests/cases/roundtrip_physical_plan.rs | 63 +------------------ .../library-user-guide/upgrading/55.0.0.md | 8 +++ 7 files changed, 13 insertions(+), 103 deletions(-) diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 205cf89abed1b..16b1b1532f518 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -1231,7 +1231,9 @@ message FileScanExecConf { optional uint64 batch_size = 12; optional ProjectionExprs projection_exprs = 13; - optional bool partitioned_by_file_group = 14; + // Was optional bool partitioned_by_file_group = 14. + reserved 14; + reserved "partitioned_by_file_group"; optional Partitioning output_partitioning = 15; } diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index d23f8eee5fd2c..c5d7c003013a1 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -6999,9 +6999,6 @@ impl serde::Serialize for FileScanExecConf { if self.projection_exprs.is_some() { len += 1; } - if self.partitioned_by_file_group.is_some() { - len += 1; - } if self.output_partitioning.is_some() { len += 1; } @@ -7041,9 +7038,6 @@ impl serde::Serialize for FileScanExecConf { if let Some(v) = self.projection_exprs.as_ref() { struct_ser.serialize_field("projectionExprs", v)?; } - if let Some(v) = self.partitioned_by_file_group.as_ref() { - struct_ser.serialize_field("partitionedByFileGroup", v)?; - } if let Some(v) = self.output_partitioning.as_ref() { struct_ser.serialize_field("outputPartitioning", v)?; } @@ -7074,8 +7068,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { "batchSize", "projection_exprs", "projectionExprs", - "partitioned_by_file_group", - "partitionedByFileGroup", "output_partitioning", "outputPartitioning", ]; @@ -7093,7 +7085,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { Constraints, BatchSize, ProjectionExprs, - PartitionedByFileGroup, OutputPartitioning, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -7127,7 +7118,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { "constraints" => Ok(GeneratedField::Constraints), "batchSize" | "batch_size" => Ok(GeneratedField::BatchSize), "projectionExprs" | "projection_exprs" => Ok(GeneratedField::ProjectionExprs), - "partitionedByFileGroup" | "partitioned_by_file_group" => Ok(GeneratedField::PartitionedByFileGroup), "outputPartitioning" | "output_partitioning" => Ok(GeneratedField::OutputPartitioning), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } @@ -7159,7 +7149,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { let mut constraints__ = None; let mut batch_size__ = None; let mut projection_exprs__ = None; - let mut partitioned_by_file_group__ = None; let mut output_partitioning__ = None; while let Some(k) = map_.next_key()? { match k { @@ -7234,12 +7223,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { } projection_exprs__ = map_.next_value()?; } - GeneratedField::PartitionedByFileGroup => { - if partitioned_by_file_group__.is_some() { - return Err(serde::de::Error::duplicate_field("partitionedByFileGroup")); - } - partitioned_by_file_group__ = map_.next_value()?; - } GeneratedField::OutputPartitioning => { if output_partitioning__.is_some() { return Err(serde::de::Error::duplicate_field("outputPartitioning")); @@ -7260,7 +7243,6 @@ impl<'de> serde::Deserialize<'de> for FileScanExecConf { constraints: constraints__, batch_size: batch_size__, projection_exprs: projection_exprs__, - partitioned_by_file_group: partitioned_by_file_group__, output_partitioning: output_partitioning__, }) } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index 6baabbf37a41c..2300f7192fb97 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1875,8 +1875,6 @@ pub struct FileScanExecConf { pub batch_size: ::core::option::Option, #[prost(message, optional, tag = "13")] pub projection_exprs: ::core::option::Option, - #[prost(bool, optional, tag = "14")] - pub partitioned_by_file_group: ::core::option::Option, #[prost(message, optional, tag = "15")] pub output_partitioning: ::core::option::Option, } diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs index f1b324c79d451..d5a1e0efac6b6 100644 --- a/datafusion/proto/src/physical_plan/from_proto.rs +++ b/datafusion/proto/src/physical_plan/from_proto.rs @@ -29,9 +29,7 @@ use datafusion_common::{ }; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_groups::FileGroup; -use datafusion_datasource::file_scan_config::{ - FileScanConfig, FileScanConfigBuilder, output_partitioning_from_partition_fields, -}; +use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; use datafusion_datasource::file_sink_config::FileSinkConfig; use datafusion_datasource::{FileRange, ListingTableUrl, PartitionedFile, TableSchema}; use datafusion_datasource_csv::file_format::CsvSink; @@ -582,20 +580,6 @@ pub fn parse_protobuf_file_scan_config( &schema, proto_converter, )?; - let output_partitioning = match output_partitioning { - Some(output_partitioning) => Some(output_partitioning), - None if proto.partitioned_by_file_group.unwrap_or(false) => { - // Backward compatibility: older serialized plans used only - // `partitioned_by_file_group` to declare scan output partitioning. - let table_schema = parse_table_schema_from_proto(proto)?; - output_partitioning_from_partition_fields( - &schema, - table_schema.table_partition_cols(), - file_groups.len(), - ) - } - None => None, - }; // Parse projection expressions if present and apply to file source let file_source = if let Some(proto_projection_exprs) = &proto.projection_exprs { diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs index 515a53c08746f..e13923dbb9519 100644 --- a/datafusion/proto/src/physical_plan/to_proto.rs +++ b/datafusion/proto/src/physical_plan/to_proto.rs @@ -563,9 +563,6 @@ pub fn serialize_file_scan_config( constraints: Some(conf.constraints.clone().into()), batch_size: conf.batch_size.map(|s| s as u64), projection_exprs, - // Partition grouping is now encoded in `output_partitioning`; this legacy - // wire field is left unset (readers rely on `output_partitioning`). - partitioned_by_file_group: None, output_partitioning, }) } diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 3d13ffe16e8b9..864e6d68676ee 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -133,12 +133,7 @@ use datafusion_proto::bytes::{ physical_plan_from_bytes_with_proto_converter, physical_plan_to_bytes_with_proto_converter, }; -use datafusion_proto::physical_plan::from_proto::{ - parse_protobuf_file_scan_config, parse_table_schema_from_proto, -}; -use datafusion_proto::physical_plan::to_proto::{ - serialize_file_scan_config, serialize_physical_expr_with_converter, -}; +use datafusion_proto::physical_plan::to_proto::serialize_physical_expr_with_converter; use datafusion_proto::physical_plan::{ AsExecutionPlan, DeduplicatingProtoConverter, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext, @@ -4785,62 +4780,6 @@ fn roundtrip_parquet_exec_output_partitioning() -> Result<()> { Ok(()) } -#[test] -fn parse_legacy_partitioned_by_file_group_as_output_partitioning() -> Result<()> { - let file_schema = - Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)])); - let table_schema = TableSchema::builder(Arc::clone(&file_schema)) - .with_table_partition_cols(vec![Arc::new(Field::new( - "part", - DataType::Utf8, - false, - ))]) - .build(); - let file_source = Arc::new(ParquetSource::new(table_schema)); - let scan_config = - FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) - .with_file_groups(vec![ - FileGroup::new(vec![PartitionedFile::new( - "/path/to/file1.parquet".to_string(), - 1024, - )]), - FileGroup::new(vec![PartitionedFile::new( - "/path/to/file2.parquet".to_string(), - 1024, - )]), - ]) - .build(); - - let codec = DefaultPhysicalExtensionCodec {}; - let proto_converter = DefaultPhysicalProtoConverter {}; - let mut proto = serialize_file_scan_config(&scan_config, &codec, &proto_converter)?; - proto.partitioned_by_file_group = Some(true); - proto.output_partitioning = None; - - let ctx = SessionContext::new(); - let task_ctx = ctx.task_ctx(); - let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec); - let parsed = parse_protobuf_file_scan_config( - &proto, - &decode_ctx, - &proto_converter, - Arc::new(ParquetSource::new(parse_table_schema_from_proto(&proto)?)), - )?; - - match parsed.output_partitioning { - Some(Partitioning::Hash(exprs, partition_count)) => { - assert_eq!(partition_count, 2); - assert_eq!(exprs.len(), 1); - let column = exprs[0].downcast_ref::().unwrap(); - assert_eq!(column.name(), "part"); - assert_eq!(column.index(), 1); - } - other => panic!("Expected legacy hash output partitioning, got {other:?}"), - } - - Ok(()) -} - #[test] fn roundtrip_parquet_exec_range_output_partitioning() -> Result<()> { let file_schema = diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index 6ca4d71a7883f..df268792d8cc7 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -73,12 +73,17 @@ let df = df.fill_null(&ScalarValue::from(0), &[])?; `FileScanConfigBuilder::with_partitioned_by_file_group(...)` have been removed. Use `FileScanConfig::output_partitioning` and `FileScanConfigBuilder::with_output_partitioning(...)` instead. +The corresponding +`datafusion_proto::protobuf::FileScanExecConf::partitioned_by_file_group` +field has also been removed. **Who is affected:** - Users who accessed `FileScanConfig::partitioned_by_file_group` directly. - Users who called `FileScanConfigBuilder::with_partitioned_by_file_group(true)`. +- Users who constructed or accessed + `datafusion_proto::protobuf::FileScanExecConf::partitioned_by_file_group`. **Migration guide:** @@ -108,6 +113,9 @@ otherwise. If you construct the partitioning manually, pass `Some(Partitioning::Hash(partition_exprs, partition_count))` to `with_output_partitioning(...)`. +When constructing `FileScanExecConf`, omit `partitioned_by_file_group` and set +`output_partitioning` instead. + ### User `SpillFile` traits instead of [`RefCountedTempFile`] Spill file APIs now use the `datafusion_execution::SpillFile` trait instead of From 5975a2d57cdfd908bd7acd27e7f875877f354ca4 Mon Sep 17 00:00:00 2001 From: Moe Date: Sun, 26 Jul 2026 11:47:03 -0700 Subject: [PATCH 033/109] fix: skip dynamic filter pushdown for null-aware anti joins with a nullable build key (#23173) ## Which issue does this close? - Closes #23126. ## Rationale for this change `x NOT IN (subquery)` plans to a null-aware `LeftAnti` hash join (build = outer `x`, probe = subquery). Join dynamic filter pushdown pushes a bounds + membership filter, built from the build keys, onto the probe scan. That filter can prune every probe row. A null-aware `LeftAnti` reads an empty probe as a genuinely-empty subquery, so it emits build-side NULL rows that should drop: `NULL NOT IN (non-empty)` is UNKNOWN, not TRUE. The result is scan-dependent, so it's a silent correctness bug. A `VALUES` scan ignores the pushed filter and stays correct; a parquet scan applies it and is wrong. #23103 (the probe-side NULL drop) is orthogonal; this is the build-side NULL. ## What changes are included in this PR? Skip join dynamic filter pushdown for a null-aware anti join when the build key can be NULL. The build-side NULL emission depends on whether the probe is truly empty, which the pushed filter can change by emptying it. A NOT NULL build key has no such NULL, so it keeps the pushdown. The check is static: a schema-nullable build key disables the pushdown even when the data contains no NULLs. A runtime alternative (keep the pushdown and neutralize the filter only when the build actually holds a NULL key) would restore the optimization for those cases. I'd leave that as a follow-up. ## Are these changes tested? Yes. A `push_down_filter_parquet.slt` case reproduces it (build-side NULL, a non-matching parquet probe) and asserts the single correct row. Without the change it returns the extra NULL. In addition, unit tests pin both directions of the guard: a nullable build key rejects the pushdown and a NOT NULL build key keeps it. ## Are there any user-facing changes? `NOT IN` over a parquet (or otherwise prunable) scan with a nullable outer key now returns correct results. Such joins lose the dynamic filter pushdown. --- .../physical-plan/src/joins/hash_join/exec.rs | 85 +++++++++++++++++++ .../dynamic_filter_pushdown_config.slt | 4 +- .../sqllogictest/test_files/explain_tree.slt | 5 +- .../test_files/push_down_filter_parquet.slt | 66 ++++++++++++++ 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index ccdb050d168e0..9d9c867c2724b 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -871,6 +871,18 @@ impl HashJoinExec { return false; } + // A null-aware anti join emits a build-side NULL only when the probe + // is truly empty. The pushed filter can empty the probe by pruning + // every row, which would surface that NULL wrongly. A NOT NULL build + // key cannot produce such a NULL, so the filter stays there. + if self.null_aware + && self.on.iter().any(|(build_key, _)| { + build_key.nullable(&self.left.schema()).unwrap_or(true) + }) + { + return false; + } + // `preserve_file_partitions` can report Hash partitioning for Hive-style // file groups, but those partitions are not actually hash-distributed. // Partitioned dynamic filters rely on hash routing, so disable them in @@ -6949,6 +6961,79 @@ mod tests { Ok(()) } + #[test] + fn test_dynamic_filter_pushdown_rejects_null_aware_nullable_build_key() -> Result<()> + { + let left = build_table_two_cols( + ("a1", &vec![Some(1), None]), + ("b1", &vec![Some(1), Some(2)]), + ); + let right = build_table_two_cols( + ("a2", &vec![Some(2), Some(3)]), + ("b2", &vec![Some(1), Some(2)]), + ); + let on = vec![( + Arc::new(Column::new_with_schema("a1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a2", &right.schema())?) as _, + )]; + + let mut session_config = SessionConfig::default(); + session_config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = true; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options())); + + Ok(()) + } + + #[test] + fn test_dynamic_filter_pushdown_allows_null_aware_non_null_build_key() -> Result<()> { + // A NOT NULL build key cannot surface a build-side NULL, so the + // pushdown must stay enabled. + let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1])); + let right = build_table(("a2", &vec![2]), ("b2", &vec![2]), ("c2", &vec![2])); + let on = vec![( + Arc::new(Column::new_with_schema("a1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("a2", &right.schema())?) as _, + )]; + + let mut session_config = SessionConfig::default(); + session_config + .options_mut() + .optimizer + .enable_join_dynamic_filter_pushdown = true; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + assert!(join.allow_join_dynamic_filter_pushdown(session_config.options())); + + Ok(()) + } + #[test] fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()> { diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt index c58047c4abe10..c51a127986421 100644 --- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -383,7 +383,7 @@ logical_plan physical_plan 01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet # LEFT MARK JOIN: the OR prevents decorrelation to LeftSemi, so the optimizer # uses LeftMark. Self-generated dynamic filter pushes to the probe side. @@ -479,7 +479,7 @@ physical_plan 01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] 02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware 03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible # Correctness check query IT diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 8588c0e7ba2ae..4e0397bb41e2e 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -1120,10 +1120,7 @@ physical_plan 13)│ -------------------- ││ -------------------- │ 14)│ files: 1 ││ files: 1 │ 15)│ format: csv ││ format: parquet │ -16)│ ││ │ -17)│ ││ predicate: │ -18)│ ││ DynamicFilter [ empty ] │ -19)└───────────────────────────┘└───────────────────────────┘ +16)└───────────────────────────┘└───────────────────────────┘ # Query with nested loop join. query TT diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index e879947e324bb..f1e787441d5e1 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1066,6 +1066,72 @@ statement ok drop table nej_probe; +######## +# Regression test for build-NULL + emptied-probe interaction in null-aware LeftAnti joins. +# +# `x NOT IN (subquery)` plans as a null-aware LeftAnti hash join where `x` is +# the build (left) side. The dynamic-filter pushdown derives a bounds/membership +# filter from the build keys and pushes it onto the probe scan. When the build +# contains a NULL key and the filter prunes every probe row, the probe looks +# empty to the join. A null-aware LeftAnti treats an empty probe as a genuinely- +# absent subquery, so it emits the build-side NULL as a matching row. That is +# wrong: `NULL NOT IN (non-empty set)` must be UNKNOWN, not TRUE. +# +# The fix: suppress dynamic-filter pushdown whenever the build key is nullable +# and the join is null-aware, so the probe is never artificially emptied. +######## + +statement ok +set datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +# Build side: `ao` has a nullable `id` column; the NULL row is the one that +# must NOT appear in the output. +query I +COPY (SELECT * FROM (VALUES (5), (NULL)) v(id)) +TO 'test_files/scratch/push_down_filter_parquet/ao_p.parquet' +STORED AS PARQUET; +---- +2 + +# Probe / subquery side: `i_disj` has two non-NULL values that don't match 5, +# and no NULLs. The subquery is non-empty, so `NULL NOT IN (...)` is UNKNOWN. +query I +COPY (SELECT * FROM (VALUES (2), (3)) v(eid)) +TO 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet' +STORED AS PARQUET; +---- +2 + +statement ok +CREATE EXTERNAL TABLE ao_p (id INT) STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/ao_p.parquet'; + +statement ok +CREATE EXTERNAL TABLE i_disj_p (eid INT) STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet'; + +# Must return only `5`. `NULL NOT IN (2, 3)` is UNKNOWN, so that row is dropped. +query I +SELECT id FROM ao_p WHERE id NOT IN (SELECT eid FROM i_disj_p) ORDER BY id; +---- +5 + +statement ok +drop table ao_p; + +statement ok +drop table i_disj_p; + +statement ok +RESET datafusion.optimizer.enable_join_dynamic_filter_pushdown; + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; + + # Config reset statement ok RESET datafusion.explain.physical_plan_only; From 50ae0762d393549b3052682522e81989e11a76bb Mon Sep 17 00:00:00 2001 From: Jeffrey Vo Date: Mon, 27 Jul 2026 03:48:36 +0900 Subject: [PATCH 034/109] Various `ScalarValue` numeric method fixes & refactors (especially decimal) (#23631) ## Which issue does this PR close? N/A ## Rationale for this change I noticed there were some subtle errors with how decimals were handled in scalar values, and also opportunity to remove power calls in favour of precomputed constant tables. Also filling out some other missing support. ## What changes are included in this PR? I recommend looking at the commits as they are self contained with detailed messages for each. ## Are these changes tested? Yes ## Are there any user-facing changes? No --- datafusion/common/src/scalar/consts.rs | 63 ++++ datafusion/common/src/scalar/mod.rs | 310 +++++++++++------- .../simplify_expressions/expr_simplifier.rs | 35 -- .../src/simplify_expressions/utils.rs | 102 +----- 4 files changed, 271 insertions(+), 239 deletions(-) diff --git a/datafusion/common/src/scalar/consts.rs b/datafusion/common/src/scalar/consts.rs index 599c2523cd2c7..df12265a3723c 100644 --- a/datafusion/common/src/scalar/consts.rs +++ b/datafusion/common/src/scalar/consts.rs @@ -17,6 +17,9 @@ // Constants defined for scalar construction. +use arrow::datatypes::{Decimal32Type, Decimal64Type, Decimal128Type, DecimalType}; +use arrow::datatypes::{Decimal256Type, i256}; + // Next F16 value above π (upper bound) pub(super) const PI_UPPER_F16: half::f16 = half::f16::from_bits(0x4249); @@ -54,3 +57,63 @@ pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F32: f32 = // Next f64 value below -π/2 (lower bound) pub(super) const NEGATIVE_FRAC_PI_2_LOWER_F64: f64 = (-std::f64::consts::FRAC_PI_2).next_down(); + +// Generate lookup table for 1 values of decimals (1, 10, 100, etc.) +macro_rules! decimal_ones_lut { + () => {{ + let mut values = [1; _]; + let mut i = 1; + while i < values.len() { + values[i] = values[i - 1] * 10; + i += 1; + } + values + }}; +} + +// 1, 10, 100 values meant to be indexed by scale. We omit handling for MAX_SCALE +// itself (we don't go to MAX_SCALE + 1) since we can't represent a 1 value at +// that scale. +pub(super) const DECIMAL32_ONES: [i32; Decimal32Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL64_ONES: [i64; Decimal64Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL128_ONES: [i128; Decimal128Type::MAX_SCALE as usize] = + decimal_ones_lut!(); +pub(super) const DECIMAL256_ONES: [i256; Decimal256Type::MAX_SCALE as usize] = { + // This code was generated by codex and frankly I don't know how it works, + // but the test below verifies it outputs the correct values so ¯\_(ツ)_/¯ + // + // This is mainly a shortcut for not needing to manually list out each value + // anyway. + // + // TODO: simplify this after https://github.com/apache/arrow-rs/pull/10363 + // lands upstream + let mut values = [i256::ONE; _]; + let mut i = 1; + while i < values.len() { + let (low, high) = values[i - 1].to_parts(); + let low_product = (low as u64 as u128) * 10; + let high_product = (low >> 64) * 10 + (low_product >> 64); + let low = ((high_product as u64 as u128) << 64) | low_product as u64 as u128; + let carry = (high_product >> 64) as i128; + values[i] = i256::from_parts(low, high * 10 + carry); + i += 1; + } + values +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ensure_correct_decimal256_ones() { + for (scale, val) in DECIMAL256_ONES.iter().enumerate() { + let zeros = "0".repeat(scale); + let num = "1".to_string() + &zeros; + let num = i256::from_string(&num).unwrap(); + assert_eq!(num, *val, "{scale}"); + } + } +} diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index ddfe32edd41cc..924620a930869 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -54,6 +54,9 @@ use crate::cast::{ use crate::error::{_exec_err, _internal_err, _not_impl_err, DataFusionError, Result}; use crate::format::DEFAULT_CAST_OPTIONS; use crate::hash_utils::create_hashes; +use crate::scalar::consts::{ + DECIMAL32_ONES, DECIMAL64_ONES, DECIMAL128_ONES, DECIMAL256_ONES, +}; use crate::utils::SingleRowListArrayBuilder; use crate::{_internal_datafusion_err, arrow_datafusion_err}; use arrow::array::{ @@ -83,10 +86,14 @@ use arrow::datatypes::{ Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, DecimalType, Field, FieldRef, Float32Type, Int8Type, Int16Type, Int32Type, Int64Type, IntervalDayTime, IntervalDayTimeType, IntervalMonthDayNano, IntervalMonthDayNanoType, IntervalUnit, - IntervalYearMonthType, RunEndIndexType, TimeUnit, TimestampMicrosecondType, - TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt8Type, - UInt16Type, UInt32Type, UInt64Type, UnionFields, UnionMode, i256, - validate_decimal_precision_and_scale, + IntervalYearMonthType, MAX_DECIMAL32_FOR_EACH_PRECISION, + MAX_DECIMAL64_FOR_EACH_PRECISION, MAX_DECIMAL128_FOR_EACH_PRECISION, + MAX_DECIMAL256_FOR_EACH_PRECISION, MIN_DECIMAL32_FOR_EACH_PRECISION, + MIN_DECIMAL64_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION, + MIN_DECIMAL256_FOR_EACH_PRECISION, RunEndIndexType, TimeUnit, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, UnionFields, + UnionMode, i256, validate_decimal_precision_and_scale, }; use arrow::util::display::{ArrayFormatter, FormatOptions, array_value_to_string}; use cache::{get_or_create_cached_key_array, get_or_create_cached_null_array}; @@ -1804,48 +1811,56 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL32_ONES[*scale as usize]; + ScalarValue::Decimal32(Some(one), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL64_ONES[*scale as usize]; + ScalarValue::Decimal64(Some(one), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL128_ONES[*scale as usize]; + ScalarValue::Decimal128(Some(one), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL256_ONES[*scale as usize]; + ScalarValue::Decimal256(Some(one), *precision, *scale) } _ => { return _not_impl_err!( @@ -1858,10 +1873,10 @@ impl ScalarValue { /// Create a negative one value in the given type. pub fn new_negative_one(datatype: &DataType) -> Result { Ok(match datatype { - DataType::Int8 | DataType::UInt8 => ScalarValue::Int8(Some(-1)), - DataType::Int16 | DataType::UInt16 => ScalarValue::Int16(Some(-1)), - DataType::Int32 | DataType::UInt32 => ScalarValue::Int32(Some(-1)), - DataType::Int64 | DataType::UInt64 => ScalarValue::Int64(Some(-1)), + DataType::Int8 => ScalarValue::Int8(Some(-1)), + DataType::Int16 => ScalarValue::Int16(Some(-1)), + DataType::Int32 => ScalarValue::Int32(Some(-1)), + DataType::Int64 => ScalarValue::Int64(Some(-1)), DataType::Float16 => ScalarValue::Float16(Some(f16::NEG_ONE)), DataType::Float32 => ScalarValue::Float32(Some(-1.0)), DataType::Float64 => ScalarValue::Float64(Some(-1.0)), @@ -1870,48 +1885,56 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL32_ONES[*scale as usize]; + ScalarValue::Decimal32(Some(-one), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL64_ONES[*scale as usize]; + ScalarValue::Decimal64(Some(-one), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent negative one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL128_ONES[*scale as usize]; + ScalarValue::Decimal128(Some(-one), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow(*scale as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(-value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + *precision != *scale as u8, + "Can't represent one at scale {} with precision {}", + *scale, + *precision + ); + let one = DECIMAL256_ONES[*scale as usize]; + ScalarValue::Decimal256(Some(-one), *precision, *scale) } _ => { return _not_impl_err!( @@ -1939,48 +1962,64 @@ impl ScalarValue { *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match 10_i32.checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal32(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL32_ONES[*scale as usize + 1]; + ScalarValue::Decimal32(Some(ten), *precision, *scale) } DataType::Decimal64(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i64::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal64(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL64_ONES[*scale as usize + 1]; + ScalarValue::Decimal64(Some(ten), *precision, *scale) } DataType::Decimal128(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i128::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal128(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL128_ONES[*scale as usize + 1]; + ScalarValue::Decimal128(Some(ten), *precision, *scale) } DataType::Decimal256(precision, scale) => { Self::validate_decimal_or_internal_err::( *precision, *scale, )?; assert_or_internal_err!(*scale >= 0, "Negative scale is not supported"); - match i256::from(10).checked_pow((*scale + 1) as u32) { - Some(value) => { - ScalarValue::Decimal256(Some(value), *precision, *scale) - } - None => return _internal_err!("Unsupported scale {scale}"), - } + assert_or_internal_err!( + (*precision - *scale as u8) > 1, + "Can't represent ten at scale {} with precision {}", + *scale, + *precision + ); + // +1 safe since we validate above that scale must be less than + // the max possible scale + let ten = DECIMAL256_ONES[*scale as usize + 1]; + ScalarValue::Decimal256(Some(ten), *precision, *scale) } _ => { return _not_impl_err!( @@ -2299,7 +2338,18 @@ impl ScalarValue { | ScalarValue::Int64(None) | ScalarValue::Float16(None) | ScalarValue::Float32(None) - | ScalarValue::Float64(None) => Ok(self.clone()), + | ScalarValue::Float64(None) + | ScalarValue::IntervalYearMonth(None) + | ScalarValue::IntervalDayTime(None) + | ScalarValue::IntervalMonthDayNano(None) + | ScalarValue::Decimal32(None, _, _) + | ScalarValue::Decimal64(None, _, _) + | ScalarValue::Decimal128(None, _, _) + | ScalarValue::Decimal256(None, _, _) + | ScalarValue::TimestampSecond(None, _) + | ScalarValue::TimestampMillisecond(None, _) + | ScalarValue::TimestampMicrosecond(None, _) + | ScalarValue::TimestampNanosecond(None, _) => Ok(self.clone()), ScalarValue::Float16(Some(v)) => Ok(ScalarValue::Float16(Some(-v))), ScalarValue::Float64(Some(v)) => Ok(ScalarValue::Float64(Some(-v))), ScalarValue::Float32(Some(v)) => Ok(ScalarValue::Float32(Some(-v))), @@ -5053,28 +5103,21 @@ impl ScalarValue { DataType::Float16 => Some(ScalarValue::Float16(Some(f16::NEG_INFINITY))), DataType::Float32 => Some(ScalarValue::Float32(Some(f32::NEG_INFINITY))), DataType::Float64 => Some(ScalarValue::Float64(Some(f64::NEG_INFINITY))), + DataType::Decimal32(precision, scale) => { + let min = MIN_DECIMAL32_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal32(Some(min), *precision, *scale)) + } + DataType::Decimal64(precision, scale) => { + let min = MIN_DECIMAL64_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal64(Some(min), *precision, *scale)) + } DataType::Decimal128(precision, scale) => { - // For decimal, min is -10^(precision-scale) + 10^(-scale) - // But for simplicity, we use the minimum i128 value that fits the precision - let max_digits = 10_i128.pow(*precision as u32) - 1; - Some(ScalarValue::Decimal128( - Some(-max_digits), - *precision, - *scale, - )) + let min = MIN_DECIMAL128_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal128(Some(min), *precision, *scale)) } DataType::Decimal256(precision, scale) => { - // Similar to Decimal128 but with i256 - // For now, use a large negative value - let max_digits = i256::from_i128(10_i128) - .checked_pow(*precision as u32) - .and_then(|v| v.checked_sub(i256::from_i128(1))) - .unwrap_or(i256::MAX); - Some(ScalarValue::Decimal256( - Some(max_digits.neg_wrapping()), - *precision, - *scale, - )) + let min = MIN_DECIMAL256_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal256(Some(min), *precision, *scale)) } DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MIN))), DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MIN))), @@ -5149,27 +5192,21 @@ impl ScalarValue { DataType::Float16 => Some(ScalarValue::Float16(Some(f16::INFINITY))), DataType::Float32 => Some(ScalarValue::Float32(Some(f32::INFINITY))), DataType::Float64 => Some(ScalarValue::Float64(Some(f64::INFINITY))), + DataType::Decimal32(precision, scale) => { + let max = MAX_DECIMAL32_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal32(Some(max), *precision, *scale)) + } + DataType::Decimal64(precision, scale) => { + let max = MAX_DECIMAL64_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal64(Some(max), *precision, *scale)) + } DataType::Decimal128(precision, scale) => { - // For decimal, max is 10^(precision-scale) - 10^(-scale) - // But for simplicity, we use the maximum i128 value that fits the precision - let max_digits = 10_i128.pow(*precision as u32) - 1; - Some(ScalarValue::Decimal128( - Some(max_digits), - *precision, - *scale, - )) + let max = MAX_DECIMAL128_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal128(Some(max), *precision, *scale)) } DataType::Decimal256(precision, scale) => { - // Similar to Decimal128 but with i256 - let max_digits = i256::from_i128(10_i128) - .checked_pow(*precision as u32) - .and_then(|v| v.checked_sub(i256::from_i128(1))) - .unwrap_or(i256::MAX); - Some(ScalarValue::Decimal256( - Some(max_digits), - *precision, - *scale, - )) + let max = MAX_DECIMAL256_FOR_EACH_PRECISION[*precision as usize]; + Some(ScalarValue::Decimal256(Some(max), *precision, *scale)) } DataType::Date32 => Some(ScalarValue::Date32(Some(i32::MAX))), DataType::Date64 => Some(ScalarValue::Date64(Some(i64::MAX))), @@ -11470,4 +11507,41 @@ mod tests { .unwrap(); assert_eq!(s.to_string(), "[]"); } + + #[test] + fn test_decimal_value_bounds() { + fn run_tests() { + // 0.1111, 0.2222, etc. + let max_scale = D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE); + // 1.111, 2.222, etc. + let max_scale_less_one = + D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 1); + // 11.11, 22.22, etc. + let max_scale_less_two = + D::TYPE_CONSTRUCTOR(D::MAX_PRECISION, D::MAX_SCALE - 2); + + // Invalid (can't represent the value) + assert!(ScalarValue::new_one(&max_scale).is_err()); + assert!(ScalarValue::new_negative_one(&max_scale).is_err()); + assert!(ScalarValue::new_ten(&max_scale).is_err()); + assert!(ScalarValue::new_ten(&max_scale_less_one).is_err()); + + // Valid + let one = ScalarValue::Int32(Some(1)); + let neg_one = ScalarValue::Int32(Some(-1)); + let ten = ScalarValue::Int32(Some(10)); + + let num = ScalarValue::new_one(&max_scale_less_one).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), one); + let num = ScalarValue::new_negative_one(&max_scale_less_one).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), neg_one); + let num = ScalarValue::new_ten(&max_scale_less_two).unwrap(); + assert_eq!(num.cast_to(&DataType::Int32).unwrap(), ten); + } + + run_tests::(); + run_tests::(); + run_tests::(); + run_tests::(); + } } diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index e4a22a341992e..f5ea75dde8612 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -3067,17 +3067,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_and() { - // !c4 & c4 --> 0 - let expr = (-col("c4_non_null")) & col("c4_non_null"); - let expected = lit(0u32); - - assert_eq!(simplify(expr), expected); - // c4 & !c4 --> 0 - let expr = col("c4_non_null") & (-col("c4_non_null")); - let expected = lit(0u32); - - assert_eq!(simplify(expr), expected); - // !c3 & c3 --> 0 let expr = (-col("c3_non_null")) & col("c3_non_null"); let expected = lit(0i64); @@ -3092,18 +3081,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_or() { - // !c4 | c4 --> -1 - let expr = (-col("c4_non_null")) | col("c4_non_null"); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - - // c4 | !c4 --> -1 - let expr = col("c4_non_null") | (-col("c4_non_null")); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - // !c3 | c3 --> -1 let expr = (-col("c3_non_null")) | col("c3_non_null"); let expected = lit(-1i64); @@ -3119,18 +3096,6 @@ mod tests { #[test] fn test_simplify_negated_bitwise_xor() { - // !c4 ^ c4 --> -1 - let expr = (-col("c4_non_null")) ^ col("c4_non_null"); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - - // c4 ^ !c4 --> -1 - let expr = col("c4_non_null") ^ (-col("c4_non_null")); - let expected = lit(-1i32); - - assert_eq!(simplify(expr), expected); - // !c3 ^ c3 --> -1 let expr = (-col("c3_non_null")) ^ col("c3_non_null"); let expected = lit(-1i64); diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index b0908b47602f7..89bb762d59ce2 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -17,7 +17,6 @@ //! Utility functions for expression simplification -use arrow::datatypes::i256; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ Case, Expr, Like, Operator, @@ -25,47 +24,6 @@ use datafusion_expr::{ expr_fn::{and, bitwise_and, bitwise_or, or}, }; -pub static POWS_OF_TEN: [i128; 38] = [ - 1, - 10, - 100, - 1000, - 10000, - 100000, - 1000000, - 10000000, - 100000000, - 1000000000, - 10000000000, - 100000000000, - 1000000000000, - 10000000000000, - 100000000000000, - 1000000000000000, - 10000000000000000, - 100000000000000000, - 1000000000000000000, - 10000000000000000000, - 100000000000000000000, - 1000000000000000000000, - 10000000000000000000000, - 100000000000000000000000, - 1000000000000000000000000, - 10000000000000000000000000, - 100000000000000000000000000, - 1000000000000000000000000000, - 10000000000000000000000000000, - 100000000000000000000000000000, - 1000000000000000000000000000000, - 10000000000000000000000000000000, - 100000000000000000000000000000000, - 1000000000000000000000000000000000, - 10000000000000000000000000000000000, - 100000000000000000000000000000000000, - 1000000000000000000000000000000000000, - 10000000000000000000000000000000000000, -]; - /// returns true if `needle` is found in a chain of search_op /// expressions. Such as: (A AND B) AND C fn expr_contains_inner(expr: &Expr, needle: &Expr, search_op: Operator) -> bool { @@ -139,54 +97,26 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) -> } pub fn is_zero(s: &Expr) -> bool { - match s { - Expr::Literal(ScalarValue::Int8(Some(0)), _) - | Expr::Literal(ScalarValue::Int16(Some(0)), _) - | Expr::Literal(ScalarValue::Int32(Some(0)), _) - | Expr::Literal(ScalarValue::Int64(Some(0)), _) - | Expr::Literal(ScalarValue::UInt8(Some(0)), _) - | Expr::Literal(ScalarValue::UInt16(Some(0)), _) - | Expr::Literal(ScalarValue::UInt32(Some(0)), _) - | Expr::Literal(ScalarValue::UInt64(Some(0)), _) => true, - Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 0. => true, - Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 0. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, _s), _) if *v == 0 => true, - Expr::Literal(ScalarValue::Decimal256(Some(v), _p, _s), _) - if *v == i256::ZERO => - { - true - } - _ => false, + if let Expr::Literal(sv, _) = s + && sv.data_type().is_numeric() + { + // unwrap safe since numeric types always have a 0 value + sv == &ScalarValue::new_zero(&sv.data_type()).unwrap() + } else { + false } } pub fn is_one(s: &Expr) -> bool { - match s { - Expr::Literal(ScalarValue::Int8(Some(1)), _) - | Expr::Literal(ScalarValue::Int16(Some(1)), _) - | Expr::Literal(ScalarValue::Int32(Some(1)), _) - | Expr::Literal(ScalarValue::Int64(Some(1)), _) - | Expr::Literal(ScalarValue::UInt8(Some(1)), _) - | Expr::Literal(ScalarValue::UInt16(Some(1)), _) - | Expr::Literal(ScalarValue::UInt32(Some(1)), _) - | Expr::Literal(ScalarValue::UInt64(Some(1)), _) => true, - Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 1. => true, - Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 1. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, s), _) => { - *s >= 0 - && POWS_OF_TEN - .get(*s as usize) - .map(|x| x == v) - .unwrap_or_default() - } - Expr::Literal(ScalarValue::Decimal256(Some(v), _p, s), _) => { - *s >= 0 - && match i256::from(10).checked_pow(*s as u32) { - Some(res) => res == *v, - None => false, - } - } - _ => false, + if let Expr::Literal(sv, _) = s + && sv.data_type().is_numeric() + // there are edge cases like negative scale decimals not being able to + // create a one value so this can fail + && let Ok(one) = ScalarValue::new_one(&sv.data_type()) + { + sv == &one + } else { + false } } From dd4dcbc2e425d1e6f547179269af31d3dcc00cd6 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Sun, 26 Jul 2026 19:48:49 +0100 Subject: [PATCH 035/109] fix: Handle `input_file_name()` pushdown into `ParquetSource` with filter pushdown enabled (#23638) ## Which issue does this PR close? - Closes #23531. ## Rationale for this change `input_file_name()` is `FileSource` dependent like `file_row_index()`, and therefore shouldn't be pushed down into a filter. ## What changes are included in this PR? `PushdownChecker` now handles both UDFs consistently. If we keep adding this sort of metadata functions, we might want a better API to detect them, but for now I think this is a reasonable approach that isn't very invasive. ## Are these changes tested? Additional SLT test that verifies that both function behave correctly with pushdown either enabled or disabled. ## Are there any user-facing changes? None --------- Signed-off-by: Adam Gutglick --- .../src/projection_read_plan.rs | 5 +++- .../test_files/input_file_name.slt | 5 ++-- .../test_files/parquet_metadata_functions.slt | 28 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rs index c9d8beab1466d..96c99ab20750e 100644 --- a/datafusion/datasource-parquet/src/projection_read_plan.rs +++ b/datafusion/datasource-parquet/src/projection_read_plan.rs @@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion_functions::core::input_file_name::InputFileNameFunc; use parquet::arrow::ProjectionMask; use parquet::schema::types::SchemaDescriptor; @@ -313,8 +314,10 @@ impl TreeNodeVisitor<'_> for PushdownChecker<'_> { return Ok(recursion); } - if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + if ScalarFunctionExpr::try_downcast_func::(node.as_ref()) .is_some() + || ScalarFunctionExpr::try_downcast_func::(node.as_ref()) + .is_some() { self.has_unpushable_udfs = true; return Ok(TreeNodeRecursion::Jump); diff --git a/datafusion/sqllogictest/test_files/input_file_name.slt b/datafusion/sqllogictest/test_files/input_file_name.slt index 8fb72d4a9d14b..32110aa2d69af 100644 --- a/datafusion/sqllogictest/test_files/input_file_name.slt +++ b/datafusion/sqllogictest/test_files/input_file_name.slt @@ -121,8 +121,7 @@ physical_plan 01)SortPreservingMergeExec: [column1@0 ASC NULLS LAST] 02)--SortExec: expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[true] 03)----FilterExec: __datafusion_extracted_1@0 LIKE %first.parquet, projection=[column1@1] -04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 -05)--------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet, predicate=input_file_name() LIKE %first.parquet +04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/input_file_name/parquet/second.parquet]]}, projection=[input_file_name() as __datafusion_extracted_1, column1], file_type=parquet statement ok -DROP TABLE pq_table; \ No newline at end of file +DROP TABLE pq_table; diff --git a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt index 773ab6761fd26..25a3c4eb4c6fa 100644 --- a/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt +++ b/datafusion/sqllogictest/test_files/parquet_metadata_functions.slt @@ -52,5 +52,33 @@ logical_plan 02)--TableScan: test_table projection=[column1] physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet]]}, projection=[input_file_name() as input_file_name(), CAST(__datafusion_file_row_index@1 AS Int64) as file_row_index(), column1], file_type=parquet + +# Make sure it also behaves consistently regardless of filter pushdown + +statement ok +SET datafusion.execution.parquet.pushdown_filters = false; + +query TII rowsort +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +SET datafusion.execution.parquet.pushdown_filters = true; + +query TII rowsort +SELECT input_file_name(), file_row_index(), column1 +FROM test_table +WHERE file_row_index() = 2 AND input_file_name() LIKE '%parquet'; +---- +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/first.parquet 2 30 +WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_metadata_functions/second.parquet 2 60 + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; + statement ok DROP TABLE test_table; From 7576762766d0f8081d7cd487e92be4861cf0c485 Mon Sep 17 00:00:00 2001 From: Namgung Chan <33323415+getChan@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:49:53 +0900 Subject: [PATCH 036/109] fix: exclude precision-losing integer-to-float conversions from CastExpr::check_bigger_cast (#23808) (#23809) ## Which issue does this PR close? - Closes #23808. ## Rationale for this change ref. #23807 `CastExpr::check_bigger_cast` is used to determine whether a cast is a widening (order-preserving) conversion. Currently, it classifies `Int32 -> Float32`, `UInt32 -> Float32`, `Int64 -> Float64`, and `UInt64 -> Float64` as widening casts. However, integer-to-float conversions for 32-bit and 64-bit integers lose precision when values exceed the mantissa bit limit (24 bits for `Float32`, 53 bits for `Float64`). For example: `16_777_216_i32 as f32 == 16_777_217_i32 as f32` (both yield 16777216.0f32). Because distinct integer inputs can collapse to the same float output, these casts are not strictly 1-to-1 (injective) and can break suffix sort key ordering in multi-column ordering analysis (e.g. `[CAST(a AS Float32), b]`). ## What changes are included in this PR? - Updated `CastExpr::check_bigger_cast` to exclude precision-losing integer-to-float conversions (`Int32/UInt32 -> Float32` and `Int64/UInt64 -> Float64`). - Added unit test `test_check_bigger_cast_precision_loss` to verify precision-losing casts return `false` while exact conversions (`Int16 -> Float32`, `Int32 -> Float64`, etc.) continue to return `true`. ## Are these changes tested? Yes, new unit test `test_check_bigger_cast_precision_loss` in `cast.rs`. ## Are there any user-facing changes? No breaking API changes. Internal optimizer behavior fix. --- .../data/int_to_float_cast_precision.csv | 3 + .../physical-expr/src/expressions/cast.rs | 32 ++++++++-- .../test_files/monotonic_projection_test.slt | 63 +++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 datafusion/core/tests/data/int_to_float_cast_precision.csv diff --git a/datafusion/core/tests/data/int_to_float_cast_precision.csv b/datafusion/core/tests/data/int_to_float_cast_precision.csv new file mode 100644 index 0000000000000..187d7affca616 --- /dev/null +++ b/datafusion/core/tests/data/int_to_float_cast_precision.csv @@ -0,0 +1,3 @@ +k,v +1,16777217 +2,16777216 diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 26f06b546ad1d..8be2e187d72f7 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -179,11 +179,8 @@ impl CastExpr { | (UInt8, UInt16 | UInt32 | UInt64) | (UInt16, UInt32 | UInt64) | (UInt32, UInt64) - | ( - Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32, - Float32 | Float64 - ) - | (Int64 | UInt64, Float64) + | (Int8 | Int16 | UInt8 | UInt16, Float32) + | (Int8 | Int16 | Int32 | UInt8 | UInt16 | UInt32, Float64) | (Utf8, LargeUtf8) ) } @@ -1208,6 +1205,31 @@ mod tests { Ok(()) } + + #[test] + fn test_check_bigger_cast_precision_loss() { + use DataType::*; + + // Exact conversions without precision loss + assert!(CastExpr::check_bigger_cast(&Int16, &Int8)); + assert!(CastExpr::check_bigger_cast(&Int64, &Int32)); + assert!(CastExpr::check_bigger_cast(&Float32, &Int16)); + assert!(CastExpr::check_bigger_cast(&Float32, &UInt16)); + assert!(CastExpr::check_bigger_cast(&Float64, &Int32)); + assert!(CastExpr::check_bigger_cast(&Float64, &UInt32)); + assert!(CastExpr::check_bigger_cast(&LargeUtf8, &Utf8)); + + // Precision-losing int-to-float conversions should return false + assert!(!CastExpr::check_bigger_cast(&Float32, &Int32)); + assert!(!CastExpr::check_bigger_cast(&Float32, &UInt32)); + assert!(!CastExpr::check_bigger_cast(&Float64, &Int64)); + assert!(!CastExpr::check_bigger_cast(&Float64, &UInt64)); + + // Signed <-> Unsigned conversions should return false (not order-preserving due to negative values) + assert!(!CastExpr::check_bigger_cast(&UInt16, &Int8)); + assert!(!CastExpr::check_bigger_cast(&UInt32, &Int16)); + assert!(!CastExpr::check_bigger_cast(&Int16, &UInt8)); + } } /// Tests for the `try_to_proto` / `try_from_proto` hooks. diff --git a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt index 0045e51715980..71e5fbc08e3eb 100644 --- a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt +++ b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt @@ -252,3 +252,66 @@ ORDER BY a, b; ---- a 1 a0 1 + +# Test that precision-losing int-to-float casts do not invalidate suffix sort keys. +# +# When CAST(Int32 AS Float32) collapses distinct integer values (e.g., 16777216 and +# 16777217 both become 16777216.0), the suffix sort key (k) must still be sorted +# correctly. Before the fix, the optimizer incorrectly reused the pre-existing sort +# order and dropped the SortExec, producing wrong results. +# +# t1 is declared with a sort order, t2 is not — their results should be identical +# since CAST(v AS FLOAT) is not injective for 32-bit integers. +statement ok +CREATE EXTERNAL TABLE t1_int_float (k int, v int) +STORED AS CSV +WITH ORDER (v DESC, k DESC) +LOCATION '../core/tests/data/int_to_float_cast_precision.csv' +OPTIONS ('format.has_header' 'true'); + +statement ok +CREATE EXTERNAL TABLE t2_int_float (k int, v int) +STORED AS CSV +LOCATION '../core/tests/data/int_to_float_cast_precision.csv' +OPTIONS ('format.has_header' 'true'); + +# Both queries must return the same result: k=2 before k=1. +# (v_=16777216.0 for both rows; when tied on v_, DESC on k means k=2 comes first) +query IR +SELECT k, cast(v as float) v_ FROM t1_int_float ORDER BY v_ DESC, k DESC; +---- +2 16777216 +1 16777216 + +query IR +SELECT k, cast(v as float) v_ FROM t2_int_float ORDER BY v_ DESC, k DESC; +---- +2 16777216 +1 16777216 + +# Widening cast (Int32 -> Int64) is strictly 1-to-1, so the optimizer CAN +# legally reuse the pre-existing sort order and omit a SortExec. +statement ok +CREATE EXTERNAL TABLE t3_int_bigint (k int, v int) +STORED AS CSV +WITH ORDER (v DESC, k DESC) +LOCATION '../core/tests/data/int_to_float_cast_precision.csv' +OPTIONS ('format.has_header' 'true'); + +# CAST(Int32 AS BIGINT) is injective, so suffix key ordering is preserved. +query II +SELECT k, cast(v as bigint) v_ FROM t3_int_bigint ORDER BY v_ DESC, k DESC; +---- +1 16777217 +2 16777216 + +# Cleanup +statement ok +DROP TABLE t1_int_float; + +statement ok +DROP TABLE t2_int_float; + +statement ok +DROP TABLE t3_int_bigint; + From 77b172e1505283f9ac6af050ee8547f910c28262 Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:55:27 +0300 Subject: [PATCH 037/109] chore: simplify SortPreservingMergeStream to be as textbook-like as possible (#23702) ## Which issue does this PR close? N/A ## Rationale for this change `SortPreservingMergeStream` is a little complex, so add some guiding comments and make it as textbook-like as possible ## What changes are included in this PR? Added comments, reorder code While this was done this also fixed couple of bugs due to how it work: 1. leftover drain was not counted in the `elapsed_compute` 2. `limit(0)` returns 0 rows and not 1 ## Are these changes tested? Existing tests ## Are there any user-facing changes? Not API ones. `limit(0)` now returns 0 rows --- datafusion/physical-plan/src/sorts/merge.rs | 153 ++++++++++++------ .../src/sorts/streaming_merge.rs | 127 ++++++++++++++- 2 files changed, 220 insertions(+), 60 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/merge.rs b/datafusion/physical-plan/src/sorts/merge.rs index 986da549f75c8..310416c22d982 100644 --- a/datafusion/physical-plan/src/sorts/merge.rs +++ b/datafusion/physical-plan/src/sorts/merge.rs @@ -32,9 +32,9 @@ use crate::stream::{ObservedStream, RecordBatchStreamAdapter}; use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; -use datafusion_common::Result; -use datafusion_execution::async_try_stream; +use datafusion_common::{DataFusionError, Result, assert_or_internal_err, internal_err}; use datafusion_execution::memory_pool::MemoryReservation; +use datafusion_execution::{TryEmitter, async_try_stream}; use futures::Stream; /// A fallible [`PartitionedStream`] of [`Cursor`] and [`RecordBatch`] @@ -146,6 +146,9 @@ impl SortPreservingMergeStream { reservation: MemoryReservation, enable_round_robin_tie_breaker: bool, ) -> Self { + assert_ne!(batch_size, 0, "batch size cannot be 0"); + assert_ne!(fetch, Some(0), "fetch must not be Some(0)"); + let stream_count = streams.partitions(); Self { @@ -173,7 +176,6 @@ impl SortPreservingMergeStream { let schema_clone = Arc::clone(self.in_progress.schema()); let cloned_metrics = self.metrics.clone(); - let stream = Box::pin(RecordBatchStreamAdapter::new( schema_clone, self.create_stream(), @@ -212,79 +214,122 @@ impl SortPreservingMergeStream { result } - fn create_stream(mut self) -> impl Stream> { - async_try_stream(|mut emitter| async move { - // This vector contains the indices of the partitions that have not started emitting yet. - let mut uninitiated_partitions = - (0..self.streams.partitions()).collect::>(); + async fn flush_in_progress( + &mut self, + mut emitter: TryEmitter, + ) -> Result<()> { + if self.in_progress.is_empty() { + return Ok(()); + } - poll_fn(|cx| self.initialize_all_partitions(&mut uninitiated_partitions, cx)) - .await?; + let elapsed_compute = self.metrics.elapsed_compute().clone(); + let mut timer = elapsed_compute.timer(); + + // When `build_record_batch()` hits an i32 offset overflow (e.g. + // combined string offsets exceed 2 GB), it emits a partial batch + // and keeps the remaining rows in `self.in_progress.indices`. + // Drain those leftover rows before terminating the stream, + // otherwise they would be silently dropped. + // Repeated overflows are fine — each poll emits another partial + // batch until `in_progress` is fully drained. + while let Some(batch) = self.emit_in_progress_batch()? { + drop(timer); + emitter.emit(batch).await; + timer = elapsed_compute.timer(); + } - assert_eq!(uninitiated_partitions.len(), 0); + Ok(()) + } - // If there are no more uninitiated partitions, set up the loser tree and continue - // to the next phase. + fn create_stream(mut self) -> impl Stream> { + async_try_stream(|mut emitter| async move { + // 1. Make sure we have data from each stream so we can initialize the loser tree + { + // This vector contains the indices of the partitions that have not started emitting yet. + let mut uninitiated_partitions = + (0..self.streams.partitions()).collect::>(); + + poll_fn(|cx| { + self.initialize_all_partitions(&mut uninitiated_partitions, cx) + }) + .await?; - // Claim the memory for the uninitiated partitions - drop(uninitiated_partitions); - self.init_loser_tree(); + assert_eq!(uninitiated_partitions.len(), 0); + } - // NB timer records time taken on drop, so there are no - // calls to `timer.done()` below. let elapsed_compute = self.metrics.elapsed_compute().clone(); let mut timer = elapsed_compute.timer(); - loop { - let stream_idx = self.loser_tree[0]; - if !self.advance_cursors(stream_idx) { - break; - } - self.in_progress.push_row(stream_idx); + // 2. Init loser tree + self.init_loser_tree(); - // stop sorting if fetch has been reached + // 3. loop until all streams have been exhausted + while !self.is_exhausted() { + // 3.1. add loser_tree[0] (minimum) stream to pending record batch + let winner_stream = self.loser_tree[0]; + self.in_progress.push_row(winner_stream); + + // 3.2. If the new row reached the limit if self.fetch_reached() { break; } - if self.in_progress.len() >= self.batch_size - && let Some(batch) = self.emit_in_progress_batch()? - { + // 3.3. if there is enough to emit for a full record batch + if self.in_progress.len() >= self.batch_size { + // 3.3.1 build pending record batch and reset builder + let Some(batch) = self.emit_in_progress_batch()? else { + return internal_err!("must have batch in progress to emit"); + }; + + // 3.3.2 emit pending record batch drop(timer); emitter.emit(batch).await; timer = elapsed_compute.timer(); } - let winner = self.loser_tree[0]; - // Fast path: skip the `maybe_poll_stream` call (and its `Poll` - // plumbing) unless the winner's cursor is exhausted and needs a - // fresh batch — it is live for almost every row. - if self.cursors[winner].is_none() { - drop(timer); - poll_fn(|cx| self.maybe_poll_stream(cx, winner)).await?; - timer = elapsed_compute.timer(); + // 3.4. advance cursor for the winner stream + { + let should_poll_next_batch_for_stream = + self.advance_cursors(winner_stream); + + // Fast path: skip the `maybe_poll_stream` call (and its `Poll` + // plumbing) unless the winner's cursor is exhausted and needs a + // fresh batch — it is live for almost every row. + if should_poll_next_batch_for_stream { + assert_or_internal_err!( + self.cursors[winner_stream].is_none(), + "cursor should be exhausted" + ); + + drop(timer); + poll_fn(|cx| self.maybe_poll_stream(cx, winner_stream)).await?; + timer = elapsed_compute.timer(); + } } - // Adjusting the loser tree if necessary + // 3.5. Adjusting the loser tree if necessary self.update_loser_tree(); } - drop(timer); + // 4. Flush any remaining rows in `self.in_progress` + self.flush_in_progress(emitter).await?; - // When `build_record_batch()` hits an i32 offset overflow (e.g. - // combined string offsets exceed 2 GB), it emits a partial batch - // and keeps the remaining rows in `self.in_progress.indices`. - // Drain those leftover rows before terminating the stream, - // otherwise they would be silently dropped. - // Repeated overflows are fine — each poll emits another partial - // batch until `in_progress` is fully drained. - while let Some(batch) = self.emit_in_progress_batch()? { - emitter.emit(batch).await; - } Ok(()) }) } + /// Returns `true` once every input stream is exhausted. + /// + /// Should only be called for valid adjusted tree, i.e. the initial tree or after [`Self::update_loser_tree`] call + fn is_exhausted(&self) -> bool { + let winner = self.loser_tree[0]; + + // Checking only the tree root suffices for valid tree + // since the winner of the tree cannot be an exhausted stream for a valid tree + // as what value is winning over the non exhausted stream? + self.cursors[winner].is_none() + } + /// Initialize all partitions, return `Poll::Pending` if any partition returns `Poll::Pending` /// /// This DOES NOT return `Poll::Pending` as soon as the first uninitiated partition returns `Poll::Pending` @@ -369,18 +414,20 @@ impl SortPreservingMergeStream { /// Advances the actual cursor. If it reaches its end, update the /// previous cursor with it. /// - /// If the given partition is not exhausted, the function returns `true`. + /// If the given partition batch is exhausted, return `true` to signal a poll is needed fn advance_cursors(&mut self, stream_idx: usize) -> bool { if let Some(cursor) = &mut self.cursors[stream_idx] { let _ = cursor.advance(); - if cursor.is_finished() { + let finished = cursor.is_finished(); + if finished { // Take the current cursor, leaving `None` in its place self.prev_cursors[stream_idx] = self.cursors[stream_idx].take(); } - true - } else { - false + return finished; } + + // the entire stream is exhausted, so return true (poll won't help here anyway) + true } /// Returns `true` if the cursor at index `a` is greater than at index `b`. diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs index e96138ef1306c..81adad8e9ec84 100644 --- a/datafusion/physical-plan/src/sorts/streaming_merge.rs +++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs @@ -24,7 +24,7 @@ use crate::sorts::{ merge::SortPreservingMergeStream, stream::{FieldCursorStream, RowCursorStream}, }; -use crate::{SendableRecordBatchStream, SpillManager}; +use crate::{EmptyRecordBatchStream, SendableRecordBatchStream, SpillManager}; use arrow::array::*; use arrow::datatypes::{DataType, SchemaRef}; use datafusion_common::human_readable_size; @@ -195,13 +195,22 @@ impl<'a> StreamingMergeBuilder<'a> { let Some(expressions) = expressions else { return internal_err!("Sort expressions cannot be empty for streaming merge"); }; + let schema = schema.expect("Schema cannot be empty for streaming merge"); + + if fetch.is_some_and(|fetch| fetch == 0) { + return Ok(Box::pin(EmptyRecordBatchStream::new(schema))); + } + + let batch_size = + batch_size.expect("Batch size cannot be empty for streaming merge"); + + if batch_size == 0 { + return internal_err!("Batch size cannot be zero for streaming merge"); + } if !sorted_spill_files.is_empty() { // Unwrapping mandatory fields - let schema = schema.expect("Schema cannot be empty for streaming merge"); let metrics = metrics.expect("Metrics cannot be empty for streaming merge"); - let batch_size = - batch_size.expect("Batch size cannot be empty for streaming merge"); let reservation = reservation.expect("Reservation cannot be empty for streaming merge"); @@ -227,10 +236,7 @@ impl<'a> StreamingMergeBuilder<'a> { ); // Unwrapping mandatory fields - let schema = schema.expect("Schema cannot be empty for streaming merge"); let metrics = metrics.expect("Metrics cannot be empty for streaming merge"); - let batch_size = - batch_size.expect("Batch size cannot be empty for streaming merge"); let reservation = reservation.expect("Reservation cannot be empty for streaming merge"); @@ -267,3 +273,110 @@ impl<'a> StreamingMergeBuilder<'a> { .into_stream()) } } + +#[cfg(test)] +mod tests { + use crate::{common::collect, stream::RecordBatchStreamAdapter}; + use std::sync::Arc; + + use super::*; + + use arrow::array::{ArrayRef, RecordBatch}; + use arrow_schema::SortOptions; + use datafusion_common::Result; + use datafusion_execution::TaskContext; + use datafusion_physical_expr::{PhysicalSortExpr, expressions::col}; + use datafusion_physical_expr_common::metrics::{ + ExecutionPlanMetricsSet, SpillMetrics, + }; + + #[tokio::test] + async fn test_sort_merge_fetch_zero_with_only_1_stream() { + test_fetch_0_should_output_0_rows(1, 0).await.unwrap(); + } + #[tokio::test] + async fn test_sort_merge_fetch_zero_with_2_streams() { + test_fetch_0_should_output_0_rows(2, 0).await.unwrap(); + } + #[tokio::test] + async fn test_sort_merge_fetch_zero_with_only_1_spill_file() { + test_fetch_0_should_output_0_rows(0, 1).await.unwrap(); + } + #[tokio::test] + async fn test_sort_merge_fetch_zero_with_2_spill_files() { + test_fetch_0_should_output_0_rows(0, 2).await.unwrap(); + } + #[tokio::test] + async fn test_sort_merge_fetch_zero_with_1_stream_and_1_spill_file() { + test_fetch_0_should_output_0_rows(1, 1).await.unwrap(); + } + + async fn test_fetch_0_should_output_0_rows( + number_of_streams: usize, + number_of_spilled_files: usize, + ) -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 7, 9, 3])); + let b: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])); + let batch = RecordBatch::try_from_iter(vec![("a", a), ("b", b)]).unwrap(); + let schema = batch.schema(); + + let sort: LexOrdering = [PhysicalSortExpr { + expr: col("b", &schema).unwrap(), + options: SortOptions { + descending: false, + nulls_first: true, + }, + }] + .into(); + + let streams = (0..number_of_streams) + .map(|_| { + Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&schema), + futures::stream::iter(vec![Ok(batch.clone())]), + )) as SendableRecordBatchStream + }) + .collect::>(); + + let spill_manager = SpillManager::new( + task_ctx.runtime_env(), + SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + Arc::clone(&schema), + ); + + let mut sorted_spill_files: Vec = vec![]; + + for _ in 0..number_of_spilled_files { + let file = spill_manager + .spill_record_batch_and_finish(std::slice::from_ref(&batch), "spill") + .unwrap() + .unwrap(); + sorted_spill_files.push(SortedSpillFile { + file, + max_record_batch_memory: batch.get_array_memory_size(), + }); + } + + let sorted_output_stream = StreamingMergeBuilder::new() + .with_batch_size(100) + .with_metrics(BaselineMetrics::new(&ExecutionPlanMetricsSet::new(), 0)) + // Just to avoid having to provide memory pool + .with_bypass_mempool() + .with_schema(schema) + .with_streams(streams) + .with_sorted_spill_files(sorted_spill_files) + .with_spill_manager(spill_manager) + .with_expressions(&sort) + // The whole point of the test - fetch is 0 + .with_fetch(Some(0)) + .build() + .unwrap(); + + let collected = collect(sorted_output_stream).await.unwrap(); + let total: usize = collected.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, 0, "fetch=Some(0) must emit zero rows, got {total}"); + + Ok(()) + } +} From e8a65f2d6f24afa75192fc63aca38011ede88800 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 27 Jul 2026 12:28:42 +0800 Subject: [PATCH 038/109] feat: add BuildHasher variants for hash_utils (#21820) ## Which issue does this PR close? - Closes https://github.com/apache/datafusion/issues/21428. ## Rationale for this change This PR adds `BuildHasher`-based variants for `hash_utils` so callers can compute row hashes with a caller-provided hash builder instead of always using DataFusion's default `RandomState`. The main constraint is performance: `with_hashes` is a hot path, especially for string, dictionary, and nested array hashing. A previous version in #21429 caused measurable regressions in the default `RandomState` path, for example `large_utf8: single, no nulls` regressed from roughly `26.7us` to `36.3us`, and `large_utf8: multiple, no nulls` from roughly `112us` to `127us`. This version keeps the default path performance-oriented by avoiding a fully generic `BuildHasher` rewrite of the existing hot loops. ## What changes are included in this PR? This PR adds: - `with_hashes_with_hasher` - `create_hashes_with_hasher` - custom-hasher implementations for primitive, string, binary, byte-view, dictionary, and nested arrays - tests covering custom hashers, multi-column hashing, and dictionary equivalence The implementation intentionally uses a hybrid design: - Default `RandomState` leaf hot paths remain specialized. - Custom `BuildHasher` leaf paths live separately in `hash_utils/build_hasher.rs`. - Nested/structural logic is shared through an internal child-hashing adapter, so struct/list/map/union/run/dictionary behavior does not need to be broadly duplicated. The trade-off is that there is still some duplication for primitive/string/binary leaf loops. That duplication is intentional: those are the hottest loops, and keeping them separate prevents the existing `RandomState` path from becoming generic over `BuildHasher` or being perturbed by the custom-hasher implementation. ## Are these changes tested? Yes. --------- Co-authored-by: Dmitrii Blaginin Co-authored-by: Andrew Lamb --- datafusion/common/src/hash_utils.rs | 494 +++++++++++++++--- .../common/src/hash_utils/build_hasher.rs | 494 ++++++++++++++++++ 2 files changed, 911 insertions(+), 77 deletions(-) create mode 100644 datafusion/common/src/hash_utils/build_hasher.rs diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index 1443b6152b5ac..cfe57999689b1 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -91,6 +91,8 @@ use crate::error::Result; use crate::error::{_internal_datafusion_err, _internal_err}; use std::cell::RefCell; +mod build_hasher; + // Combines two hashes into one hash #[inline] pub fn combine_hashes(l: u64, r: u64) -> u64 { @@ -186,13 +188,32 @@ where }).map_err(|_| _internal_datafusion_err!("with_hashes cannot access thread-local storage during or after thread destruction"))? } +/// Creates hashes for the given arrays using a thread-local buffer and a custom +/// hash builder, then calls the provided callback with the computed hashes. +/// +/// Hash compatibility with [`with_hashes`] follows the rules documented on +/// [`create_hashes_with_hasher`]. +pub fn with_hashes_with_hasher( + arrays: I, + hash_builder: &S, + callback: F, +) -> Result +where + I: IntoIterator, + T: AsDynArray, + F: FnOnce(&[u64]) -> Result, + S: BuildHasher, +{ + build_hasher::with_hashes_with_hasher(arrays, hash_builder, callback) +} + #[cfg(not(feature = "force_hash_collisions"))] fn hash_null( random_state: &S, hashes_buffer: &'_ mut [u64], - mul_col: bool, + multi_col: bool, ) { - if mul_col { + if multi_col { hashes_buffer.iter_mut().for_each(|hash| { // stable hash for null value *hash = combine_hashes(random_state.hash_one(1), *hash); @@ -254,6 +275,30 @@ macro_rules! hash_float_value { } hash_float_value!((half::f16, u16), (f32, u32), (f64, u64)); +#[cfg(not(feature = "force_hash_collisions"))] +trait ChildHashing { + fn create_hashes(&self, arrays: I, hashes_buffer: &mut [u64]) -> Result<()> + where + I: IntoIterator, + T: AsDynArray; +} + +#[cfg(not(feature = "force_hash_collisions"))] +struct HashStateChildHashing<'a, S> { + hash_state: &'a S, +} + +#[cfg(not(feature = "force_hash_collisions"))] +impl ChildHashing for HashStateChildHashing<'_, S> { + fn create_hashes(&self, arrays: I, hashes_buffer: &mut [u64]) -> Result<()> + where + I: IntoIterator, + T: AsDynArray, + { + create_hashes(arrays, self.hash_state, hashes_buffer).map(|_| ()) + } +} + /// Builds hash values of PrimitiveArray and writes them into `hashes_buffer` /// If `rehash==true` this folds the existing hash into the hasher state /// and hashes only the new value (avoiding a separate combine step). @@ -472,31 +517,25 @@ fn hash_generic_byte_view_array( } } -/// Hash dictionary array with compile-time specialization for null handling. +/// Scatter precomputed dictionary value hashes to key positions. /// -/// Uses const generics to eliminate runtim branching in the hot loop: +/// Uses const generics to eliminate runtime branching in the hot loop: /// - `HAS_NULL_KEYS`: Whether to check for null dictionary keys /// - `HAS_NULL_VALUES`: Whether to check for null dictionary values /// - `MULTI_COL`: Whether to combine with existing hash (true) or initialize (false) #[cfg(not(feature = "force_hash_collisions"))] #[inline(never)] -fn hash_dictionary_inner< +fn hash_dictionary_scatter< K: ArrowDictionaryKeyType, const HAS_NULL_KEYS: bool, const HAS_NULL_VALUES: bool, const MULTI_COL: bool, >( array: &DictionaryArray, - random_state: &impl HashState, + dict_hashes: &[u64], hashes_buffer: &mut [u64], -) -> Result<()> { - // Hash each dictionary value once, and then use that computed - // hash for each key value to avoid a potentially expensive - // redundant hashing for large dictionary elements (e.g. strings) +) { let dict_values = array.values(); - let mut dict_hashes = vec![0; dict_values.len()]; - create_hashes([dict_values], random_state, &mut dict_hashes)?; - if HAS_NULL_KEYS { for (hash, key) in hashes_buffer.iter_mut().zip(array.keys().iter()) { if let Some(key) = key { @@ -522,70 +561,98 @@ fn hash_dictionary_inner< } } } - Ok(()) } -/// Hash the values in a dictionary array #[cfg(not(feature = "force_hash_collisions"))] -fn hash_dictionary( +fn dispatch_dictionary_scatter( array: &DictionaryArray, - random_state: &impl HashState, + dict_hashes: &[u64], hashes_buffer: &mut [u64], multi_col: bool, -) -> Result<()> { +) { let has_null_keys = array.keys().null_count() != 0; let has_null_values = array.values().null_count() != 0; - // Dispatcher based on null presence and multi-column mode - // Should reduce branching within hot loops match (has_null_keys, has_null_values, multi_col) { - (false, false, false) => hash_dictionary_inner::( + (false, false, false) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (false, false, true) => hash_dictionary_inner::( + (false, false, true) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (false, true, false) => hash_dictionary_inner::( + (false, true, false) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (false, true, true) => hash_dictionary_inner::( + (false, true, true) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (true, false, false) => hash_dictionary_inner::( + (true, false, false) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (true, false, true) => hash_dictionary_inner::( + (true, false, true) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (true, true, false) => hash_dictionary_inner::( + (true, true, false) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), - (true, true, true) => hash_dictionary_inner::( + (true, true, true) => hash_dictionary_scatter::( array, - random_state, + dict_hashes, hashes_buffer, ), } } +/// Hash the values in a dictionary array. +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_dictionary( + array: &DictionaryArray, + random_state: &impl HashState, + hashes_buffer: &mut [u64], + multi_col: bool, +) -> Result<()> { + // Hash each dictionary value once, and then use that computed + // hash for each key value to avoid a potentially expensive + // redundant hashing for large dictionary elements (e.g. strings) + let dict_values = array.values(); + let mut dict_hashes = vec![0; dict_values.len()]; + create_hashes([dict_values], random_state, &mut dict_hashes)?; + dispatch_dictionary_scatter(array, &dict_hashes, hashes_buffer, multi_col); + Ok(()) +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_dictionary_with_child_hashing( + array: &DictionaryArray, + child_hashing: &impl ChildHashing, + hashes_buffer: &mut [u64], + multi_col: bool, +) -> Result<()> { + let dict_values = array.values(); + let mut dict_hashes = vec![0; dict_values.len()]; + child_hashing.create_hashes([dict_values], &mut dict_hashes)?; + dispatch_dictionary_scatter(array, &dict_hashes, hashes_buffer, multi_col); + Ok(()) +} + #[cfg(not(feature = "force_hash_collisions"))] fn hash_struct_array( array: &StructArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { let nulls = array.nulls(); @@ -593,7 +660,7 @@ fn hash_struct_array( // Create hashes for each row that combines the hashes over all the column at that row. let mut values_hashes = vec![0u64; row_len]; - create_hashes(array.columns(), random_state, &mut values_hashes)?; + child_hashing.create_hashes(array.columns(), &mut values_hashes)?; // Separate paths to avoid allocating Vec when there are no nulls if let Some(nulls) = nulls { @@ -615,7 +682,7 @@ fn hash_struct_array( #[cfg(not(feature = "force_hash_collisions"))] fn hash_map_array( array: &MapArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { let nulls = array.nulls(); @@ -634,7 +701,7 @@ fn hash_map_array( .iter() .map(|col| col.slice(first_offset, entries_len)) .collect(); - create_hashes(&sliced_columns, random_state, &mut values_hashes)?; + child_hashing.create_hashes(&sliced_columns, &mut values_hashes)?; // Combine the hashes for entries on each row with each other and previous hash for that row // Adjust indices by first_offset since values_hashes is sliced starting from first_offset @@ -666,7 +733,7 @@ fn hash_map_array( #[cfg(not(feature = "force_hash_collisions"))] fn hash_list_array( array: &GenericListArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> where @@ -677,11 +744,10 @@ where let last_offset = array.value_offsets().last().cloned().unwrap_or_default(); let value_bytes_len = (last_offset - first_offset).as_usize(); let mut values_hashes = vec![0u64; value_bytes_len]; - create_hashes( + child_hashing.create_hashes( [array .values() .slice(first_offset.as_usize(), value_bytes_len)], - random_state, &mut values_hashes, )?; @@ -717,7 +783,7 @@ where #[cfg(not(feature = "force_hash_collisions"))] fn hash_list_view_array( array: &GenericListViewArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> where @@ -728,7 +794,7 @@ where let sizes = array.value_sizes(); let nulls = array.nulls(); let mut values_hashes = vec![0u64; values.len()]; - create_hashes([values], random_state, &mut values_hashes)?; + child_hashing.create_hashes([values], &mut values_hashes)?; if let Some(nulls) = nulls { for (i, (offset, size)) in offsets.iter().zip(sizes.iter()).enumerate() { if nulls.is_valid(i) { @@ -756,7 +822,7 @@ where #[cfg(not(feature = "force_hash_collisions"))] fn hash_union_array( array: &UnionArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { let DataType::Union(union_fields, _mode) = array.data_type() else { @@ -766,12 +832,12 @@ fn hash_union_array( if array.is_dense() { // Dense union: children only contain values of their type, so they're already compact. // Use the default hashing approach which is efficient for dense unions. - hash_union_array_default(array, union_fields, random_state, hashes_buffer) + hash_union_array_default(array, union_fields, child_hashing, hashes_buffer) } else { // Sparse union: each child has the same length as the union array. // Optimization: only hash the elements that are actually referenced by type_ids, // instead of hashing all K*N elements (where K = num types, N = array length). - hash_sparse_union_array(array, union_fields, random_state, hashes_buffer) + hash_sparse_union_array(array, union_fields, child_hashing, hashes_buffer) } } @@ -788,7 +854,7 @@ fn hash_union_array( fn hash_union_array_default( array: &UnionArray, union_fields: &UnionFields, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { let mut child_hashes: HashMap> = @@ -798,7 +864,7 @@ fn hash_union_array_default( for (type_id, _field) in union_fields.iter() { let child = array.child(type_id); let mut child_hash_buffer = vec![0; child.len()]; - create_hashes([child], random_state, &mut child_hash_buffer)?; + child_hashing.create_hashes([child], &mut child_hash_buffer)?; child_hashes.insert(type_id, child_hash_buffer); } @@ -829,7 +895,7 @@ fn hash_union_array_default( fn hash_sparse_union_array( array: &UnionArray, union_fields: &UnionFields, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { use std::collections::HashMap; @@ -840,7 +906,7 @@ fn hash_sparse_union_array( return hash_union_array_default( array, union_fields, - random_state, + child_hashing, hashes_buffer, ); } @@ -868,7 +934,7 @@ fn hash_sparse_union_array( // Hash the filtered array let mut filtered_hashes = vec![0u64; filtered.len()]; - create_hashes([&filtered], random_state, &mut filtered_hashes)?; + child_hashing.create_hashes([&filtered], &mut filtered_hashes)?; // Scatter hashes back to correct positions for (hash, &idx) in filtered_hashes.iter().zip(indices.iter()) { @@ -884,14 +950,14 @@ fn hash_sparse_union_array( #[cfg(not(feature = "force_hash_collisions"))] fn hash_fixed_list_array( array: &FixedSizeListArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], ) -> Result<()> { let values = array.values(); let value_length = array.value_length() as usize; let nulls = array.nulls(); let mut values_hashes = vec![0u64; values.len()]; - create_hashes([values], random_state, &mut values_hashes)?; + child_hashing.create_hashes([values], &mut values_hashes)?; if let Some(nulls) = nulls { for i in 0..array.len() { if nulls.is_valid(i) { @@ -919,11 +985,12 @@ fn hash_fixed_list_array( #[cfg(not(feature = "force_hash_collisions"))] fn hash_run_array_inner< R: RunEndIndexType, + C: ChildHashing + ?Sized, const HAS_NULL_VALUES: bool, const REHASH: bool, >( array: &RunArray, - random_state: &impl HashState, + child_hashing: &C, hashes_buffer: &mut [u64], ) -> Result<()> { // We find the relevant runs that cover potentially sliced arrays, so we can only hash those @@ -950,11 +1017,8 @@ fn hash_run_array_inner< end_physical_index - start_physical_index, ); let mut values_hashes = vec![0u64; sliced_values.len()]; - create_hashes( - std::slice::from_ref(&sliced_values), - random_state, - &mut values_hashes, - )?; + child_hashing + .create_hashes(std::slice::from_ref(&sliced_values), &mut values_hashes)?; let mut start_in_slice = 0; for (adjusted_physical_index, &absolute_run_end) in run_ends_values @@ -990,24 +1054,26 @@ fn hash_run_array_inner< #[cfg(not(feature = "force_hash_collisions"))] fn hash_run_array( array: &RunArray, - random_state: &impl HashState, + child_hashing: &impl ChildHashing, hashes_buffer: &mut [u64], rehash: bool, ) -> Result<()> { let has_null_values = array.values().null_count() != 0; match (has_null_values, rehash) { - (false, false) => { - hash_run_array_inner::(array, random_state, hashes_buffer) - } + (false, false) => hash_run_array_inner::( + array, + child_hashing, + hashes_buffer, + ), (false, true) => { - hash_run_array_inner::(array, random_state, hashes_buffer) + hash_run_array_inner::(array, child_hashing, hashes_buffer) } (true, false) => { - hash_run_array_inner::(array, random_state, hashes_buffer) + hash_run_array_inner::(array, child_hashing, hashes_buffer) } (true, true) => { - hash_run_array_inner::(array, random_state, hashes_buffer) + hash_run_array_inner::(array, child_hashing, hashes_buffer) } } } @@ -1041,38 +1107,67 @@ fn hash_single_array( } DataType::Struct(_) => { let array = as_struct_array(array)?; - hash_struct_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_struct_array(array, &child_hashing, hashes_buffer)?; } DataType::List(_) => { let array = as_list_array(array)?; - hash_list_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_list_array(array, &child_hashing, hashes_buffer)?; } DataType::LargeList(_) => { let array = as_large_list_array(array)?; - hash_list_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_list_array(array, &child_hashing, hashes_buffer)?; } DataType::ListView(_) => { let array = as_list_view_array(array)?; - hash_list_view_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_list_view_array(array, &child_hashing, hashes_buffer)?; } DataType::LargeListView(_) => { let array = as_large_list_view_array(array)?; - hash_list_view_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_list_view_array(array, &child_hashing, hashes_buffer)?; } DataType::Map(_, _) => { let array = as_map_array(array)?; - hash_map_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_map_array(array, &child_hashing, hashes_buffer)?; } DataType::FixedSizeList(_,_) => { let array = as_fixed_size_list_array(array)?; - hash_fixed_list_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_fixed_list_array(array, &child_hashing, hashes_buffer)?; } DataType::Union(_, _) => { let array = as_union_array(array)?; - hash_union_array(array, random_state, hashes_buffer)?; + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_union_array(array, &child_hashing, hashes_buffer)?; } DataType::RunEndEncoded(_, _) => downcast_run_array! { - array => hash_run_array(array, random_state, hashes_buffer, rehash)?, + array => { + let child_hashing = HashStateChildHashing { + hash_state: random_state, + }; + hash_run_array(array, &child_hashing, hashes_buffer, rehash)? + }, _ => unreachable!() } _ => { @@ -1158,8 +1253,36 @@ where Ok(hashes_buffer) } +/// Creates hash values for every row using a caller-provided hash builder. +/// +/// The number of rows to hash is determined by `hashes_buffer.len()`. +/// `hashes_buffer` should be pre-sized appropriately. +/// +/// # Hash compatibility +/// +/// Hash values are not guaranteed to be bit-for-bit identical to those from +/// [`create_hashes`], even when `hash_builder` also implements [`HashState`]. +/// The optimized [`HashState`] path seeds the hasher from the previous hash +/// when rehashing some primitive and byte-view values, whereas this function +/// combines independently computed hashes. Use one API consistently if hashes +/// are persisted or exchanged. +pub fn create_hashes_with_hasher<'a, I, T, S>( + arrays: I, + hash_builder: &S, + hashes_buffer: &'a mut [u64], +) -> Result<&'a mut [u64]> +where + I: IntoIterator, + T: AsDynArray, + S: BuildHasher, +{ + build_hasher::create_hashes_with_hasher(arrays, hash_builder, hashes_buffer) +} + #[cfg(test)] mod tests { + #[cfg(not(feature = "force_hash_collisions"))] + use std::hash::{BuildHasherDefault, Hasher}; use std::sync::Arc; use arrow::array::*; @@ -1168,6 +1291,23 @@ mod tests { use super::*; + #[cfg(not(feature = "force_hash_collisions"))] + #[derive(Default)] + struct TestHasher(u64); + + #[cfg(not(feature = "force_hash_collisions"))] + impl Hasher for TestHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, bytes: &[u8]) { + for byte in bytes { + self.0 = self.0.wrapping_mul(37).wrapping_add(u64::from(*byte)); + } + } + } + #[test] fn create_hashes_for_decimal_array() -> Result<()> { let array = vec![1, 2, 3, 4] @@ -1404,6 +1544,206 @@ mod tests { Ok(()) } + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_create_hashes_with_custom_hasher() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 1, 4])); + let hash_builder = BuildHasherDefault::::default(); + + let mut custom_hashes = vec![0; array.len()]; + create_hashes_with_hasher([&array], &hash_builder, &mut custom_hashes).unwrap(); + + let random_state = RandomState::with_seed(0); + let mut default_hashes = vec![0; array.len()]; + create_hashes([&array], &random_state, &mut default_hashes).unwrap(); + + assert_eq!(custom_hashes[0], custom_hashes[2]); + assert_ne!(custom_hashes[0], custom_hashes[1]); + assert_ne!(custom_hashes, default_hashes); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_create_hashes_with_custom_hasher_normalizes_negative_zero() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![0.0, -0.0])); + let hash_builder = BuildHasherDefault::::default(); + let mut hashes = vec![0; array.len()]; + + create_hashes_with_hasher([&array], &hash_builder, &mut hashes).unwrap(); + + assert_eq!(hashes[0], hashes[1]); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_create_hashes_dictionary_with_custom_hasher() { + let strings = [Some("foo"), None, Some("bar"), Some("foo"), None]; + let string_array: ArrayRef = + Arc::new(strings.iter().cloned().collect::()); + let dict_array: ArrayRef = Arc::new( + strings + .iter() + .cloned() + .collect::>(), + ); + let hash_builder = BuildHasherDefault::::default(); + + let mut string_hashes = vec![0; strings.len()]; + create_hashes_with_hasher([&string_array], &hash_builder, &mut string_hashes) + .unwrap(); + + let mut dict_hashes = vec![0; strings.len()]; + create_hashes_with_hasher([&dict_array], &hash_builder, &mut dict_hashes) + .unwrap(); + + assert_eq!(string_hashes, dict_hashes); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_create_hashes_struct_with_custom_hasher() { + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("int", DataType::Int32, false)), + Arc::new(Int32Array::from(vec![1, 2, 1, 3])) as ArrayRef, + ), + ( + Arc::new(Field::new("string", DataType::Utf8, false)), + Arc::new(StringArray::from(vec!["alpha", "beta", "alpha", "alpha"])) + as ArrayRef, + ), + ]); + let hash_builder = BuildHasherDefault::::default(); + + let mut child_hashes = vec![0; struct_array.len()]; + create_hashes_with_hasher( + struct_array.columns(), + &hash_builder, + &mut child_hashes, + ) + .unwrap(); + let expected_hashes = child_hashes + .into_iter() + .map(|hash| combine_hashes(0, hash)) + .collect::>(); + + let array: ArrayRef = Arc::new(struct_array); + let mut actual_hashes = vec![0; array.len()]; + create_hashes_with_hasher([&array], &hash_builder, &mut actual_hashes).unwrap(); + + assert_eq!(actual_hashes, expected_hashes); + assert_eq!(actual_hashes[0], actual_hashes[2]); + assert_ne!(actual_hashes[0], actual_hashes[3]); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_create_hashes_long_utf8_view_with_custom_hasher() { + let values = vec![ + Some("this string is longer than twelve bytes"), + None, + Some("another string longer than twelve bytes"), + Some("this string is longer than twelve bytes"), + ]; + let view_array = StringViewArray::from(values.clone()); + assert!(!view_array.data_buffers().is_empty()); + let view_array: ArrayRef = Arc::new(view_array); + let hash_builder = BuildHasherDefault::::default(); + + let mut view_hashes = vec![0; view_array.len()]; + create_hashes_with_hasher([&view_array], &hash_builder, &mut view_hashes) + .unwrap(); + let expected_hashes = values + .iter() + .map(|value| { + value + .map(|value| hash_builder.hash_one(value.as_bytes())) + .unwrap_or_default() + }) + .collect::>(); + assert_eq!(view_hashes, expected_hashes); + + let prefix_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 1])); + let mut expected_hashes = vec![0; prefix_array.len()]; + create_hashes_with_hasher([&prefix_array], &hash_builder, &mut expected_hashes) + .unwrap(); + for (hash, value) in expected_hashes.iter_mut().zip(&values) { + if let Some(value) = value { + *hash = combine_hashes(hash_builder.hash_one(value.as_bytes()), *hash); + } + } + + let mut view_hashes = vec![0; view_array.len()]; + create_hashes_with_hasher( + [&prefix_array, &view_array], + &hash_builder, + &mut view_hashes, + ) + .unwrap(); + assert_eq!(view_hashes, expected_hashes); + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_single_column_leaf_hashes_match_with_same_hasher() { + let arrays: Vec = vec![ + Arc::new(Int32Array::from(vec![Some(1), None, Some(-1)])), + Arc::new(Float64Array::from(vec![Some(0.0), Some(-0.0), None])), + Arc::new(StringArray::from(vec![Some("foo"), None, Some("bar")])), + Arc::new(BinaryArray::from(vec![ + Some(&b"short"[..]), + None, + Some(&b"longer than twelve bytes"[..]), + ])), + Arc::new(StringViewArray::from(vec![ + Some("short"), + None, + Some("longer than twelve bytes"), + ])), + ]; + let random_state = RandomState::with_seed(0); + + for array in arrays { + let mut default_hashes = vec![0; array.len()]; + create_hashes([&array], &random_state, &mut default_hashes).unwrap(); + + let mut custom_hashes = vec![0; array.len()]; + create_hashes_with_hasher([&array], &random_state, &mut custom_hashes) + .unwrap(); + + assert_eq!( + custom_hashes, + default_hashes, + "single-column parity failed for {}", + array.data_type() + ); + } + } + + #[test] + #[cfg(not(feature = "force_hash_collisions"))] + fn test_with_hashes_with_custom_hasher() { + let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let str_array: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let hash_builder = BuildHasherDefault::::default(); + + let mut expected_hashes = vec![0; int_array.len()]; + create_hashes_with_hasher( + [&int_array, &str_array], + &hash_builder, + &mut expected_hashes, + ) + .unwrap(); + + let actual_hashes = + with_hashes_with_hasher([&int_array, &str_array], &hash_builder, |hashes| { + Ok(hashes.to_vec()) + }) + .unwrap(); + + assert_eq!(actual_hashes, expected_hashes); + } + #[test] // Tests actual values of hashes, which are different if forcing collisions #[cfg(not(feature = "force_hash_collisions"))] diff --git a/datafusion/common/src/hash_utils/build_hasher.rs b/datafusion/common/src/hash_utils/build_hasher.rs new file mode 100644 index 0000000000000..12258beb11403 --- /dev/null +++ b/datafusion/common/src/hash_utils/build_hasher.rs @@ -0,0 +1,494 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{AsDynArray, HASH_BUFFER, MAX_BUFFER_SIZE}; +#[cfg(not(feature = "force_hash_collisions"))] +use super::{ + ChildHashing, combine_hashes, hash_dictionary_with_child_hashing, + hash_fixed_list_array, hash_list_array, hash_list_view_array, hash_map_array, + hash_run_array, hash_struct_array, hash_union_array, +}; +#[cfg(not(feature = "force_hash_collisions"))] +use crate::cast::{ + as_binary_view_array, as_boolean_array, as_fixed_size_list_array, + as_generic_binary_array, as_large_list_array, as_large_list_view_array, + as_list_array, as_list_view_array, as_map_array, as_string_array, + as_string_view_array, as_struct_array, as_union_array, +}; +use crate::error::Result; +use crate::error::{_internal_datafusion_err, _internal_err}; +#[cfg(feature = "force_hash_collisions")] +use arrow::array::Array; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano}; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::array::*; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::datatypes::*; +#[cfg(not(feature = "force_hash_collisions"))] +use arrow::{downcast_dictionary_array, downcast_primitive_array}; +use std::hash::BuildHasher; + +pub(super) fn with_hashes_with_hasher( + arrays: I, + hash_builder: &S, + callback: F, +) -> Result +where + I: IntoIterator, + T: AsDynArray, + F: FnOnce(&[u64]) -> Result, + S: BuildHasher, +{ + let mut iter = arrays.into_iter().peekable(); + + let required_size = match iter.peek() { + Some(arr) => arr.as_dyn_array().len(), + None => { + return _internal_err!("with_hashes_with_hasher requires at least one array"); + } + }; + + HASH_BUFFER.try_with(|cell| { + let mut buffer = cell.try_borrow_mut().map_err(|_| { + _internal_datafusion_err!( + "with_hashes_with_hasher cannot be called reentrantly on the same thread" + ) + })?; + + buffer.clear(); + buffer.resize(required_size, 0); + + create_hashes_with_hasher_impl(iter, hash_builder, &mut buffer[..required_size])?; + + let result = callback(&buffer[..required_size])?; + + if buffer.capacity() > MAX_BUFFER_SIZE { + buffer.truncate(MAX_BUFFER_SIZE); + buffer.shrink_to_fit(); + } + + Ok(result) + }).map_err(|_| { + _internal_datafusion_err!( + "with_hashes_with_hasher cannot access thread-local storage during or after thread destruction" + ) + })? +} + +pub(super) fn create_hashes_with_hasher<'a, I, T, S>( + arrays: I, + hash_builder: &S, + hashes_buffer: &'a mut [u64], +) -> Result<&'a mut [u64]> +where + I: IntoIterator, + T: AsDynArray, + S: BuildHasher, +{ + create_hashes_with_hasher_impl(arrays, hash_builder, hashes_buffer) +} + +fn create_hashes_with_hasher_impl<'a, I, T, S>( + arrays: I, + hash_builder: &S, + hashes_buffer: &'a mut [u64], +) -> Result<&'a mut [u64]> +where + I: IntoIterator, + T: AsDynArray, + S: BuildHasher, +{ + for (i, array) in arrays.into_iter().enumerate() { + let rehash = i >= 1; + hash_single_array_with_hasher( + array.as_dyn_array(), + hash_builder, + hashes_buffer, + rehash, + )?; + } + Ok(hashes_buffer) +} + +#[cfg(not(feature = "force_hash_collisions"))] +struct BuildHasherChildHashing<'a, S> { + hash_builder: &'a S, +} + +#[cfg(not(feature = "force_hash_collisions"))] +impl ChildHashing for BuildHasherChildHashing<'_, S> { + fn create_hashes(&self, arrays: I, hashes_buffer: &mut [u64]) -> Result<()> + where + I: IntoIterator, + T: AsDynArray, + { + create_hashes_with_hasher_impl(arrays, self.hash_builder, hashes_buffer) + .map(|_| ()) + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +trait BuildHasherHashValue { + fn hash_one_with_hasher(&self, state: &S) -> u64; +} + +#[cfg(not(feature = "force_hash_collisions"))] +impl BuildHasherHashValue for &T { + fn hash_one_with_hasher(&self, state: &S) -> u64 { + T::hash_one_with_hasher(self, state) + } +} + +macro_rules! build_hasher_hash_value { + ($($t:ty),+) => { + $(#[cfg(not(feature = "force_hash_collisions"))] + impl BuildHasherHashValue for $t { + fn hash_one_with_hasher(&self, state: &S) -> u64 { + state.hash_one(self) + } + })+ + }; +} +build_hasher_hash_value!(i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128); +build_hasher_hash_value!(bool, str, [u8], IntervalDayTime, IntervalMonthDayNano); + +macro_rules! build_hasher_hash_float_value { + ($(($t:ty, $i:ty)),+) => { + $(#[cfg(not(feature = "force_hash_collisions"))] + impl BuildHasherHashValue for $t { + fn hash_one_with_hasher(&self, state: &S) -> u64 { + let bits = <$i>::from_ne_bytes(self.to_ne_bytes()); + let bits = if bits << 1 == 0 { 0 } else { bits }; + state.hash_one(bits) + } + })+ + }; +} +build_hasher_hash_float_value!((half::f16, u16), (f32, u32), (f64, u64)); + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_null_with_hasher( + hash_builder: &S, + hashes_buffer: &mut [u64], + multi_col: bool, +) { + if hashes_buffer.is_empty() { + return; + } + + let null_hash = hash_builder.hash_one(1); + if multi_col { + hashes_buffer.iter_mut().for_each(|hash| { + *hash = combine_hashes(null_hash, *hash); + }) + } else { + hashes_buffer.fill(null_hash); + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_array_primitive_with_hasher( + array: &PrimitiveArray, + hash_builder: &S, + hashes_buffer: &mut [u64], + rehash: bool, +) where + T: ArrowPrimitiveType, + S: BuildHasher, +{ + assert_eq!( + hashes_buffer.len(), + array.len(), + "hashes_buffer and array should be of equal length" + ); + + if array.null_count() == 0 { + if rehash { + for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) { + *hash = combine_hashes(value.hash_one_with_hasher(hash_builder), *hash); + } + } else { + for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) { + *hash = value.hash_one_with_hasher(hash_builder); + } + } + } else if rehash { + for i in array.nulls().unwrap().valid_indices() { + let value = unsafe { array.value_unchecked(i) }; + hashes_buffer[i] = combine_hashes( + value.hash_one_with_hasher(hash_builder), + hashes_buffer[i], + ); + } + } else { + for i in array.nulls().unwrap().valid_indices() { + let value = unsafe { array.value_unchecked(i) }; + hashes_buffer[i] = value.hash_one_with_hasher(hash_builder); + } + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_array_with_hasher( + array: &T, + hash_builder: &S, + hashes_buffer: &mut [u64], + rehash: bool, +) where + T: ArrayAccessor, + T::Item: BuildHasherHashValue, + S: BuildHasher, +{ + assert_eq!( + hashes_buffer.len(), + array.len(), + "hashes_buffer and array should be of equal length" + ); + + if array.null_count() == 0 { + if rehash { + for (i, hash) in hashes_buffer.iter_mut().enumerate() { + let value = unsafe { array.value_unchecked(i) }; + *hash = combine_hashes(value.hash_one_with_hasher(hash_builder), *hash); + } + } else { + for (i, hash) in hashes_buffer.iter_mut().enumerate() { + let value = unsafe { array.value_unchecked(i) }; + *hash = value.hash_one_with_hasher(hash_builder); + } + } + } else if rehash { + for i in array.nulls().unwrap().valid_indices() { + let value = unsafe { array.value_unchecked(i) }; + hashes_buffer[i] = combine_hashes( + value.hash_one_with_hasher(hash_builder), + hashes_buffer[i], + ); + } + } else { + for i in array.nulls().unwrap().valid_indices() { + let value = unsafe { array.value_unchecked(i) }; + hashes_buffer[i] = value.hash_one_with_hasher(hash_builder); + } + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +#[inline(never)] +fn hash_string_view_array_inner_with_hasher< + T: ByteViewType, + S: BuildHasher, + const HAS_NULLS: bool, + const HAS_BUFFERS: bool, + const REHASH: bool, +>( + array: &GenericByteViewArray, + hash_builder: &S, + hashes_buffer: &mut [u64], +) { + assert_eq!( + hashes_buffer.len(), + array.len(), + "hashes_buffer and array should be of equal length" + ); + + let buffers = array.data_buffers(); + let view_bytes = |view_len: u32, view: u128| { + let view = ByteView::from(view); + let offset = view.offset as usize; + unsafe { + let data = buffers.get_unchecked(view.buffer_index as usize); + data.get_unchecked(offset..offset + view_len as usize) + } + }; + + let hashes_and_views = hashes_buffer.iter_mut().zip(array.views().iter()); + for (i, (hash, &v)) in hashes_and_views.enumerate() { + if HAS_NULLS && array.is_null(i) { + continue; + } + let view_len = v as u32; + if !HAS_BUFFERS || view_len <= 12 { + if REHASH { + *hash = combine_hashes(v.hash_one_with_hasher(hash_builder), *hash); + } else { + *hash = v.hash_one_with_hasher(hash_builder); + } + continue; + } + let value = view_bytes(view_len, v); + if REHASH { + *hash = combine_hashes(value.hash_one_with_hasher(hash_builder), *hash); + } else { + *hash = value.hash_one_with_hasher(hash_builder); + } + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_generic_byte_view_array_with_hasher( + array: &GenericByteViewArray, + hash_builder: &S, + hashes_buffer: &mut [u64], + rehash: bool, +) { + match ( + array.null_count() != 0, + !array.data_buffers().is_empty(), + rehash, + ) { + (false, false, false) => { + for (hash, &view) in hashes_buffer.iter_mut().zip(array.views().iter()) { + *hash = view.hash_one_with_hasher(hash_builder); + } + } + (false, false, true) => { + for (hash, &view) in hashes_buffer.iter_mut().zip(array.views().iter()) { + *hash = combine_hashes(view.hash_one_with_hasher(hash_builder), *hash); + } + } + (false, true, false) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + (false, true, true) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + (true, false, false) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + (true, false, true) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + (true, true, false) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + (true, true, true) => { + hash_string_view_array_inner_with_hasher::( + array, + hash_builder, + hashes_buffer, + ) + } + } +} + +#[cfg(not(feature = "force_hash_collisions"))] +fn hash_single_array_with_hasher( + array: &dyn Array, + hash_builder: &S, + hashes_buffer: &mut [u64], + rehash: bool, +) -> Result<()> { + let child_hashing = BuildHasherChildHashing { hash_builder }; + + downcast_primitive_array! { + array => hash_array_primitive_with_hasher(array, hash_builder, hashes_buffer, rehash), + DataType::Null => hash_null_with_hasher(hash_builder, hashes_buffer, rehash), + DataType::Boolean => hash_array_with_hasher(&as_boolean_array(array)?, hash_builder, hashes_buffer, rehash), + DataType::Utf8 => hash_array_with_hasher(&as_string_array(array)?, hash_builder, hashes_buffer, rehash), + DataType::Utf8View => hash_generic_byte_view_array_with_hasher(as_string_view_array(array)?, hash_builder, hashes_buffer, rehash), + DataType::LargeUtf8 => hash_array_with_hasher(&as_largestring_array(array), hash_builder, hashes_buffer, rehash), + DataType::Binary => hash_array_with_hasher(&as_generic_binary_array::(array)?, hash_builder, hashes_buffer, rehash), + DataType::BinaryView => hash_generic_byte_view_array_with_hasher(as_binary_view_array(array)?, hash_builder, hashes_buffer, rehash), + DataType::LargeBinary => hash_array_with_hasher(&as_generic_binary_array::(array)?, hash_builder, hashes_buffer, rehash), + DataType::FixedSizeBinary(_) => { + let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap(); + hash_array_with_hasher(&array, hash_builder, hashes_buffer, rehash) + } + DataType::Dictionary(_, _) => downcast_dictionary_array! { + array => hash_dictionary_with_child_hashing(array, &child_hashing, hashes_buffer, rehash)?, + _ => unreachable!() + } + DataType::Struct(_) => { + let array = as_struct_array(array)?; + hash_struct_array(array, &child_hashing, hashes_buffer)?; + } + DataType::List(_) => { + let array = as_list_array(array)?; + hash_list_array(array, &child_hashing, hashes_buffer)?; + } + DataType::LargeList(_) => { + let array = as_large_list_array(array)?; + hash_list_array(array, &child_hashing, hashes_buffer)?; + } + DataType::ListView(_) => { + let array = as_list_view_array(array)?; + hash_list_view_array(array, &child_hashing, hashes_buffer)?; + } + DataType::LargeListView(_) => { + let array = as_large_list_view_array(array)?; + hash_list_view_array(array, &child_hashing, hashes_buffer)?; + } + DataType::Map(_, _) => { + let array = as_map_array(array)?; + hash_map_array(array, &child_hashing, hashes_buffer)?; + } + DataType::FixedSizeList(_,_) => { + let array = as_fixed_size_list_array(array)?; + hash_fixed_list_array(array, &child_hashing, hashes_buffer)?; + } + DataType::Union(_, _) => { + let array = as_union_array(array)?; + hash_union_array(array, &child_hashing, hashes_buffer)?; + } + DataType::RunEndEncoded(_, _) => downcast_run_array! { + array => hash_run_array(array, &child_hashing, hashes_buffer, rehash)?, + _ => unreachable!() + } + _ => { + return _internal_err!( + "Unsupported data type in hasher: {}", + array.data_type() + ); + } + } + Ok(()) +} + +#[cfg(feature = "force_hash_collisions")] +fn hash_single_array_with_hasher( + _array: &dyn Array, + _hash_builder: &S, + hashes_buffer: &mut [u64], + _rehash: bool, +) -> Result<()> { + for hash in hashes_buffer.iter_mut() { + *hash = 0; + } + Ok(()) +} From 1c3232ce2262523fb4043e7264185b2f8f929b0d Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:14:54 +0300 Subject: [PATCH 039/109] `ScalarUdfImpl::strictly_order_preserving`: Allow expression to report whether they keep the same ordering of the input (#23807) ## Which issue does this PR close? - Closes #23798 Related to: - #16217 ## Rationale for this change To be able to keep the same sorting order allowing for more optimizations Now comet own `cast` implementation can be recognized as not modifying sort order in the same cases that datafusion cast does. ## What changes are included in this PR? added `strictly_order_preserving` property to `ExprProperties` + varius other places and replaced the hard coded logic for cast (`substitute_cast_ordering`) about keeping input order with more generic approach that now any expression can implement and have the same advantage of sort elimination also marked from_unixtime as keeping ordering to show a case of this optimization ## Are these changes tested? yes ## Are there any user-facing changes? yes, breaking change, added `strictly_order_preserving` property to `ExprProperties` and to `FFI_ExprProperties` this property means that given expression `f` and 2 values from the input column `a` and `b` the following variants are kept: 1. `a.cmp(b) == f(a).cmp(f(b))` 2. nulls maps to nulls Example of satisfying expression: `cast(col_a as BIGINT)` where `col_a` is `INT` it is keeping the properties Example of not satisfying: `floor` - floor can not `array_repeat(my_col, 2)` which might look like at first glance as keeping the property as well but in fact it does not. the reason is that `array_repeat(null, 2)` will output list of 2 nulls which breaks the 2nd property that nulls must be kept as nulls how to migrate: Option 1 - keeping the old behavior (safest but least performant) set `strictly_order_preserving` to false Option 2 - Using the new optimization that this opens up: set `strictly_order_preserving` only if the expression keep both variants --- .../physical_optimizer/enforce_sorting.rs | 157 +++++++++++++++++- datafusion/expr-common/src/sort_properties.rs | 65 ++++++++ datafusion/expr/src/udf.rs | 19 +++ datafusion/ffi/src/expr/expr_properties.rs | 3 + .../functions/src/datetime/from_unixtime.rs | 19 +++ datafusion/functions/src/math/monotonicity.rs | 1 + .../physical-expr/src/equivalence/ordering.rs | 41 ++++- .../src/equivalence/properties/mod.rs | 146 ++++++++++------ .../physical-expr/src/expressions/binary.rs | 8 + .../physical-expr/src/expressions/cast.rs | 14 +- .../physical-expr/src/expressions/literal.rs | 2 + .../physical-expr/src/expressions/negative.rs | 2 + .../physical-expr/src/scalar_function.rs | 2 + datafusion/sqllogictest/test_files/order.slt | 98 +++++++++++ 14 files changed, 519 insertions(+), 58 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs index 9338fcc0bce35..d94253a84aa5f 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs @@ -33,7 +33,7 @@ use arrow::compute::{SortOptions}; use arrow::datatypes::{DataType, SchemaRef}; use datafusion_common::config::{ConfigOptions, CsvOptions}; use datafusion_common::tree_node::{TreeNode, TransformedResult}; -use datafusion_common::{create_array, NullEquality, Result, TableReference}; +use datafusion_common::{create_array, DataFusionError, NullEquality, Result, TableReference}; use datafusion_datasource::file_scan_config::FileScanConfigBuilder; use datafusion_datasource::source::DataSourceExec; use datafusion_expr_common::operator::Operator; @@ -49,7 +49,7 @@ use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec}; use datafusion_physical_plan::repartition::RepartitionExec; use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion_physical_plan::sorts::sort::SortExec; -use datafusion_physical_plan::{displayable, get_plan_string, ExecutionPlan}; +use datafusion_physical_plan::{displayable, get_plan_string, ExecutionPlan, ExecutionPlanProperties}; use datafusion::datasource::physical_plan::CsvSource; use datafusion::datasource::listing::PartitionedFile; use datafusion_physical_optimizer::enforce_sorting::{PlanWithCorrespondingCoalescePartitions, PlanWithCorrespondingSort, parallelize_sorts, ensure_sorting}; @@ -60,12 +60,15 @@ use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements; use datafusion_physical_optimizer::output_requirements::OutputRequirementExec; use datafusion_physical_optimizer::PhysicalOptimizerRule; use datafusion::prelude::*; -use arrow::array::{record_batch, ArrayRef, Int32Array, RecordBatch}; +use arrow::array::{record_batch, Array, ArrayRef, Int32Array, RecordBatch}; use arrow::datatypes::{Field}; use arrow_schema::Schema; use datafusion_execution::TaskContext; use datafusion_catalog::streaming::StreamingTable; +use datafusion_expr_common::columnar_value::ColumnarValue; +use datafusion_physical_expr::projection::ProjectionExpr; +use datafusion_physical_plan::projection::ProjectionExec; use futures::StreamExt; use insta::{Settings, assert_snapshot}; @@ -3255,3 +3258,151 @@ async fn test_does_not_push_fetch_sort_through_projection_over_union() -> Result Ok(()) } + +/// A pass-through wrapper around a column: just assert that column does not contain any nulls +#[derive(Debug, Eq)] +struct AssertNotNull { + inner: Arc, +} + +impl AssertNotNull { + fn new(inner: Arc) -> Arc { + Arc::new(Self { inner }) + } +} + +impl PartialEq for AssertNotNull { + fn eq(&self, other: &Self) -> bool { + self.inner.eq(&other.inner) + } +} + +impl std::hash::Hash for AssertNotNull { + fn hash(&self, state: &mut H) { + self.inner.hash(state); + } +} + +impl std::fmt::Display for AssertNotNull { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "assert_not_null({})", self.inner) + } +} + +impl PhysicalExpr for AssertNotNull { + fn data_type(&self, input_schema: &Schema) -> Result { + self.inner.data_type(input_schema) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(false) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let child = self.inner.evaluate(batch)?; + match child { + ColumnarValue::Array(a) if a.logical_null_count() > 0 => Err( + DataFusionError::Internal("AssertNotNull evaluated to null".to_string()), + ), + ColumnarValue::Scalar(s) if s.is_null() => Err(DataFusionError::Internal( + "AssertNotNull evaluated to null".to_string(), + )), + child => Ok(child), + } + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.inner] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(AssertNotNull { + inner: Arc::clone(&children[0]), + })) + } + + fn get_properties( + &self, + children: &[datafusion_expr::sort_properties::ExprProperties], + ) -> Result { + Ok(children[0].clone()) + } + + fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "assert_not_null({})", self.inner) + } +} + +#[tokio::test] +async fn test_passthrough_wrapper_projection_keeps_ordering() -> Result<()> { + fn sort_expr(name: &str, schema: &Schema) -> PhysicalSortExpr { + PhysicalSortExpr { + expr: col(name, schema).unwrap(), + options: Default::default(), + } + } + + pub fn projection_exec( + expr: Vec<(Arc, String)>, + input: Arc, + ) -> Result> { + let proj_exprs: Vec = expr + .into_iter() + .map(|(expr, alias)| ProjectionExpr { expr, alias }) + .collect(); + Ok(Arc::new(ProjectionExec::try_new(proj_exprs, input)?)) + } + + let batch = record_batch!( + ("a", Utf8, ["x", "y"]), + ("b", Utf8, ["1", "2"]), + ("c", Utf8, ["1", "2"]) + )?; + let schema = batch.schema(); + let source = Arc::new(DataSourceExec::new(Arc::new( + datafusion::datasource::memory::MemorySourceConfig::try_new( + &[vec![batch]], + schema.clone(), + None, + )? + .try_with_sort_information(vec![ + LexOrdering::new([ + sort_expr("a", &schema), + sort_expr("b", &schema), + sort_expr("c", &schema), + ]) + .unwrap(), + ])?, + ))) as Arc; + + let projection = projection_exec( + vec![ + (AssertNotNull::new(col("a", &schema)?), "a".to_string()), + (AssertNotNull::new(col("b", &schema)?), "b".to_string()), + (AssertNotNull::new(col("c", &schema)?), "c".to_string()), + ], + source, + )?; + + let ordering = LexOrdering::new([ + sort_expr("a", &projection.schema()), + sort_expr("b", &projection.schema()), + sort_expr("c", &projection.schema()), + ]) + .unwrap(); + + let sort_satisfied = projection + .equivalence_properties() + .ordering_satisfy(ordering.clone())?; + + let plan_str = displayable(projection.as_ref()).indent(true).to_string(); + assert!( + sort_satisfied, + "sort should be satisfied, ordering: {ordering}\nplan:\n{plan_str}" + ); + + Ok(()) +} diff --git a/datafusion/expr-common/src/sort_properties.rs b/datafusion/expr-common/src/sort_properties.rs index 04da574882d30..74d644f79faef 100644 --- a/datafusion/expr-common/src/sort_properties.rs +++ b/datafusion/expr-common/src/sort_properties.rs @@ -141,7 +141,61 @@ pub struct ExprProperties { pub range: Interval, /// Indicates whether the expression preserves lexicographical ordering /// of its inputs. + /// + /// This is a *non-strict* (monotone) property: inputs advancing in + /// lexicographical order never make the output decrease, but distinct + /// inputs may map to equal outputs (ties). See + /// [`Self::strictly_order_preserving`] for the strict variant and an + /// explanation of the difference. pub preserves_lex_ordering: bool, + /// Indicates whether the expression is strictly order-preserving with + /// respect to its inputs that are `Ordered`: the output is ordered in the + /// same direction, equal outputs can only result from equal values of + /// those inputs (i.e. the mapping is one-to-one), and nulls map to nulls. + /// + /// i.e. setting this to true means that `a.cmp(b) == f(a).cmp(f(b))` + /// + /// # Difference from [`Self::preserves_lex_ordering`] + /// + /// The two properties differ in both their premise and their strictness: + /// + /// - `preserves_lex_ordering` assumes the inputs advance in + /// *lexicographical* order (a later input may decrease whenever an + /// earlier one increases), and only promises a non-decreasing output, + /// allowing distinct inputs to collapse into equal outputs; `floor`, + /// `date_trunc` and narrowing casts do exactly that. + /// - `strictly_order_preserving` assumes every `Ordered` input advances + /// *simultaneously* (component-wise, which is what actually holds when + /// all of them are sorted in the data), and promises a strict output: + /// equal outputs only from equal inputs. + /// + /// For an expression with a single ordered input the premises coincide, + /// and this field is simply the stronger claim: it implies + /// `preserves_lex_ordering`. With multiple ordered inputs, neither + /// implies the other: a lexicographical-ordering-preserving expression + /// need not be strict (distinct inputs may still produce equal outputs), + /// while `a + b` over two ordered, overflow-free inputs is strict but not + /// lexicographical (under the lexicographical premise `b` may decrease + /// while `a` increases, making the sum decrease). + /// + /// The distinction matters for suffix sort keys. Optimizers use this + /// field to substitute a sort key with an expression computed from it: + /// if data is sorted by `[x, y]`, it is also sorted by `[expr(x), y]`. + /// That claim requires `y` to be sorted within each run of equal + /// `expr(x)` values, which only holds if equal outputs imply equal `x` + /// values. With a merely monotone expression such as `floor`, one output + /// run can span several `x` groups, and `y` restarts at each group: + /// + /// ```text + /// sorted by [x, y]: (1.2, 5), (1.8, 1), (2.5, 3) + /// [floor(x), y]: (1, 5), (1, 1), (2, 3) <-- y not sorted within + /// the "1" run + /// ``` + /// + /// Hence a monotone expression only justifies the length-1 ordering + /// `[expr(x)]`, while a strictly order-preserving one keeps the entire + /// suffix valid. When in doubt, set to `false`. + pub strictly_order_preserving: bool, } impl ExprProperties { @@ -152,6 +206,7 @@ impl ExprProperties { sort_properties: SortProperties::default(), range: Interval::make_unbounded(&DataType::Null).unwrap(), preserves_lex_ordering: false, + strictly_order_preserving: false, } } @@ -172,4 +227,14 @@ impl ExprProperties { self.preserves_lex_ordering = preserves_lex_ordering; self } + + /// Sets whether the expression is strictly order-preserving and returns + /// the modified instance. + pub fn with_strictly_order_preserving( + mut self, + strictly_order_preserving: bool, + ) -> Self { + self.strictly_order_preserving = strictly_order_preserving; + self + } } diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs index 4c51ff46f7365..2de3be4c10fa4 100644 --- a/datafusion/expr/src/udf.rs +++ b/datafusion/expr/src/udf.rs @@ -380,6 +380,11 @@ impl ScalarUDF { self.inner.preserves_lex_ordering(inputs) } + /// See [`ScalarUDFImpl::strictly_order_preserving`] for more details. + pub fn strictly_order_preserving(&self, inputs: &[ExprProperties]) -> Result { + self.inner.strictly_order_preserving(inputs) + } + /// See [`ScalarUDFImpl::coerce_types`] for more details. pub fn coerce_types(&self, arg_types: &[DataType]) -> Result> { self.inner.coerce_types(arg_types) @@ -979,10 +984,20 @@ pub trait ScalarUDFImpl: Debug + DynEq + DynHash + Send + Sync + Any { /// Returns true if the function preserves lexicographical ordering based on /// the input ordering. + /// + /// See [`ExprProperties::preserves_lex_ordering`] for more details fn preserves_lex_ordering(&self, _inputs: &[ExprProperties]) -> Result { Ok(false) } + /// Returns true if the function is strictly order-preserving with respect + /// to its `Ordered` inputs, i.e. `a.cmp(b) == f(a).cmp(f(b))`. + /// + /// See [`ExprProperties::strictly_order_preserving`] for more details + fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result { + Ok(false) + } + /// Coerce arguments of a function call to types that the function can evaluate. /// /// This function is only called if [`ScalarUDFImpl::signature`] returns @@ -1194,6 +1209,10 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl { self.inner.preserves_lex_ordering(inputs) } + fn strictly_order_preserving(&self, inputs: &[ExprProperties]) -> Result { + self.inner.strictly_order_preserving(inputs) + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { self.inner.coerce_types(arg_types) } diff --git a/datafusion/ffi/src/expr/expr_properties.rs b/datafusion/ffi/src/expr/expr_properties.rs index 5b37cc6a28535..584f774c7b26e 100644 --- a/datafusion/ffi/src/expr/expr_properties.rs +++ b/datafusion/ffi/src/expr/expr_properties.rs @@ -29,6 +29,7 @@ pub struct FFI_ExprProperties { sort_properties: FFI_SortProperties, range: FFI_Interval, preserves_lex_ordering: bool, + strictly_order_preserving: bool, } impl TryFrom<&ExprProperties> for FFI_ExprProperties { @@ -41,6 +42,7 @@ impl TryFrom<&ExprProperties> for FFI_ExprProperties { sort_properties, range, preserves_lex_ordering: value.preserves_lex_ordering, + strictly_order_preserving: value.strictly_order_preserving, }) } } @@ -54,6 +56,7 @@ impl TryFrom for ExprProperties { sort_properties, range, preserves_lex_ordering: value.preserves_lex_ordering, + strictly_order_preserving: value.strictly_order_preserving, }) } } diff --git a/datafusion/functions/src/datetime/from_unixtime.rs b/datafusion/functions/src/datetime/from_unixtime.rs index 4787c75b610b6..85494f3abff73 100644 --- a/datafusion/functions/src/datetime/from_unixtime.rs +++ b/datafusion/functions/src/datetime/from_unixtime.rs @@ -22,6 +22,7 @@ use arrow::datatypes::TimeUnit::Second; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::TypeSignature::Exact; +use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -147,6 +148,24 @@ impl ScalarUDFImpl for FromUnixtimeFunc { } } + fn output_ordering(&self, inputs: &[ExprProperties]) -> Result { + // The optional timezone argument must be a constant string and only + // affects the display metadata, not the stored epoch value, so the + // output ordering follows the first argument. + Ok(inputs[0].sort_properties) + } + + fn preserves_lex_ordering(&self, _inputs: &[ExprProperties]) -> Result { + Ok(true) + } + + fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result { + // `from_unixtime` stores the input's exact `Int64` value as a + // `Timestamp(Second)`: the mapping is one-to-one, order-preserving, + // and maps nulls to nulls. + Ok(true) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/datafusion/functions/src/math/monotonicity.rs b/datafusion/functions/src/math/monotonicity.rs index 52449f9c9e0b9..d1174d77b9db1 100644 --- a/datafusion/functions/src/math/monotonicity.rs +++ b/datafusion/functions/src/math/monotonicity.rs @@ -761,6 +761,7 @@ mod tests { .unwrap(), sort_properties: sp, preserves_lex_ordering: false, + strictly_order_preserving: false, } } diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index 2ce8a8d246fe7..15637d24e8a4b 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -329,7 +329,7 @@ mod tests { EquivalenceClass, EquivalenceGroup, EquivalenceProperties, OrderingEquivalenceClass, convert_to_orderings, convert_to_sort_exprs, }; - use crate::expressions::{BinaryExpr, Column, col}; + use crate::expressions::{BinaryExpr, CastExpr, Column, col}; use crate::utils::tests::TestScalarUDF; use crate::{ AcrossPartitions, ConstExpr, PhysicalExpr, PhysicalExprRef, PhysicalSortExpr, @@ -376,6 +376,45 @@ mod tests { Ok(()) } + #[test] + fn test_ordering_satisfy_strictly_order_preserving() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int64, true), + ])); + let col_a = col("a", &schema)?; + let col_b = col("b", &schema)?; + let asc = SortOptions::default(); + let sort_a = PhysicalSortExpr::new(Arc::clone(&col_a), asc); + let sort_b = PhysicalSortExpr::new(Arc::clone(&col_b), asc); + let eq_properties = EquivalenceProperties::new_with_orderings( + Arc::clone(&schema), + [vec![sort_a.clone(), sort_b.clone()]], + ); + + assert!(eq_properties.ordering_satisfy(vec![sort_a.clone(), sort_b.clone()])?); + assert!(eq_properties.ordering_satisfy(vec![sort_a.clone()])?); + + // A widening cast is strictly order-preserving: `a` is constant + // within each group of equal `CAST(a AS BIGINT)` values, so `b` + // remains sorted within those groups. + let widening = Arc::new(CastExpr::new(Arc::clone(&col_a), DataType::Int64, None)) + as PhysicalExprRef; + let sort_widening = PhysicalSortExpr::new(widening, asc); + assert!(eq_properties.ordering_satisfy(vec![sort_widening, sort_b.clone()])?); + + // A narrowing cast is only monotonic: it satisfies as a leading key, + // but it may collapse distinct `a` values, so `b` is not guaranteed + // to be sorted within its tie groups. + let narrowing = Arc::new(CastExpr::new(Arc::clone(&col_a), DataType::Int16, None)) + as PhysicalExprRef; + let sort_narrowing = PhysicalSortExpr::new(narrowing, asc); + assert!(eq_properties.ordering_satisfy(vec![sort_narrowing.clone()])?); + assert!(!eq_properties.ordering_satisfy(vec![sort_narrowing, sort_b.clone()])?); + + Ok(()) + } + #[test] fn test_ordering_satisfy_with_equivalence2() -> Result<()> { let test_schema = create_test_schema()?; diff --git a/datafusion/physical-expr/src/equivalence/properties/mod.rs b/datafusion/physical-expr/src/equivalence/properties/mod.rs index 17c3898fd9c89..22b3382f50638 100644 --- a/datafusion/physical-expr/src/equivalence/properties/mod.rs +++ b/datafusion/physical-expr/src/equivalence/properties/mod.rs @@ -33,13 +33,13 @@ use self::dependency::{ use crate::equivalence::{ AcrossPartitions, EquivalenceGroup, OrderingEquivalenceClass, ProjectionMapping, }; -use crate::expressions::{CastExpr, Column, Literal, with_new_schema}; +use crate::expressions::{Column, Literal, with_new_schema}; use crate::{ ConstExpr, LexOrdering, LexRequirement, PhysicalExpr, PhysicalSortExpr, PhysicalSortRequirement, }; -use arrow::datatypes::{DataType, SchemaRef}; +use arrow::datatypes::SchemaRef; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_common::{Constraint, Constraints, HashMap, Result, plan_err}; use datafusion_expr::interval_arithmetic::Interval; @@ -195,24 +195,30 @@ impl OrderingEquivalenceCache { } impl EquivalenceProperties { - /// Helper used by the ordering equivalence rule when considering whether a - /// cast-bearing expression can replace an existing sort key without - /// invalidating the ordering. + /// Helper used by the ordering equivalence rule when considering whether + /// an expression can replace an existing sort key without invalidating + /// the ordering. /// - /// The substitution is only allowed when the cast wraps the very same child - /// expression that the original sort used and the casted type is a - /// widening/order-preserving conversion. Without those restrictions, a - /// narrowing cast could collapse distinct values and violate the existing + /// The substitution is only allowed when, treating the sort key as the + /// only ordered input, the expression reports the same ordering *and* + /// that it is a one-to-one, order-preserving function of it (see + /// [`ExprProperties::strictly_order_preserving`]). For example, a + /// widening `CAST` of the sort key qualifies, while a narrowing one does + /// not, as it could collapse distinct values and violate the existing /// sort order. - fn substitute_cast_ordering( + fn substitute_order_preserving_ordering( r_expr: Arc, sort_expr: &PhysicalSortExpr, - expr_type: &DataType, + schema: &SchemaRef, ) -> Option { - let cast_expr = r_expr.downcast_ref::()?; - - (cast_expr.expr().eq(&sort_expr.expr) - && CastExpr::check_bigger_cast(cast_expr.cast_type(), expr_type)) + if r_expr.eq(&sort_expr.expr) { + // No point in substituting an expression with itself. + return None; + } + let dependencies = Dependencies::new(std::iter::once(sort_expr.clone())); + let properties = get_expr_properties(&r_expr, &dependencies, schema).ok()?; + (properties.strictly_order_preserving + && properties.sort_properties == SortProperties::Ordered(sort_expr.options)) .then(|| PhysicalSortExpr::new(r_expr, sort_expr.options)) } @@ -482,6 +488,7 @@ impl EquivalenceProperties { sort_properties: SortProperties::Ordered(next.options), range: Interval::make_unbounded(&data_type)?, preserves_lex_ordering: true, + strictly_order_preserving: true, }); } // Check if the expression is monotonic in all arguments: @@ -626,24 +633,55 @@ impl EquivalenceProperties { if !satisfy { return Ok(false); } - // Treat satisfied keys as constants in subsequent iterations. We - // can do this because the "next" key only matters in a lexicographical - // ordering when the keys to its left have the same values. - // - // Note that these expressions are not properly "constants". This is just - // an implementation strategy confined to this function. - // - // For example, assume that the requirement is `[a ASC, (b + c) ASC]`, - // and existing equivalent orderings are `[a ASC, b ASC]` and `[c ASC]`. - // From the analysis above, we know that `[a ASC]` is satisfied. Then, - // we add column `a` as constant to the algorithm state. This enables us - // to deduce that `(b + c) ASC` is satisfied, given `a` is constant. - let const_expr = ConstExpr::from(element.expr); - eq_properties.add_constants(std::iter::once(const_expr))?; + // Treat satisfied keys (and the sub-expressions they pin down) as + // constants in subsequent iterations. See + // [`Self::add_satisfied_key_constants`] for the rationale. + eq_properties.add_satisfied_key_constants(element.expr)?; } Ok(true) } + /// Registers a satisfied sort key as a constant for subsequent iterations + /// of the ordering satisfaction checks. We can do this because the "next" + /// key only matters in a lexicographical ordering when the keys to its + /// left have the same values (i.e. within a single tie group). Note that + /// these expressions are not properly "constants"; this is just an + /// implementation strategy confined to the satisfaction checks. + /// + /// For example, assume that the requirement is `[a ASC, (b + c) ASC]`, + /// and existing equivalent orderings are `[a ASC, b ASC]` and `[c ASC]`. + /// Once we deduce that `[a ASC]` is satisfied, we add column `a` as a + /// constant to the algorithm state. This enables us to deduce that + /// `(b + c) ASC` is satisfied, given `a` is constant. + /// + /// In addition to the key itself, this also registers any sub-expressions + /// whose values the key pins down: if an expression is strictly + /// order-preserving, equal outputs imply equal values of its ordered + /// children, so within a tie group of the key those children are constant + /// as well. For example, if data is sorted by `[a, b]`, the requirement + /// `[CAST(a AS BIGINT) ASC, b ASC]` is satisfied: `a` is constant within + /// each group of equal `CAST(a AS BIGINT)` values, and hence `b` is + /// sorted within each such group. + fn add_satisfied_key_constants(&mut self, expr: Arc) -> Result<()> { + let mut stack = vec![expr]; + while let Some(expr) = stack.pop() { + let properties = self.get_expr_properties(Arc::clone(&expr)); + if properties.strictly_order_preserving { + for child in expr.children() { + let child_properties = self.get_expr_properties(Arc::clone(child)); + if matches!( + child_properties.sort_properties, + SortProperties::Ordered(_) + ) { + stack.push(Arc::clone(child)); + } + } + } + self.add_constants(std::iter::once(ConstExpr::from(expr)))?; + } + Ok(()) + } + /// Returns the number of consecutive sort expressions (starting from the /// left) that are satisfied by the existing ordering. fn common_sort_prefix_length(&self, normal_ordering: &LexOrdering) -> Result { @@ -676,20 +714,10 @@ impl EquivalenceProperties { // many we've satisfied so far: return Ok(idx); } - // Treat satisfied keys as constants in subsequent iterations. We - // can do this because the "next" key only matters in a lexicographical - // ordering when the keys to its left have the same values. - // - // Note that these expressions are not properly "constants". This is just - // an implementation strategy confined to this function. - // - // For example, assume that the requirement is `[a ASC, (b + c) ASC]`, - // and existing equivalent orderings are `[a ASC, b ASC]` and `[c ASC]`. - // From the analysis above, we know that `[a ASC]` is satisfied. Then, - // we add column `a` as constant to the algorithm state. This enables us - // to deduce that `(b + c) ASC` is satisfied, given `a` is constant. - let const_expr = ConstExpr::from(Arc::clone(&element.expr)); - eq_properties.add_constants(std::iter::once(const_expr))? + // Treat satisfied keys (and the sub-expressions they pin down) as + // constants in subsequent iterations. See + // [`Self::add_satisfied_key_constants`] for the rationale. + eq_properties.add_satisfied_key_constants(Arc::clone(&element.expr))?; } // All sort expressions are satisfied, return full length: Ok(full_length) @@ -840,7 +868,9 @@ impl EquivalenceProperties { /// /// TODO: Handle all scenarios that allow substitution; e.g. when `x` is /// sorted, `atan(x + 1000)` should also be substituted. For now, we - /// only consider single-column `CAST` expressions. + /// consider widening `CAST` expressions and single-child expressions + /// that declare themselves one-to-one order-preserving via + /// [`ExprProperties::strictly_order_preserving`]. fn substitute_oeq_class( schema: &SchemaRef, mapping: &ProjectionMapping, @@ -852,21 +882,17 @@ impl EquivalenceProperties { order .into_iter() .map(|sort_expr| { - // The sort expression comes from this schema, so the - // following call to `unwrap` is safe. - let expr_type = sort_expr.expr.data_type(schema).unwrap(); let original_sort_expr = sort_expr.clone(); - // TODO: Add one-to-one analysis for ScalarFunctions. mapping .iter() .map(|(source, _target)| source) .filter(|source| expr_refers(source, &original_sort_expr.expr)) .cloned() .filter_map(|r_expr| { - Self::substitute_cast_ordering( + Self::substitute_order_preserving_ordering( r_expr, &original_sort_expr, - &expr_type, + schema, ) }) .chain(std::iter::once(sort_expr)) @@ -1407,7 +1433,10 @@ fn update_properties( } else if node.expr.is::() { // We have a Column, which is the other possible leaf node type: node.data.range = - Interval::make_unbounded(&node.expr.data_type(eq_properties.schema())?)? + Interval::make_unbounded(&node.expr.data_type(eq_properties.schema())?)?; + // A column is the identity mapping of itself, which is trivially + // strict: + node.data.strictly_order_preserving = true; } // Now, check what we know about orderings: let normal_expr = eq_properties @@ -1469,23 +1498,36 @@ fn get_expr_properties( schema: &SchemaRef, ) -> Result { if let Some(column_order) = dependencies.iter().find(|&order| expr.eq(&order.expr)) { - // If exact match is found, return its ordering. + // If exact match is found, return its ordering. This is a base case + // of the recursion: the expression is treated as an atomic ordered + // input from here on, so `strictly_order_preserving` states only that + // it is a one-to-one mapping *of itself* (the identity), which holds + // for any expression. It makes no claim about the expression being + // one-to-one in its own inputs (e.g. `floor(x)` as a sort key), and + // it does not need to: parent expressions are substituted for this + // sort key, so their strictness only has to be relative to it. Ok(ExprProperties { sort_properties: SortProperties::Ordered(column_order.options), range: Interval::make_unbounded(&expr.data_type(schema)?)?, preserves_lex_ordering: false, + strictly_order_preserving: true, }) } else if expr.downcast_ref::().is_some() { Ok(ExprProperties { sort_properties: SortProperties::Unordered, range: Interval::make_unbounded(&expr.data_type(schema)?)?, preserves_lex_ordering: false, + // A base case of the recursion: a column is the identity mapping + // of itself, which is trivially one-to-one. + strictly_order_preserving: true, }) } else if let Some(literal) = expr.downcast_ref::() { Ok(ExprProperties { sort_properties: SortProperties::Singleton, range: literal.value().into(), preserves_lex_ordering: true, + // Vacuously true: a literal has no ordered inputs. + strictly_order_preserving: true, }) } else { // Find orderings of its children diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 89828620a5930..8b71b3ec409c7 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -764,41 +764,49 @@ impl PhysicalExpr for BinaryExpr { sort_properties: l_order.add(&r_order), range: l_range.add(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::Minus => Ok(ExprProperties { sort_properties: l_order.sub(&r_order), range: l_range.sub(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::Gt => Ok(ExprProperties { sort_properties: l_order.gt_or_gteq(&r_order), range: l_range.gt(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::GtEq => Ok(ExprProperties { sort_properties: l_order.gt_or_gteq(&r_order), range: l_range.gt_eq(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::Lt => Ok(ExprProperties { sort_properties: r_order.gt_or_gteq(&l_order), range: l_range.lt(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::LtEq => Ok(ExprProperties { sort_properties: r_order.gt_or_gteq(&l_order), range: l_range.lt_eq(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::And => Ok(ExprProperties { sort_properties: r_order.and_or(&l_order), range: l_range.and(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), Operator::Or => Ok(ExprProperties { sort_properties: r_order.and_or(&l_order), range: l_range.or(r_range)?, preserves_lex_ordering: false, + strictly_order_preserving: false, }), _ => Ok(ExprProperties::new_unknown()), } diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 8be2e187d72f7..dbb91e365af90 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -211,8 +211,18 @@ pub(crate) fn cast_expr_properties( target_type: &DataType, ) -> Result { let unbounded = Interval::make_unbounded(target_type)?; - if is_order_preserving_cast_family(&child.range.data_type(), target_type) { - Ok(child.clone().with_range(unbounded)) + let source_type = child.range.data_type(); + // A widening cast is additionally one-to-one, so it is strictly + // order-preserving; a narrowing cast may collapse distinct values, + // breaking the ordering of subsequent sort keys. + let bigger_cast = CastExpr::check_bigger_cast(target_type, &source_type); + if is_order_preserving_cast_family(&source_type, target_type) || bigger_cast { + Ok(child + .clone() + .with_range(unbounded) + .with_strictly_order_preserving( + child.strictly_order_preserving && bigger_cast, + )) } else { Ok(ExprProperties::new_unknown().with_range(unbounded)) } diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs index 5fb9a3b2cd29b..a7af824230780 100644 --- a/datafusion/physical-expr/src/expressions/literal.rs +++ b/datafusion/physical-expr/src/expressions/literal.rs @@ -123,6 +123,8 @@ impl PhysicalExpr for Literal { sort_properties: SortProperties::Singleton, range: Interval::try_new(self.value().clone(), self.value().clone())?, preserves_lex_ordering: true, + // Vacuously true: a literal has no ordered inputs. + strictly_order_preserving: true, }) } diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index 9fbf38361c89c..c894c12784dc5 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -166,6 +166,8 @@ impl PhysicalExpr for NegativeExpr { sort_properties: -children[0].sort_properties, range: children[0].range.clone().arithmetic_negate()?, preserves_lex_ordering: false, + // Negation is one-to-one but reverses the ordering direction. + strictly_order_preserving: false, }) } diff --git a/datafusion/physical-expr/src/scalar_function.rs b/datafusion/physical-expr/src/scalar_function.rs index 418d005c971ea..6a5ab219aa8dd 100644 --- a/datafusion/physical-expr/src/scalar_function.rs +++ b/datafusion/physical-expr/src/scalar_function.rs @@ -316,6 +316,7 @@ impl PhysicalExpr for ScalarFunctionExpr { fn get_properties(&self, children: &[ExprProperties]) -> Result { let sort_properties = self.fun.output_ordering(children)?; let preserves_lex_ordering = self.fun.preserves_lex_ordering(children)?; + let strictly_order_preserving = self.fun.strictly_order_preserving(children)?; let children_range = children .iter() .map(|props| &props.range) @@ -326,6 +327,7 @@ impl PhysicalExpr for ScalarFunctionExpr { sort_properties, range, preserves_lex_ordering, + strictly_order_preserving, }) } diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt index 978fcc197c0de..a267ddddddd54 100644 --- a/datafusion/sqllogictest/test_files/order.slt +++ b/datafusion/sqllogictest/test_files/order.slt @@ -709,6 +709,104 @@ physical_plan statement ok drop table multiple_ordered_table; + +# Create a table having dependent sort order +statement ok +CREATE EXTERNAL TABLE multiple_ordered_table ( + a0 INTEGER, + a INTEGER, + b INTEGER, + c INTEGER, + d INTEGER +) +STORED AS CSV +WITH ORDER (a ASC, b ASC, c ASC) +LOCATION '../core/tests/data/window_2.csv' +OPTIONS ('format.has_header' 'true'); + +# Test without repartition so removal of sort is more apperant +statement ok +set datafusion.execution.target_partitions = 1; + +# A strictly order-preserving scalar function is one-to-one, so an ordering on +# its argument carries over to its result. `from_unixtime` reinterprets the +# input integer as a timestamp without changing the value, so the whole +# ordering is preserved and no SortExec is needed. +query TT +EXPLAIN SELECT from_unixtime(a) AS a_, from_unixtime(b) AS b_, from_unixtime(c) AS c_ +FROM multiple_ordered_table +ORDER BY a_, b_, c_; +---- +logical_plan +01)Sort: a_ ASC NULLS LAST, b_ ASC NULLS LAST, c_ ASC NULLS LAST +02)--Projection: from_unixtime(CAST(multiple_ordered_table.a AS Int64)) AS a_, from_unixtime(CAST(multiple_ordered_table.b AS Int64)) AS b_, from_unixtime(CAST(multiple_ordered_table.c AS Int64)) AS c_ +03)----TableScan: multiple_ordered_table projection=[a, b, c] +physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[from_unixtime(CAST(a@1 AS Int64)) as a_, from_unixtime(CAST(b@2 AS Int64)) as b_, from_unixtime(CAST(c@3 AS Int64)) as c_], file_type=csv, has_header=true + +# Being one-to-one also justifies keeping the *suffix* sort keys: data sorted +# by [a, b] is also sorted by [from_unixtime(a), b], because rows with equal +# `a_` have equal `a`, within which `b` is already sorted. +query TT +EXPLAIN SELECT from_unixtime(a) AS a_, b +FROM multiple_ordered_table +ORDER BY a_, b; +---- +logical_plan +01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST +02)--Projection: from_unixtime(CAST(multiple_ordered_table.a AS Int64)) AS a_, multiple_ordered_table.b +03)----TableScan: multiple_ordered_table projection=[a, b] +physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[from_unixtime(CAST(a@1 AS Int64)) as a_, b], file_type=csv, has_header=true + +# A widening CAST is one-to-one too: +query TT +EXPLAIN SELECT CAST(a AS BIGINT) AS a_, b +FROM multiple_ordered_table +ORDER BY a_, b; +---- +logical_plan +01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST +02)--Projection: CAST(multiple_ordered_table.a AS Int64) AS a_, multiple_ordered_table.b +03)----TableScan: multiple_ordered_table projection=[a, b] +physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[CAST(a@1 AS Int64) as a_, b], file_type=csv, has_header=true + +# In contrast, a merely monotone (`preserves_lex_ordering`, but not strictly +# order-preserving) function such as floor() does NOT justify the suffix keys: +# in general floor() collapses distinct inputs into one output value, and `b` +# is not sorted within such a run, so a SortExec must remain. +query TT +EXPLAIN SELECT floor(CAST(a AS DOUBLE)) AS a_, b +FROM multiple_ordered_table +ORDER BY a_, b; +---- +logical_plan +01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST +02)--Projection: floor(CAST(multiple_ordered_table.a AS Float64)) AS a_, multiple_ordered_table.b +03)----TableScan: multiple_ordered_table projection=[a, b] +physical_plan +01)SortExec: expr=[a_@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[floor(CAST(a@1 AS Float64)) as a_, b], file_type=csv, has_header=true + +# Monotonicity alone is still enough when the expression is the *only* sort +# key, so here the SortExec is removed even though floor() is not strict: +query TT +EXPLAIN SELECT floor(CAST(a AS DOUBLE)) AS a_ +FROM multiple_ordered_table +ORDER BY a_; +---- +logical_plan +01)Sort: a_ ASC NULLS LAST +02)--Projection: floor(CAST(multiple_ordered_table.a AS Float64)) AS a_ +03)----TableScan: multiple_ordered_table projection=[a] +physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[floor(CAST(a@1 AS Float64)) as a_], file_type=csv, has_header=true + +# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# reset it explicitly. +statement ok +set datafusion.execution.target_partitions = 4; + +statement ok +drop table multiple_ordered_table; + # Create tables having some ordered columns. In the next step, we will expect to observe that scalar # functions, such as mathematical functions like atan(), ceil(), sqrt(), or date_time functions # like date_bin() and date_trunc(), will maintain the order of its argument columns. From 9facaaabd001d0f82ecc6bcd4d09adcc6f731109 Mon Sep 17 00:00:00 2001 From: Amogh Ramesh Date: Mon, 27 Jul 2026 19:58:38 +0530 Subject: [PATCH 040/109] FFI: forward ScalarUDF preserves_lex_ordering (#23069) ## Which issue does this PR close? Part of #22330. ## Rationale for this change `ForeignScalarUDF` inherits the default `preserves_lex_ordering`, so producer overrides are lost across the FFI boundary. ## What changes are included in this PR? - Forward `preserves_lex_ordering` through `FFI_ScalarUDF`. - Reuse the existing placement UDF for unit and dynamic-library coverage. ## Are these changes tested? - `cargo test -p datafusion-ffi --features integration-tests` - `cargo clippy --all-targets --all-features -- -D warnings` ## Are there any user-facing changes? The FFI ABI changes. Foreign libraries must rebuild against the new DataFusion version. --------- Signed-off-by: Amogh Ramesh --- datafusion/ffi/src/tests/udf_udaf_udwf.rs | 8 ++++ datafusion/ffi/src/udf/mod.rs | 57 +++++++++++++++++++++++ datafusion/ffi/tests/ffi_udf.rs | 10 +++- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/datafusion/ffi/src/tests/udf_udaf_udwf.rs b/datafusion/ffi/src/tests/udf_udaf_udwf.rs index b393f5db3a506..a84df52b8dbee 100644 --- a/datafusion/ffi/src/tests/udf_udaf_udwf.rs +++ b/datafusion/ffi/src/tests/udf_udaf_udwf.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use arrow_schema::DataType; use datafusion_catalog::TableFunctionImpl; use datafusion_common::ScalarValue; +use datafusion_expr::sort_properties::ExprProperties; use datafusion_expr::{ AggregateUDF, ColumnarValue, ExpressionPlacement, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, WindowUDF, @@ -152,6 +153,13 @@ impl ScalarUDFImpl for PlacementUDF { ExpressionPlacement::KeepInPlace } } + + fn preserves_lex_ordering( + &self, + inputs: &[ExprProperties], + ) -> datafusion_common::Result { + Ok(inputs.iter().all(|input| input.preserves_lex_ordering)) + } } pub(crate) extern "C" fn create_placement_func() -> FFI_ScalarUDF { diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rs index 4fc22e859f9fb..8e96dd9013e2a 100644 --- a/datafusion/ffi/src/udf/mod.rs +++ b/datafusion/ffi/src/udf/mod.rs @@ -26,6 +26,7 @@ use arrow::ffi::{FFI_ArrowSchema, from_ffi, to_ffi}; use arrow_schema::FieldRef; use datafusion_common::config::ConfigOptions; use datafusion_common::{DataFusionError, Result, internal_err}; +use datafusion_expr::sort_properties::ExprProperties; use datafusion_expr::type_coercion::functions::fields_with_udf; use datafusion_expr::{ ColumnarValue, ExpressionPlacement, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, @@ -41,6 +42,7 @@ use stabby::vec::Vec as SVec; use crate::arrow_wrappers::{WrappedArray, WrappedSchema}; use crate::config::FFI_ConfigOptions; use crate::expr::columnar_value::FFI_ColumnarValue; +use crate::expr::expr_properties::FFI_ExprProperties; use crate::placement::FFI_ExpressionPlacement; use crate::util::{ FFI_Result, rvec_wrapped_to_vec_datatype, vec_datatype_to_rvec_wrapped, @@ -115,6 +117,12 @@ pub struct FFI_ScalarUDF { /// the foreign interface. See [`crate::get_library_marker_id`] and /// the crate's `README.md` for more information. pub library_marker_id: extern "C" fn() -> usize, + + /// FFI equivalent to [`ScalarUDFImpl::preserves_lex_ordering`]. + pub preserves_lex_ordering: unsafe extern "C" fn( + udf: &Self, + inputs: SVec, + ) -> FFI_Result, } unsafe impl Send for FFI_ScalarUDF {} @@ -178,6 +186,19 @@ unsafe extern "C" fn placement_fn_wrapper( udf.inner().placement(&args).into() } +unsafe extern "C" fn preserves_lex_ordering_fn_wrapper( + udf: &FFI_ScalarUDF, + inputs: SVec, +) -> FFI_Result { + let result = inputs + .into_iter() + .map(ExprProperties::try_from) + .collect::>>() + .and_then(|inputs| udf.inner().preserves_lex_ordering(&inputs)); + + sresult!(result) +} + unsafe extern "C" fn invoke_with_args_fn_wrapper( udf: &FFI_ScalarUDF, args: SVec, @@ -276,6 +297,7 @@ impl From> for FFI_ScalarUDF { release: release_fn_wrapper, private_data: Box::into_raw(private_data) as *mut c_void, library_marker_id: crate::get_library_marker_id, + preserves_lex_ordering: preserves_lex_ordering_fn_wrapper, } } } @@ -460,6 +482,18 @@ impl ScalarUDFImpl for ForeignScalarUDF { result.into() } + + fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { + inputs + .iter() + .map(FFI_ExprProperties::try_from) + .collect::>>() + .and_then(|inputs| { + let result = + unsafe { (self.udf.preserves_lex_ordering)(&self.udf, inputs) }; + df_result!(result) + }) + } } #[cfg(test)] @@ -500,6 +534,14 @@ mod tests { ExpressionPlacement::KeepInPlace } } + + fn preserves_lex_ordering(&self, inputs: &[ExprProperties]) -> Result { + if inputs.is_empty() { + return internal_err!("preserves_lex_ordering requires an input"); + } + + Ok(inputs.iter().all(|input| input.preserves_lex_ordering)) + } } #[test] @@ -572,6 +614,21 @@ mod tests { ); assert_eq!(foreign_udf.placement(&[]), ExpressionPlacement::KeepInPlace); + let preserves = ExprProperties::new_unknown().with_preserves_lex_ordering(true); + let does_not_preserve = ExprProperties::new_unknown(); + + assert!( + foreign_udf + .preserves_lex_ordering(std::slice::from_ref(&preserves)) + .unwrap() + ); + assert!( + !foreign_udf + .preserves_lex_ordering(&[preserves, does_not_preserve]) + .unwrap() + ); + assert!(foreign_udf.preserves_lex_ordering(&[]).is_err()); + Ok(()) } } diff --git a/datafusion/ffi/tests/ffi_udf.rs b/datafusion/ffi/tests/ffi_udf.rs index 617cbc196b1ac..d9e7263ccd44d 100644 --- a/datafusion/ffi/tests/ffi_udf.rs +++ b/datafusion/ffi/tests/ffi_udf.rs @@ -26,6 +26,7 @@ mod tests { use datafusion::prelude::{SessionContext, col}; use datafusion_execution::config::SessionConfig; use datafusion_expr::lit; + use datafusion_expr::sort_properties::ExprProperties; use datafusion_ffi::tests::create_record_batch; use datafusion_ffi::tests::utils::get_module; use std::sync::Arc; @@ -90,8 +91,7 @@ mod tests { Ok(()) } - /// This test validates that a producer's `placement` override survives the - /// FFI boundary instead of collapsing to the default `KeepInPlace`. + /// Checks planning-property overrides across the FFI boundary. #[tokio::test] async fn test_scalar_udf_placement() -> Result<()> { let module = get_module()?; @@ -112,6 +112,12 @@ mod tests { ExpressionPlacement::KeepInPlace ); + let preserves = ExprProperties::new_unknown().with_preserves_lex_ordering(true); + let does_not_preserve = ExprProperties::new_unknown(); + + assert!(foreign_func.preserves_lex_ordering(std::slice::from_ref(&preserves))?); + assert!(!foreign_func.preserves_lex_ordering(&[preserves, does_not_preserve])?); + Ok(()) } From 8d2ffc96bcf65f5931f131dbe891f5ff9912e21d Mon Sep 17 00:00:00 2001 From: Pavan51 Date: Mon, 27 Jul 2026 11:12:48 -0500 Subject: [PATCH 041/109] perf(functions-aggregate): optimize sliding window MIN/MAX using monotonic deques (#23826) (#23827) ## Which issue does this PR close? - Closes #23826 ### What changes are included in this PR? This PR optimizes sliding window `MIN`/`MAX` aggregate functions using a **Sequence-Numbered Monotonic Deque** instead of a Two-Stack Queue. **Revised Design:** - We store `(sequence_number, value)` pairs in a single `VecDeque`, kept in strictly monotonic order. - **`push(val)`**: Evicts dominated elements from the back, then pushes the new value with the current `push_seq` and increments `push_seq`. - **`pop()`**: Increments `pop_seq`. If the front elements sequence number equals the old `pop_seq`, it is expired and popped. - **Benefits**: No secondary FIFO queue (lower memory) and no `clone()` overhead. ### Are these changes tested? Yes, existing tests pass. Added tests for empty-window `pop()` and duplicate-heavy scenarios. ### Are there any user-facing changes? **API Change**: `MovingMin` and `MovingMax` were changed to `pub(crate)` visibility, and their `pop()` methods now return `()` instead of returning a value. Performance is significantly improved (2x-3.5x throughput). --------- Co-authored-by: Pavan --- datafusion/functions-aggregate/Cargo.toml | 4 + .../benches/sliding_max.rs | 113 ++++++ datafusion/functions-aggregate/src/min_max.rs | 366 +++++++++--------- .../library-user-guide/upgrading/55.0.0.md | 10 + 4 files changed, 315 insertions(+), 178 deletions(-) create mode 100644 datafusion/functions-aggregate/benches/sliding_max.rs diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml index c1b992a6d89b0..e75afc8f0b4bc 100644 --- a/datafusion/functions-aggregate/Cargo.toml +++ b/datafusion/functions-aggregate/Cargo.toml @@ -95,5 +95,9 @@ harness = false name = "percentile_cont" harness = false +[[bench]] +name = "sliding_max" +harness = false + [features] force_hash_collisions = ["datafusion-common/force_hash_collisions"] diff --git a/datafusion/functions-aggregate/benches/sliding_max.rs b/datafusion/functions-aggregate/benches/sliding_max.rs new file mode 100644 index 0000000000000..d5de001a1a79d --- /dev/null +++ b/datafusion/functions-aggregate/benches/sliding_max.rs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::{ArrayRef, Int64Array, StringArray}; +use arrow::datatypes::DataType; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use datafusion_expr::Accumulator; +use datafusion_functions_aggregate::min_max::SlidingMaxAccumulator; +use rand::Rng; +use rand::SeedableRng; +use rand::rngs::StdRng; +use std::sync::Arc; + +fn generate_random_i64(size: usize) -> Vec { + let mut rng = StdRng::seed_from_u64(42); + (0..size).map(|_| rng.random_range(0..1_000_000)).collect() +} + +fn generate_random_strings(size: usize) -> Vec { + let mut rng = StdRng::seed_from_u64(42); + (0..size) + .map(|_| { + let len = rng.random_range(10..40); + (0..len) + .map(|_| rng.random_range(b'a'..=b'z') as char) + .collect() + }) + .collect() +} + +/// Simulates a sliding window by calling update_batch and retract_batch +/// on SlidingMaxAccumulator, mirroring how the query engine uses it. +fn bench_sliding_max_for( + c: &mut Criterion, + label: &str, + data_type: &DataType, + array: &ArrayRef, + data_size: usize, + window_size: usize, +) { + let mut group = c.benchmark_group(format!("sliding_window_max_{label}")); + group.throughput(Throughput::Elements(data_size as u64)); + + group.bench_with_input( + BenchmarkId::new("sliding_max", window_size), + &window_size, + |b, &w| { + b.iter(|| { + let mut acc = SlidingMaxAccumulator::try_new(data_type).unwrap(); + // Warm up the window + let init_batch = array.slice(0, w); + acc.update_batch(&[init_batch]).unwrap(); + + // Slide: for each subsequent element, add it and retract one + for i in w..data_size { + let new_val = array.slice(i, 1); + let old_val = array.slice(i - w, 1); + acc.update_batch(&[new_val]).unwrap(); + acc.retract_batch(&[old_val]).unwrap(); + std::hint::black_box(acc.evaluate().unwrap()); + } + }); + }, + ); + + group.finish(); +} + +fn bench_sliding_max(c: &mut Criterion) { + let data_size = 50_000; + + let i64_data: Vec = generate_random_i64(data_size); + let str_data: Vec = generate_random_strings(data_size); + + let i64_array: ArrayRef = Arc::new(Int64Array::from(i64_data)); + let str_array: ArrayRef = Arc::new(StringArray::from(str_data)); + + for window_size in [100, 1000, 5000] { + bench_sliding_max_for( + c, + "int64", + &DataType::Int64, + &i64_array, + data_size, + window_size, + ); + bench_sliding_max_for( + c, + "utf8", + &DataType::Utf8, + &str_array, + data_size, + window_size, + ); + } +} + +criterion_group!(benches, bench_sliding_max); +criterion_main!(benches); diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs index 1a3179170c7b6..b306e7db2f0b7 100644 --- a/datafusion/functions-aggregate/src/min_max.rs +++ b/datafusion/functions-aggregate/src/min_max.rs @@ -52,6 +52,7 @@ use datafusion_expr::{ use datafusion_expr::{GroupsAccumulator, StatisticsArgs}; use datafusion_macros::user_doc; use half::f16; +use std::collections::VecDeque; use std::mem::{size_of, size_of_val}; use std::ops::Deref; @@ -731,71 +732,47 @@ impl Accumulator for SlidingMinAccumulator { /// Keep track of the minimum value in a sliding window. /// -/// The implementation is taken from +/// `MovingMin` keeps track of the minimum value in a sliding window using a +/// monotonic deque. Each element is stored with its sequence number, and the +/// deque maintains candidate elements in ascending value order. /// -/// `moving min max` provides one data structure for keeping track of the -/// minimum value and one for keeping track of the maximum value in a sliding -/// window. -/// -/// Each element is stored with the current min/max. One stack to push and another one for pop. If pop stack is empty, -/// push to this stack all elements popped from first stack while updating their current min/max. Now pop from -/// the second stack (MovingMin/Max struct works as a queue). To find the minimum element of the queue, -/// look at the smallest/largest two elements of the individual stacks, then take the minimum of those two values. -/// -/// The complexity of the operations are -/// - O(1) for getting the minimum/maximum -/// - O(1) for push -/// - amortized O(1) for pop -/// -/// ``` -/// # use datafusion_functions_aggregate::min_max::MovingMin; -/// let mut moving_min = MovingMin::::new(); -/// moving_min.push(2); -/// moving_min.push(1); -/// moving_min.push(3); -/// -/// assert_eq!(moving_min.min(), Some(&1)); -/// assert_eq!(moving_min.pop(), Some(2)); -/// -/// assert_eq!(moving_min.min(), Some(&1)); -/// assert_eq!(moving_min.pop(), Some(1)); -/// -/// assert_eq!(moving_min.min(), Some(&3)); -/// assert_eq!(moving_min.pop(), Some(3)); -/// -/// assert_eq!(moving_min.min(), None); -/// assert_eq!(moving_min.pop(), None); -/// ``` +/// Complexity: +/// - O(1) for getting the minimum +/// - amortized O(1) for push +/// - O(1) for pop #[derive(Debug)] -pub struct MovingMin { - push_stack: Vec<(T, T)>, - pop_stack: Vec<(T, T)>, +pub(crate) struct MovingMin { + deque: VecDeque<(u64, T)>, + push_seq: u64, + pop_seq: u64, } -impl Default for MovingMin { +impl Default for MovingMin { fn default() -> Self { Self { - push_stack: Vec::new(), - pop_stack: Vec::new(), + deque: VecDeque::new(), + push_seq: 0, + pop_seq: 0, } } } -impl MovingMin { - /// Creates a new `MovingMin` to keep track of the minimum in a sliding - /// window. +impl MovingMin { + /// Creates a new `MovingMin` to keep track of the minimum in a sliding window. #[inline] pub fn new() -> Self { Self::default() } - /// Creates a new `MovingMin` to keep track of the minimum in a sliding - /// window with `capacity` allocated slots. + /// Creates a new `MovingMin` to keep track of the minimum in a sliding window with + /// `capacity` allocated slots. + #[cfg(test)] #[inline] pub fn with_capacity(capacity: usize) -> Self { Self { - push_stack: Vec::with_capacity(capacity), - pop_stack: Vec::with_capacity(capacity), + deque: VecDeque::with_capacity(capacity), + push_seq: 0, + pop_seq: 0, } } @@ -803,129 +780,113 @@ impl MovingMin { /// empty. #[inline] pub fn min(&self) -> Option<&T> { - match (self.push_stack.last(), self.pop_stack.last()) { - (None, None) => None, - (Some((_, min)), None) => Some(min), - (None, Some((_, min))) => Some(min), - (Some((_, a)), Some((_, b))) => Some(if a < b { a } else { b }), - } + self.deque.front().map(|(_, val)| val) + } + + #[inline] + fn check_invariants(&self) { + debug_assert!(self.pop_seq <= self.push_seq); + debug_assert!( + self.deque + .front() + .is_none_or(|&(front_seq, _)| front_seq >= self.pop_seq) + ); } /// Pushes a new element into the sliding window. #[inline] pub fn push(&mut self, val: T) { - self.push_stack.push(match self.push_stack.last() { - Some((_, min)) => { - if val > *min { - (val, min.clone()) - } else { - (val.clone(), val) - } - } - None => (val.clone(), val), - }); + let seq = self.push_seq; + self.push_seq += 1; + while self.deque.back().is_some_and(|back_val| back_val.1 >= val) { + self.deque.pop_back(); + } + self.deque.push_back((seq, val)); + + self.check_invariants(); } - /// Removes and returns the last value of the sliding window. + /// Removes the oldest value from the sliding window. + /// + /// If the window is empty, this is a no-op. #[inline] - pub fn pop(&mut self) -> Option { - if self.pop_stack.is_empty() { - match self.push_stack.pop() { - Some((val, _)) => { - let mut last = (val.clone(), val); - self.pop_stack.push(last.clone()); - while let Some((val, _)) = self.push_stack.pop() { - let min = if last.1 < val { - last.1.clone() - } else { - val.clone() - }; - last = (val.clone(), min); - self.pop_stack.push(last.clone()); - } - } - None => return None, - } + pub fn pop(&mut self) { + if self.is_empty() { + return; + } + let seq = self.pop_seq; + self.pop_seq += 1; + if self + .deque + .front() + .is_some_and(|front_val| front_val.0 == seq) + { + self.deque.pop_front(); } - self.pop_stack.pop().map(|(val, _)| val) + + self.check_invariants(); } /// Returns the number of elements stored in the sliding window. - #[inline] + #[cfg(test)] pub fn len(&self) -> usize { - self.push_stack.len() + self.pop_stack.len() + (self.push_seq - self.pop_seq) as usize } /// Returns `true` if the moving window contains no elements. #[inline] pub fn is_empty(&self) -> bool { - self.len() == 0 + self.push_seq == self.pop_seq } - /// Heap bytes owned by the two stack buffers plus each stored `T`'s + /// Heap bytes owned by the deque plus each stored `T`'s /// heap payload as reported by `elem_heap`. Excludes `size_of::()`. #[inline] fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize { - moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap) + moving_deque_heap_size(&self.deque, elem_heap) } } /// Shared implementation for [`MovingMin::heap_size`] and -/// [`MovingMax::heap_size`]. Both share the same two-stack layout. +/// [`MovingMax::heap_size`]. Both share the same deque layout. #[inline] -fn moving_stacks_heap_size( - push_stack: &Vec<(T, T)>, - pop_stack: &Vec<(T, T)>, +fn moving_deque_heap_size( + deque: &VecDeque<(u64, T)>, elem_heap: impl Fn(&T) -> usize, ) -> usize { - let buffers = (push_stack.capacity() + pop_stack.capacity()) * size_of::<(T, T)>(); - let elems: usize = push_stack - .iter() - .chain(pop_stack.iter()) - .map(|(a, b)| elem_heap(a) + elem_heap(b)) - .sum(); + let buffers = deque.capacity() * size_of::<(u64, T)>(); + let elems: usize = deque.iter().map(|(_, val)| elem_heap(val)).sum(); buffers + elems } /// Keep track of the maximum value in a sliding window. /// -/// See [`MovingMin`] for more details. -/// -/// ``` -/// # use datafusion_functions_aggregate::min_max::MovingMax; -/// let mut moving_max = MovingMax::::new(); -/// moving_max.push(2); -/// moving_max.push(3); -/// moving_max.push(1); -/// -/// assert_eq!(moving_max.max(), Some(&3)); -/// assert_eq!(moving_max.pop(), Some(2)); -/// -/// assert_eq!(moving_max.max(), Some(&3)); -/// assert_eq!(moving_max.pop(), Some(3)); +/// `MovingMax` keeps track of the maximum value in a sliding window using a +/// monotonic deque. Each element is stored with its sequence number, and the +/// deque maintains candidate elements in descending value order. /// -/// assert_eq!(moving_max.max(), Some(&1)); -/// assert_eq!(moving_max.pop(), Some(1)); -/// -/// assert_eq!(moving_max.max(), None); -/// assert_eq!(moving_max.pop(), None); -/// ``` +/// Complexity: +/// - O(1) for getting the maximum +/// - amortized O(1) for push +/// - O(1) for pop #[derive(Debug)] -pub struct MovingMax { - push_stack: Vec<(T, T)>, - pop_stack: Vec<(T, T)>, +pub(crate) struct MovingMax { + deque: VecDeque<(u64, T)>, + push_seq: u64, + pop_seq: u64, } -impl Default for MovingMax { +impl Default for MovingMax { fn default() -> Self { Self { - push_stack: Vec::new(), - pop_stack: Vec::new(), + deque: VecDeque::new(), + push_seq: 0, + pop_seq: 0, } } } -impl MovingMax { +impl MovingMax { /// Creates a new `MovingMax` to keep track of the maximum in a sliding window. #[inline] pub fn new() -> Self { @@ -934,81 +895,83 @@ impl MovingMax { /// Creates a new `MovingMax` to keep track of the maximum in a sliding window with /// `capacity` allocated slots. + #[cfg(test)] #[inline] pub fn with_capacity(capacity: usize) -> Self { Self { - push_stack: Vec::with_capacity(capacity), - pop_stack: Vec::with_capacity(capacity), + deque: VecDeque::with_capacity(capacity), + push_seq: 0, + pop_seq: 0, } } /// Returns the maximum of the sliding window or `None` if the window is empty. #[inline] pub fn max(&self) -> Option<&T> { - match (self.push_stack.last(), self.pop_stack.last()) { - (None, None) => None, - (Some((_, max)), None) => Some(max), - (None, Some((_, max))) => Some(max), - (Some((_, a)), Some((_, b))) => Some(if a > b { a } else { b }), - } + self.deque.front().map(|(_, val)| val) + } + + #[inline] + fn check_invariants(&self) { + debug_assert!(self.pop_seq <= self.push_seq); + debug_assert!( + self.deque + .front() + .is_none_or(|&(front_seq, _)| front_seq >= self.pop_seq) + ); } /// Pushes a new element into the sliding window. #[inline] pub fn push(&mut self, val: T) { - self.push_stack.push(match self.push_stack.last() { - Some((_, max)) => { - if val < *max { - (val, max.clone()) - } else { - (val.clone(), val) - } - } - None => (val.clone(), val), - }); + let seq = self.push_seq; + self.push_seq += 1; + while self.deque.back().is_some_and(|back_val| back_val.1 <= val) { + self.deque.pop_back(); + } + self.deque.push_back((seq, val)); + + self.check_invariants(); } - /// Removes and returns the last value of the sliding window. + /// Removes the oldest value from the sliding window. + /// + /// If the window is empty, this is a no-op. #[inline] - pub fn pop(&mut self) -> Option { - if self.pop_stack.is_empty() { - match self.push_stack.pop() { - Some((val, _)) => { - let mut last = (val.clone(), val); - self.pop_stack.push(last.clone()); - while let Some((val, _)) = self.push_stack.pop() { - let max = if last.1 > val { - last.1.clone() - } else { - val.clone() - }; - last = (val.clone(), max); - self.pop_stack.push(last.clone()); - } - } - None => return None, - } + pub fn pop(&mut self) { + if self.is_empty() { + return; + } + let seq = self.pop_seq; + self.pop_seq += 1; + if self + .deque + .front() + .is_some_and(|front_val| front_val.0 == seq) + { + self.deque.pop_front(); } - self.pop_stack.pop().map(|(val, _)| val) + + self.check_invariants(); } /// Returns the number of elements stored in the sliding window. - #[inline] + #[cfg(test)] pub fn len(&self) -> usize { - self.push_stack.len() + self.pop_stack.len() + (self.push_seq - self.pop_seq) as usize } /// Returns `true` if the moving window contains no elements. #[inline] pub fn is_empty(&self) -> bool { - self.len() == 0 + self.push_seq == self.pop_seq } - /// Heap bytes owned by the two stack buffers plus each stored `T`'s + /// Heap bytes owned by the deque plus each stored `T`'s /// heap payload as reported by `elem_heap`. Excludes `size_of::()`. #[inline] fn heap_size(&self, elem_heap: impl Fn(&T) -> usize) -> usize { - moving_stacks_heap_size(&self.push_stack, &self.pop_stack, elem_heap) + moving_deque_heap_size(&self.deque, elem_heap) } } @@ -1251,13 +1214,12 @@ mod tests { #[test] fn moving_min_max_heap_size_i32() { // Fixed-width `T` has no per-element heap payload, so `heap_size` - // reports exactly the two stack buffers' capacity in bytes. + // reports exactly the buffer's capacity in bytes. let mut moving_min = MovingMin::::with_capacity(4); let mut moving_max = MovingMax::::with_capacity(4); let elem = |_: &i32| 0; - // Both stacks are `with_capacity(4)`, so total slots = 8. - let buffer_only = 2 * 4 * size_of::<(i32, i32)>(); + let buffer_only = moving_min.deque.capacity() * size_of::<(u64, i32)>(); assert_eq!(moving_min.heap_size(elem), buffer_only); assert_eq!(moving_max.heap_size(elem), buffer_only); @@ -1272,8 +1234,6 @@ mod tests { #[test] fn moving_min_max_heap_size_counts_elems() { - // Each buffered slot is a `(T, T)` pair, so a stored element is - // visited twice by `heap_size` — mirrors two independent `Clone`s. let mut moving_min = MovingMin::::with_capacity(2); let mut moving_max = MovingMax::::with_capacity(2); let elem = |s: &String| s.capacity(); @@ -1281,15 +1241,65 @@ mod tests { moving_min.push("abcdef".to_string()); moving_max.push("abcdef".to_string()); - // Both `push_stack` and `pop_stack` allocate `capacity` slots. - let buffers = 2 * 2 * size_of::<(String, String)>(); - // 2 slots per stored element (value + running extremum) times the - // per-element heap payload from `elem`. - let elems = 2 * 6; + let buffers = moving_min.deque.capacity() * size_of::<(u64, String)>(); + let elems = 6; assert_eq!(moving_min.heap_size(elem), buffers + elems); assert_eq!(moving_max.heap_size(elem), buffers + elems); } + #[test] + fn test_moving_min_max_empty_pop() { + let mut moving_min = MovingMin::::new(); + moving_min.pop(); // empty pop is a no-op + assert_eq!(moving_min.len(), 0); + assert!(moving_min.is_empty()); + // Verify it still works correctly after empty pop + moving_min.push(10); + moving_min.push(20); + assert_eq!(moving_min.min(), Some(&10)); + moving_min.pop(); + assert_eq!(moving_min.min(), Some(&20)); + + let mut moving_max = MovingMax::::new(); + moving_max.pop(); // empty pop is a no-op + assert_eq!(moving_max.len(), 0); + assert!(moving_max.is_empty()); + // Verify it still works correctly after empty pop + moving_max.push(20); + moving_max.push(10); + assert_eq!(moving_max.max(), Some(&20)); + moving_max.pop(); + assert_eq!(moving_max.max(), Some(&10)); + } + + #[test] + fn test_moving_min_max_duplicate_heavy() { + let mut moving_min = MovingMin::::new(); + let mut moving_max = MovingMax::::new(); + + // Push duplicates + for _ in 0..5 { + moving_min.push(5); + moving_max.push(5); + } + + assert_eq!(moving_min.len(), 5); + assert_eq!(moving_max.len(), 5); + + // Ensure min/max query works and we can pop all duplicates correctly + for i in (1..=5).rev() { + assert_eq!(moving_min.len(), i); + assert_eq!(moving_max.len(), i); + assert_eq!(moving_min.min(), Some(&5)); + assert_eq!(moving_max.max(), Some(&5)); + moving_min.pop(); + moving_max.pop(); + } + + assert!(moving_min.is_empty()); + assert!(moving_max.is_empty()); + } + #[test] fn test_min_max_coerce_types() { // the coerced types is same with input types diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index df268792d8cc7..6097c8dc717df 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -884,6 +884,16 @@ let stream = csv_format.read_to_delimited_chunks_from_stream(input); let plan = deserialize_bytes(&proto_bytes)?; ``` +### `MovingMin` and `MovingMax` changed to `pub(crate)` + +`MovingMin` and `MovingMax` in `datafusion_functions_aggregate::min_max` have been changed from `pub` to `pub(crate)` visibility as they are internal helper data structures for DataFusion's sliding window aggregators. + +**Who is affected:** + +- Code that directly imported `MovingMin` or `MovingMax` from `datafusion_functions_aggregate`. Standard SQL window functions (`MIN(...) OVER (...)` / `MAX(...) OVER (...)`) are unaffected. + +See [PR #23827](https://github.com/apache/datafusion/pull/23827) for details. + ### `MSRV` updated to 1.94.0 The Minimum Supported Rust Version (MSRV) has been updated to [`1.94.0`]. From 9cad8617530ac40de375e654cb8cd8685fe45a9c Mon Sep 17 00:00:00 2001 From: jeroenflvr <33448728+jeroenflvr@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:46:29 +0000 Subject: [PATCH 042/109] docs: add datapress to known users list (#23919) ## Which issue does this PR close? NA ## Rationale for this change Adds [datapress](https://docs.datap-rs.org) to the list of known users in the documentation. ## What changes are included in this PR? NA ## Are these changes tested? NA ## Are there any user-facing changes? NA --- docs/source/user-guide/introduction.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/user-guide/introduction.md b/docs/source/user-guide/introduction.md index bf6809e1e9967..2d072b07197ae 100644 --- a/docs/source/user-guide/introduction.md +++ b/docs/source/user-guide/introduction.md @@ -103,6 +103,7 @@ Here are some active projects using DataFusion: - [Comet](https://github.com/apache/datafusion-comet) Apache Spark native query execution plugin - [Cube Store] Cube’s universal semantic layer platform is the next evolution of OLAP technology for AI, BI, spreadsheets, and embedded analytics - [datafusion-dft](https://github.com/datafusion-contrib/datafusion-dft) Batteries included CLI, TUI, and server implementations for DataFusion. +- [datapress](https://docs.datap-rs.org) An opinionated small and fast data server on parquet and delta tables. - [dbt Fusion engine](https://github.com/dbt-labs/dbt-fusion) The dbt Fusion engine, written in Rust, designed for speed and correctness with a native SQL understanding across DWH SQL dialects. - [delta-rs] Native Rust implementation of Delta Lake - [EDB Postgres Lakehouse] built with [Seafowl] From 9ab20683ecd2aba98f302c2d9e971804b637f8d1 Mon Sep 17 00:00:00 2001 From: H <25857835+HairstonE@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:01:00 -0400 Subject: [PATCH 043/109] fix: eliminate group by constant empty input (#22132) ## Which issue does this PR close? - Closes #11748 ## Rationale for this change `EliminateGroupByConstant` removes GROUP BY expressions that are constants. If all of the GROUP BY expressions are constants, eliminating all of them results in converting a grouped aggregate into an ungrouped (global) aggregate. This changes the semantics of the query: a grouped aggregate query on an empty input returns zero rows, whereas an ungrouped aggregate query produces a single row. ## What changes are included in this PR? `EliminateGroupByConstant` now declines to eliminate constant GROUP BY expressions, if doing so would result in removing all of the grouping expressions. ## Are these changes tested? Yes. Sqllogictest reproducer for the original issue, updated unit test and optimizer SLT expectations. ## Are there any user-facing changes? Queries with all-constant GROUP BY may now return fewer (correct) rows. --- .../src/eliminate_group_by_constant.rs | 19 ++++++++++-------- .../sqllogictest/test_files/aggregate.slt | 17 ++++++++++++++++ .../optimizer_group_by_constant.slt | 20 +++++++++---------- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/datafusion/optimizer/src/eliminate_group_by_constant.rs b/datafusion/optimizer/src/eliminate_group_by_constant.rs index e21241ba7d993..f0efe96668dba 100644 --- a/datafusion/optimizer/src/eliminate_group_by_constant.rs +++ b/datafusion/optimizer/src/eliminate_group_by_constant.rs @@ -64,10 +64,14 @@ impl OptimizerRule for EliminateGroupByConstant { .group_expr .iter() .partition(|expr| is_redundant_group_expr(expr, &group_by_columns)); - - if redundant.is_empty() - || (required.is_empty() && aggregate.aggr_expr.is_empty()) - { + // Return now if no simplification can be done. We also bail out + // if applying the optimization would eliminate all of the + // grouping expressions (e.g., GROUP BY on only constant + // expressions): this would turn a grouped aggregate into an + // ungrouped aggregate, which changes query semantics (grouped + // aggregates produce an empty result set on an empty input, + // whereas ungrouped aggregates return a single row). + if redundant.is_empty() || required.is_empty() { return Ok(Transformed::no(LogicalPlan::Aggregate(aggregate))); } @@ -221,16 +225,15 @@ mod tests { } #[test] - fn test_eliminate_constant() -> Result<()> { + fn test_no_op_only_constant_with_aggregate() -> Result<()> { let scan = test_table_scan()?; let plan = LogicalPlanBuilder::from(scan) .aggregate(vec![lit("test"), lit(123u32)], vec![count(col("c"))])? .build()?; assert_optimized_plan_equal!(plan, @r#" - Projection: Utf8("test"), UInt32(123), count(test.c) - Aggregate: groupBy=[[]], aggr=[[count(test.c)]] - TableScan: test + Aggregate: groupBy=[[Utf8("test"), UInt32(123)]], aggr=[[count(test.c)]] + TableScan: test "#) } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 8e7a9639f55f8..36677092ae374 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -8047,6 +8047,23 @@ CREATE TABLE t1(v1 int); statement error DataFusion error: Error during planning: Aggregate functions are not allowed in the WHERE clause. Consider using HAVING instead SELECT v1 FROM t1 WHERE ((count(v1) % 1) << 1) > 0; +# issue: https://github.com/apache/datafusion/issues/11748 +query R +SELECT AVG(v1) FROM t1 GROUP BY false HAVING false; +---- + +query R +SELECT AVG(v1) FROM t1 GROUP BY false; +---- + +statement ok +INSERT INTO t1 VALUES (1), (2), (3); + +query R +SELECT AVG(v1) FROM t1 GROUP BY false; +---- +2 + statement ok DROP TABLE t1; diff --git a/datafusion/sqllogictest/test_files/optimizer_group_by_constant.slt b/datafusion/sqllogictest/test_files/optimizer_group_by_constant.slt index da1e7de22bb7a..9df55512413f3 100644 --- a/datafusion/sqllogictest/test_files/optimizer_group_by_constant.slt +++ b/datafusion/sqllogictest/test_files/optimizer_group_by_constant.slt @@ -60,10 +60,9 @@ FROM test_table t group by 1, 2, 3 ---- logical_plan -01)Projection: Int64(123), Int64(456), Int64(789), count(Int64(1)), avg(t.c12) -02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1)), avg(t.c12)]] -03)----SubqueryAlias: t -04)------TableScan: test_table projection=[c12] +01)Aggregate: groupBy=[[Int64(123), Int64(456), Int64(789)]], aggr=[[count(Int64(1)), avg(t.c12)]] +02)--SubqueryAlias: t +03)----TableScan: test_table projection=[c12] query TT EXPLAIN @@ -72,8 +71,8 @@ FROM test_table t GROUP BY 1, 2 ---- logical_plan -01)Projection: Date32("2023-05-04") AS dt, Boolean(true) AS today_filter, count(Int64(1)) -02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] +01)Projection: to_date(Utf8("2023-05-04")) AS dt, date_part(Utf8("DAY"),now()) < Int64(1000) AS today_filter, count(Int64(1)) +02)--Aggregate: groupBy=[[Date32("2023-05-04") AS to_date(Utf8("2023-05-04")), Boolean(true) AS date_part(Utf8("DAY"),now()) < Int64(1000)]], aggr=[[count(Int64(1))]] 03)----SubqueryAlias: t 04)------TableScan: test_table projection=[] @@ -90,10 +89,9 @@ FROM test_table t GROUP BY 1 ---- logical_plan -01)Projection: Boolean(true) AS NOT date_part(Utf8("MONTH"),now()) BETWEEN Int64(50) AND Int64(60), count(Int64(1)) -02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] -03)----SubqueryAlias: t -04)------TableScan: test_table projection=[] +01)Aggregate: groupBy=[[Boolean(true) AS NOT date_part(Utf8("MONTH"),now()) BETWEEN Int64(50) AND Int64(60)]], aggr=[[count(Int64(1))]] +02)--SubqueryAlias: t +03)----TableScan: test_table projection=[] query TT EXPLAIN @@ -119,7 +117,7 @@ logical_plan # Config reset -# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# The SLT runner sets `target_partitions` to 4 instead of using the default, so # reset it explicitly. statement ok set datafusion.execution.target_partitions = 4; From 2028927e1f5fb00f3781eea7cd0f990eddf44639 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Mon, 27 Jul 2026 17:16:23 -0400 Subject: [PATCH 044/109] fix: sliding window `min()` returns wrong value for all-NULL windows (#23874) ## Which issue does this PR close? - Closes #23872 ## Rationale for this change `SlidingMinAccumulator::update_batch` skipped NULL values. This meant that if all the non-NULL values in a window frame were retracted, the window frame would not be empty but the `MovingMin` data structure by the `SlidingMinAccumulator` would not contain any values. This resulted in incorrectly returning a stale non-NULL value for a sliding `min()` over a window frame consisting of only NULL values. Along the way, optimize the min and max sliding window accumulators to make them both more efficient and more symmetric with one another. In the original coding, `SlidingMaxAccumulator` included NULL values but `SlidingMinAccumulator` omitted them, in part because omitting NULLs made the original `retract_batch` implementation more expensive. This PR optimizes `retract_batch`, so we can now use the same scheme for both the min and max sliding accumulators: * Omit NULLs on `update_batch` (this improves on the prior behavior of `max`) * Efficiently account for NULLs in `retract_batch` (this improves on the prior behavior of `min`) * Ensure correct results for all-NULL window frames (this fixes the prior bug in `min`). ## What changes are included in this PR? * Fix bug in `min()` over all-NULL window frames * Optimize `SlidingMinAccumulator::retract_batch` (avoid materializing values just to count NULLs) * Optimize `SlidingMinAccumulator::update_batch` (omit NULLs), also improving symmetry with `min` * Optimize both accumulators to stop caching the current `min` / `max`; this saves a few clones, but perhaps more importantly it is simpler and avoids the risk of inconsistency between the cached value and the underlying `MovingMin` / `MovingMax` data structure ## Are these changes tested? Yes, new tests added. ## Are there any user-facing changes? No, aside from the bug fix. --- datafusion/functions-aggregate/src/min_max.rs | 129 +++++++++++++----- datafusion/sqllogictest/test_files/window.slt | 14 ++ 2 files changed, 111 insertions(+), 32 deletions(-) diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs index b306e7db2f0b7..41643747e8a42 100644 --- a/datafusion/functions-aggregate/src/min_max.rs +++ b/datafusion/functions-aggregate/src/min_max.rs @@ -381,7 +381,8 @@ impl AggregateUDFImpl for Max { #[derive(Debug)] pub struct SlidingMaxAccumulator { - max: ScalarValue, + /// Typed NULL returned when the window contains no non-null values + empty_value: ScalarValue, moving_max: MovingMax, } @@ -389,30 +390,38 @@ impl SlidingMaxAccumulator { /// new max accumulator pub fn try_new(datatype: &DataType) -> Result { Ok(Self { - max: ScalarValue::try_from(datatype)?, + empty_value: ScalarValue::try_from(datatype)?, moving_max: MovingMax::::new(), }) } + + fn current_max(&self) -> ScalarValue { + match self.moving_max.max() { + Some(res) => res.clone(), + None => self.empty_value.clone(), + } + } } impl Accumulator for SlidingMaxAccumulator { fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { for idx in 0..values[0].len() { let val = ScalarValue::try_from_array(&values[0], idx)?; - self.moving_max.push(val); - } - if let Some(res) = self.moving_max.max() { - self.max = res.clone(); + if !val.is_null() { + self.moving_max.push(val); + } } Ok(()) } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - for _idx in 0..values[0].len() { - (self.moving_max).pop(); - } - if let Some(res) = self.moving_max.max() { - self.max = res.clone(); + // We assume that values are retracted in the order they were added, so + // the retracted values must be the oldest elements of `moving_max`. + // NULLs are never pushed, so be sure to only pop once per non-NULL + // value. + let valid_count = values[0].len() - values[0].logical_null_count(); + for _ in 0..valid_count { + self.moving_max.pop(); } Ok(()) } @@ -422,11 +431,11 @@ impl Accumulator for SlidingMaxAccumulator { } fn state(&mut self) -> Result> { - Ok(vec![self.max.clone()]) + Ok(vec![self.current_max()]) } fn evaluate(&mut self) -> Result { - Ok(self.max.clone()) + Ok(self.current_max()) } fn supports_retract_batch(&self) -> bool { @@ -434,8 +443,8 @@ impl Accumulator for SlidingMaxAccumulator { } fn size(&self) -> usize { - size_of_val(self) - size_of_val(&self.max) - + self.max.size() + size_of_val(self) - size_of_val(&self.empty_value) + + self.empty_value.size() + self.moving_max.heap_size(|sv| sv.size() - size_of_val(sv)) } } @@ -667,22 +676,30 @@ impl AggregateUDFImpl for Min { #[derive(Debug)] pub struct SlidingMinAccumulator { - min: ScalarValue, + /// Typed NULL returned when the window contains no non-null values + empty_value: ScalarValue, moving_min: MovingMin, } impl SlidingMinAccumulator { pub fn try_new(datatype: &DataType) -> Result { Ok(Self { - min: ScalarValue::try_from(datatype)?, + empty_value: ScalarValue::try_from(datatype)?, moving_min: MovingMin::::new(), }) } + + fn current_min(&self) -> ScalarValue { + match self.moving_min.min() { + Some(res) => res.clone(), + None => self.empty_value.clone(), + } + } } impl Accumulator for SlidingMinAccumulator { fn state(&mut self) -> Result> { - Ok(vec![self.min.clone()]) + Ok(vec![self.current_min()]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { @@ -692,21 +709,17 @@ impl Accumulator for SlidingMinAccumulator { self.moving_min.push(val); } } - if let Some(res) = self.moving_min.min() { - self.min = res.clone(); - } Ok(()) } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - for idx in 0..values[0].len() { - let val = ScalarValue::try_from_array(&values[0], idx)?; - if !val.is_null() { - (self.moving_min).pop(); - } - } - if let Some(res) = self.moving_min.min() { - self.min = res.clone(); + // We assume that values are retracted in the order they were added, so + // the retracted values must be the oldest elements of `moving_min`. + // NULLs are never pushed, so be sure to only pop once per non-NULL + // value. + let valid_count = values[0].len() - values[0].logical_null_count(); + for _ in 0..valid_count { + self.moving_min.pop(); } Ok(()) } @@ -716,7 +729,7 @@ impl Accumulator for SlidingMinAccumulator { } fn evaluate(&mut self) -> Result { - Ok(self.min.clone()) + Ok(self.current_min()) } fn supports_retract_batch(&self) -> bool { @@ -724,8 +737,8 @@ impl Accumulator for SlidingMinAccumulator { } fn size(&self) -> usize { - size_of_val(self) - size_of_val(&self.min) - + self.min.size() + size_of_val(self) - size_of_val(&self.empty_value) + + self.empty_value.size() + self.moving_min.heap_size(|sv| sv.size() - size_of_val(sv)) } } @@ -1193,6 +1206,58 @@ mod tests { Ok(()) } + #[test] + fn sliding_min_all_null_window() -> Result<()> { + let mut min_acc = SlidingMinAccumulator::try_new(&DataType::Int32)?; + + let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None])); + min_acc.update_batch(&[Arc::clone(&values)])?; + assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(3))); + + // Retract `3`; the window now contains only the NULL + let retracted: ArrayRef = Arc::new(Int32Array::from(vec![Some(3)])); + min_acc.retract_batch(&[Arc::clone(&retracted)])?; + assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(None)); + + // A subsequent non-null value must be picked up again + let update: ArrayRef = Arc::new(Int32Array::from(vec![Some(7)])); + min_acc.update_batch(&[Arc::clone(&update)])?; + assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(7))); + + // Retracting the NULL row must not pop the remaining value + let null_row: ArrayRef = Arc::new(Int32Array::from(vec![None::])); + min_acc.retract_batch(&[Arc::clone(&null_row)])?; + assert_eq!(min_acc.evaluate()?, ScalarValue::Int32(Some(7))); + + Ok(()) + } + + #[test] + fn sliding_max_all_null_window() -> Result<()> { + let mut max_acc = SlidingMaxAccumulator::try_new(&DataType::Int32)?; + + let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(3), None])); + max_acc.update_batch(&[Arc::clone(&values)])?; + assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(3))); + + // Retract `3`; the window now contains only the NULL + let retracted: ArrayRef = Arc::new(Int32Array::from(vec![Some(3)])); + max_acc.retract_batch(&[Arc::clone(&retracted)])?; + assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(None)); + + // A subsequent non-null value must be picked up again + let update: ArrayRef = Arc::new(Int32Array::from(vec![Some(7)])); + max_acc.update_batch(&[Arc::clone(&update)])?; + assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(7))); + + // Retracting the NULL row must not disturb the remaining value + let null_row: ArrayRef = Arc::new(Int32Array::from(vec![None::])); + max_acc.retract_batch(&[Arc::clone(&null_row)])?; + assert_eq!(max_acc.evaluate()?, ScalarValue::Int32(Some(7))); + + Ok(()) + } + #[test] fn moving_min_tests() -> Result<()> { moving_min_i32(100, 10)?; diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt index cbbd9b74dfc00..fd477a3386b69 100644 --- a/datafusion/sqllogictest/test_files/window.slt +++ b/datafusion/sqllogictest/test_files/window.slt @@ -6842,3 +6842,17 @@ DROP TABLE issue_20194_t1; statement ok DROP TABLE issue_20194_t2; + +# Sliding-window MIN/MAX over a frame whose non-NULL values have all been +# retracted should yield NULL. +query IIII +SELECT id, x, + MIN(x) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS min_x, + MAX(x) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS max_x +FROM (VALUES (1, 3), (2, NULL), (3, NULL), (4, 7)) t(id, x) +ORDER BY id +---- +1 3 3 3 +2 NULL 3 3 +3 NULL NULL NULL +4 7 7 7 From daacd72a17eb5ecb10f0b649a8db4c6d6e39baca Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Mon, 27 Jul 2026 20:44:07 -0400 Subject: [PATCH 045/109] chore: Squelch "unused code" warning (#23924) ## Which issue does this PR close? - N/A ## Rationale for this change ``` $ cargo test -p datafusion-functions-aggregate [...] warning: associated function `from_parts` is never used --> datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs:464:8 [...] ``` Fix this by appropriately gating the compilation of `from_parts`. ## What changes are included in this PR? * Squelch unused code warning ## Are these changes tested? Yes. ## Are there any user-facing changes? No. --- datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs index f97aa2cd73974..0fd0ad93bf94a 100644 --- a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs +++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs @@ -461,6 +461,7 @@ impl DynamicFilterPhysicalExpr { /// Rebuild a `DynamicFilterPhysicalExpr` from its stored parts. Used by /// proto deserialization. + #[cfg(any(test, feature = "proto"))] fn from_parts( children: Vec>, remapped_children: Option>>, From 0b07e58f8944a5736239fbd2af13a15eae8864d3 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 28 Jul 2026 10:51:33 +0800 Subject: [PATCH 046/109] fix: correct percentile_cont(DISTINCT) accumulation and sliding-window retract (#23913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23912. ## Rationale for this change `DistinctPercentileContAccumulator` reused the shared set-based `GenericDistinctBuffer`, which doesn't fit it: (1) the buffer asserts a single input column but `percentile_cont` passes two (value + percentile), so every `percentile_cont(DISTINCT ...)` panicked; (2) the buffer is a plain `HashSet` with no multiplicity, so sliding-window `retract_batch` dropped a value while duplicates were still in the frame. ## What changes are included in this PR? - Replace the shared buffer in this accumulator with a per-accumulator `HashMap` count map: `update_batch` reads only the value column and increments; `retract_batch` decrements and removes a key only at zero; `state`/`merge_batch` keep the same List state shape. Other `GenericDistinctBuffer` users are untouched. - Regression tests in `aggregate.slt` for the plain distinct query and the sliding-window duplicate-retract case. ## Are these changes tested? Yes — new regression tests; the full `aggregate.slt` suite passes. ## Are there any user-facing changes? `percentile_cont(DISTINCT ...)` now works instead of panicking, and returns correct results in sliding windows. --- .../src/percentile_cont.rs | 59 +++++++++++++++---- .../sqllogictest/test_files/aggregate.slt | 27 +++++++++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs index cfab1303028ad..4b5e892fb5cdf 100644 --- a/datafusion/functions-aggregate/src/percentile_cont.rs +++ b/datafusion/functions-aggregate/src/percentile_cont.rs @@ -39,7 +39,7 @@ use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator; use crate::min_max::{max_udaf, min_udaf}; use datafusion_common::{ Result, ScalarValue, exec_datafusion_err, internal_datafusion_err, - utils::take_function_args, + utils::{SingleRowListArrayBuilder, take_function_args}, }; use datafusion_expr::utils::format_state_name; use datafusion_expr::{ @@ -54,7 +54,7 @@ use datafusion_expr::{ }; use datafusion_functions_aggregate_common::aggregate::groups_accumulator::accumulate::accumulate; use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::filtered_null_mask; -use datafusion_functions_aggregate_common::utils::{GenericDistinctBuffer, Hashable}; +use datafusion_functions_aggregate_common::utils::Hashable; use datafusion_macros::user_doc; use crate::utils::validate_percentile_expr; @@ -662,16 +662,24 @@ where } } +/// Sliding-window–capable accumulator for `percentile_cont(DISTINCT ...)`. +/// +/// Distinct values are tracked with a per-value multiplicity count (how many +/// rows currently in the window carry that value) rather than a plain set, so +/// that `retract_batch` only drops a value once *all* of its occurrences have +/// left the window frame. The percentile is then computed over the set of keys +/// with a positive count. #[derive(Debug)] struct DistinctPercentileContAccumulator { - distinct_values: GenericDistinctBuffer, + /// Distinct value -> number of in-window rows carrying it. + counts: HashMap, usize>, percentile: f64, } impl DistinctPercentileContAccumulator { fn new(percentile: f64) -> Self { Self { - distinct_values: GenericDistinctBuffer::new(T::DATA_TYPE), + counts: HashMap::default(), percentile, } } @@ -684,26 +692,50 @@ where f64: AsPrimitive, { fn state(&mut self) -> Result> { - self.distinct_values.state() + // Emit the distinct keys as a single List scalar, matching the state + // shape declared in `state_fields` (a List of the input type). Counts + // are window-local bookkeeping and are intentionally not serialized: + // cross-partition merges only need the distinct key set. + let arr = Arc::new( + PrimitiveArray::::from_iter_values(self.counts.keys().map(|v| v.0)) + .with_data_type(T::DATA_TYPE), + ); + Ok(vec![ + SingleRowListArrayBuilder::new(arr).build_list_scalar(), + ]) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - self.distinct_values.update_batch(values) + // `values` may carry extra argument columns (e.g. the percentile + // literal); only the first column holds the aggregated values. + let arr = values[0].as_primitive::(); + for value in arr.iter().flatten() { + *self.counts.entry(Hashable(value)).or_default() += 1; + } + Ok(()) } fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { - self.distinct_values.merge_batch(states) + let list = states[0].as_list::(); + for values in list.iter().flatten() { + let arr = values.as_primitive::(); + for value in arr.iter().flatten() { + *self.counts.entry(Hashable(value)).or_default() += 1; + } + } + Ok(()) } fn evaluate(&mut self) -> Result { - let mut values: Vec = - self.distinct_values.values.iter().map(|v| v.0).collect(); + let mut values: Vec = self.counts.keys().map(|v| v.0).collect(); let value = calculate_percentile::(&mut values, self.percentile); ScalarValue::new_primitive::(value, &T::DATA_TYPE) } fn size(&self) -> usize { - size_of_val(self) + self.distinct_values.size() + size_of_val(self) + + self.counts.capacity() + * (size_of::>() + size_of::()) } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { @@ -713,7 +745,12 @@ where let arr = values[0].as_primitive::(); for value in arr.iter().flatten() { - self.distinct_values.values.remove(&Hashable(value)); + if let Some(count) = self.counts.get_mut(&Hashable(value)) { + *count -= 1; + if *count == 0 { + self.counts.remove(&Hashable(value)); + } + } } Ok(()) } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 36677092ae374..f6380c9a5f06d 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1104,6 +1104,33 @@ ORDER BY tags, timestamp; statement ok DROP TABLE median_window_test; +# Regression: percentile_cont(DISTINCT ...) used to forward the extra +# percentile-argument column into the distinct-values buffer (which asserts a +# single input array), panicking on every distinct query. Plain aggregate: +statement ok +CREATE TABLE distinct_pct(id INT, x DOUBLE) AS VALUES + (1, 5), (2, 5), (3, 9); + +query R +SELECT percentile_cont(DISTINCT x, 0.5) FROM distinct_pct; +---- +7 + +# Regression: distinct sliding-window percentile must count value multiplicity +# on retract. Row 3's frame is {5, 9}; the row-1 `5` leaves the frame but the +# row-2 `5` remains, so the distinct set is still {5, 9} (median 7), not {9}. +query IR +SELECT id, percentile_cont(DISTINCT x, 0.5) + OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM distinct_pct; +---- +1 5 +2 5 +3 7 + +statement ok +DROP TABLE distinct_pct; + query RT select approx_median(arrow_cast(col_f32, 'Float16')), arrow_typeof(approx_median(arrow_cast(col_f32, 'Float16'))) from median_table; ---- From bb75d925cb7a6b8e1c5dc22bead85efc49754fe3 Mon Sep 17 00:00:00 2001 From: zhigang Date: Tue, 28 Jul 2026 13:47:07 +0800 Subject: [PATCH 047/109] fix: support parentheses for negative decimal formatting (#23718) ## Which issue does this PR close? - Closes #23717. ## Rationale for this change Spark and Java format negative numeric values with parentheses when the `(` flag is present. The decimal formatting path always emitted a minus sign because it ignored `negative_in_parentheses`, while the floating-point path already handled the flag. This made `format_string` inconsistent across numeric input types. ## What changes are included in this PR? - Add a closing suffix when a negative decimal uses parentheses formatting. - Include the suffix when calculating width, left alignment, and zero padding. - Add regression coverage for grouped negative decimals with and without an explicit width. ## Are these changes tested? Yes. The following checks pass locally: - `cargo fmt --all -- --check` - `cargo clippy --all-targets --all-features -- -D warnings` - `cargo test -p datafusion-spark --lib` - The full workspace test command from the contributor guide with `avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption` enabled ## Are there any user-facing changes? Yes. Spark-compatible formatting of negative decimal values now honors the parentheses flag. There are no public API or breaking changes. --- .../src/function/string/format_string.rs | 55 ++++++++++++++----- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/datafusion/spark/src/function/string/format_string.rs b/datafusion/spark/src/function/string/format_string.rs index 68b8fe52338d4..60b6d37e55965 100644 --- a/datafusion/spark/src/function/string/format_string.rs +++ b/datafusion/spark/src/function/string/format_string.rs @@ -1982,6 +1982,7 @@ impl ConversionSpecifier { self.validate_grouping_separator()?; let mut prefix = String::new(); + let mut suffix = String::new(); let upper = self.conversion_type.is_upper(); // Parse as BigDecimal @@ -1991,15 +1992,16 @@ impl ConversionSpecifier { let decimal = BigDecimal::from_bigint(decimal, scale); // Handle sign - // TODO: `negative_in_parentheses` (the `(` flag) is not implemented here. - // Java/Spark wrap negative values in parentheses when this flag is set - // (e.g. `%(,.2f` with -1234.5 → "(1,234.50)"), but this path always - // uses a minus sign. See `format_float` for the correct implementation. let is_negative = decimal.sign() == Sign::Minus; let abs_decimal = decimal.abs(); if is_negative { - prefix.push('-'); + if self.negative_in_parentheses { + prefix.push('('); + suffix.push(')'); + } else { + prefix.push('-'); + } } else if self.space_sign { prefix.push(' '); } else if self.force_sign { @@ -2078,23 +2080,25 @@ impl ConversionSpecifier { let NumericParam::Literal(width) = self.width else { writer.push_str(&prefix); writer.push_str(&number); + writer.push_str(&suffix); return Ok(()); }; if self.left_adj { - let mut full_num = prefix + &number; + let mut full_num = prefix + &number + &suffix; while full_num.len() < width as usize { full_num.push(' '); } writer.push_str(&full_num); } else if self.zero_pad { - while prefix.len() + number.len() < width as usize { + while prefix.len() + number.len() + suffix.len() < width as usize { prefix.push('0'); } writer.push_str(&prefix); writer.push_str(&number); + writer.push_str(&suffix); } else { - let mut full_num = prefix + &number; + let mut full_num = prefix + &number + &suffix; while full_num.len() < width as usize { full_num = " ".to_owned() + &full_num; } @@ -2372,7 +2376,7 @@ mod tests { use super::*; use crate::function::utils::test::test_scalar_function; use arrow::array::StringArray; - use arrow::datatypes::DataType::Utf8; + use arrow::datatypes::{DataType::Utf8, i256}; #[test] fn test_format_string_nullability() -> Result<()> { @@ -2896,17 +2900,42 @@ mod tests { #[test] fn test_grouping_separator_parentheses_decimal() -> Result<()> { - // %(,15.2f on negative decimal — format_decimal ignores negative_in_parentheses, - // always uses '-'. Check TODO in fn format_decimal + test_scalar_function!( + FormatStringFunc::new(), + vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some("%(,.2f".to_string()))), + ColumnarValue::Scalar(ScalarValue::Decimal128(Some(-123450), 10, 2)), + ], + Ok(Some("(1,234.50)")), + &str, + Utf8, + StringArray + ); + + test_scalar_function!( + FormatStringFunc::new(), + vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some("%(,.2f".to_string()))), + ColumnarValue::Scalar(ScalarValue::Decimal256( + Some(i256::from(-123450)), + 10, + 2, + )), + ], + Ok(Some("(1,234.50)")), + &str, + Utf8, + StringArray + ); + // Java: String.format("%(,15.2f", -1234.5) → " (1,234.50)" - // Ours: " -1,234.50" (minus sign, no parens) test_scalar_function!( FormatStringFunc::new(), vec![ ColumnarValue::Scalar(ScalarValue::Utf8(Some("%(,15.2f".to_string()))), ColumnarValue::Scalar(ScalarValue::Decimal128(Some(-123450), 10, 2)), ], - Ok(Some(" -1,234.50")), + Ok(Some(" (1,234.50)")), &str, Utf8, StringArray From 1b32517ace9bca3dbc0f65072798aeceeb9ea076 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:07:03 +0200 Subject: [PATCH 048/109] Add name filter to metrics (#23719) ## Which issue does this PR close? - Closes #. ## Rationale for this change Precedes https://github.com/apache/datafusion/pull/23720. While displaying a plan with metrics, allow filtering them by name, not just by category or type. This is very useful when writing snapshot tests where only a specific metrics needs to be asserted, without other unrelated metrics polluting the snapshot assertion. Regardless of what happens with https://github.com/apache/datafusion/pull/23720, I think this PR is still worth it, as it allows creating some really nice `insta` tests asserting runtime properties reliably by just cherry picking the runtime metrics relevant for that specific test. This is relevant not only within DataFusion codebase, but also for other people's codebases using DataFusion and relying on `insta` for snapshot testing, See an example of this here: https://github.com/apache/datafusion/pull/23720/changes#diff-8281405e117428077c19c07afbbe59fed303a3460096e958f761d34f0899b618 ## What changes are included in this PR? While displaying metrics, allows filtering them by name ## Are these changes tested? Yes, by a new small test. ## Are there any user-facing changes? People using the metrics display API will be able to filter metrics by name. --- .../physical-expr-common/src/metrics/mod.rs | 40 +++++++++ datafusion/physical-plan/src/display.rs | 87 +++++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/datafusion/physical-expr-common/src/metrics/mod.rs b/datafusion/physical-expr-common/src/metrics/mod.rs index d6048a0fcd338..146c039c75f6a 100644 --- a/datafusion/physical-expr-common/src/metrics/mod.rs +++ b/datafusion/physical-expr-common/src/metrics/mod.rs @@ -418,6 +418,21 @@ impl MetricsSet { .collect::>(); Self { metrics } } + + /// Returns a new `MetricsSet` filtered by metric name. + /// Only metrics with the names appearing the list will be kept. + pub fn filter_by_names(self, names: &[String]) -> Self { + if names.is_empty() { + return Self { metrics: vec![] }; + } + + let metrics = self + .metrics + .into_iter() + .filter(|metric| names.iter().any(|name| name == metric.value().name())) + .collect::>(); + Self { metrics } + } } impl Display for MetricsSet { @@ -966,4 +981,29 @@ mod tests { metric_names(&metrics) ); } + + #[test] + fn test_filter_by_names() { + let metrics = ExecutionPlanMetricsSet::new(); + MetricBuilder::new(&metrics).output_rows(0); + MetricBuilder::new(&metrics).counter("custom_counter", 0); + + assert!( + metrics + .clone_inner() + .filter_by_names(&[]) + .iter() + .next() + .is_none() + ); + + let names = vec!["output_rows".to_string()]; + let filtered = metrics.clone_inner().filter_by_names(&names); + + assert_eq!(filtered.iter().count(), 1); + assert_eq!( + filtered.iter().next().unwrap().value().name(), + "output_rows" + ); + } } diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs index 6a4d09057bec9..34493a5f51742 100644 --- a/datafusion/physical-plan/src/display.rs +++ b/datafusion/physical-plan/src/display.rs @@ -129,6 +129,9 @@ pub struct DisplayableExecutionPlan<'a> { /// Optional filter by semantic category (rows / bytes / timing). /// `None` means show all categories; `Some(vec![])` means plan-only. metric_categories: Option>, + /// Optional filter by metric names. Only metric names in this list + /// will be rendered. + metric_names: Option>, // (TreeRender) Maximum total width of the rendered tree tree_maximum_render_width: usize, /// Optional summary totals (currently only used by `pgjson`) — the total @@ -159,6 +162,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: false, metric_types: Self::default_metric_types(), metric_categories: None, + metric_names: None, tree_maximum_render_width: 240, summary: None, } @@ -175,6 +179,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: false, metric_types: Self::default_metric_types(), metric_categories: None, + metric_names: None, tree_maximum_render_width: 240, summary: None, } @@ -191,6 +196,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: false, metric_types: Self::default_metric_types(), metric_categories: None, + metric_names: None, tree_maximum_render_width: 240, summary: None, } @@ -234,6 +240,18 @@ impl<'a> DisplayableExecutionPlan<'a> { self } + /// Specify which metric names to include. + /// + /// - An empty vector means plan-only — suppress all metrics. + /// - `vec!["metric_1"]` means show only the metric named `metric_1`. + /// + /// Name filtering is intersected with other types of filters, like metric + /// category and metric type. + pub fn set_metric_names(mut self, metric_names: Vec) -> Self { + self.metric_names = Some(metric_names); + self + } + /// Set the maximum render width for the tree format pub fn set_tree_maximum_render_width(mut self, width: usize) -> Self { self.tree_maximum_render_width = width; @@ -279,6 +297,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: bool, metric_types: Vec, metric_categories: Option>, + metric_names: Option>, } impl fmt::Display for Wrapper<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { @@ -291,6 +310,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: &self.metric_types, metric_categories: self.metric_categories.as_deref(), + metric_names: self.metric_names.as_deref(), }; accept(self.plan, &mut visitor) } @@ -303,6 +323,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: self.metric_types.clone(), metric_categories: self.metric_categories.clone(), + metric_names: self.metric_names.clone(), } } @@ -324,6 +345,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_statistics: bool, metric_types: Vec, metric_categories: Option>, + metric_names: Option>, } impl fmt::Display for Wrapper<'_> { fn fmt(&self, f: &mut Formatter) -> fmt::Result { @@ -336,6 +358,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_statistics: self.show_statistics, metric_types: &self.metric_types, metric_categories: self.metric_categories.as_deref(), + metric_names: self.metric_names.as_deref(), graphviz_builder: GraphvizBuilder::default(), parents: Vec::new(), }; @@ -355,6 +378,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_statistics: self.show_statistics, metric_types: self.metric_types.clone(), metric_categories: self.metric_categories.clone(), + metric_names: self.metric_names.clone(), } } @@ -403,6 +427,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: bool, metric_types: Vec, metric_categories: Option>, + metric_names: Option>, summary: Option, } impl fmt::Display for Wrapper<'_> { @@ -413,6 +438,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: &self.metric_types, metric_categories: self.metric_categories.as_deref(), + metric_names: self.metric_names.as_deref(), objects: HashMap::new(), parent_ids: Vec::new(), next_id: 0, @@ -446,6 +472,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: self.metric_types.clone(), metric_categories: self.metric_categories.clone(), + metric_names: self.metric_names.clone(), summary: self.summary, } } @@ -460,6 +487,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: bool, metric_types: Vec, metric_categories: Option>, + metric_names: Option>, } impl fmt::Display for Wrapper<'_> { @@ -473,6 +501,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: &self.metric_types, metric_categories: self.metric_categories.as_deref(), + metric_names: self.metric_names.as_deref(), }; visitor.pre_visit(self.plan)?; Ok(()) @@ -486,6 +515,7 @@ impl<'a> DisplayableExecutionPlan<'a> { show_schema: self.show_schema, metric_types: self.metric_types.clone(), metric_categories: self.metric_categories.clone(), + metric_names: self.metric_names.clone(), } } @@ -544,6 +574,8 @@ struct IndentVisitor<'a, 'b> { metric_types: &'a [MetricType], /// Optional filter by semantic category (rows / bytes / timing). metric_categories: Option<&'a [MetricCategory]>, + /// Optional filter by metric name. + metric_names: Option<&'a [String]>, } impl ExecutionPlanVisitor for IndentVisitor<'_, '_> { @@ -563,6 +595,9 @@ impl ExecutionPlanVisitor for IndentVisitor<'_, '_> { if let Some(cats) = self.metric_categories { metrics = metrics.filter_by_categories(cats); } + if let Some(names) = self.metric_names { + metrics = metrics.filter_by_names(names); + } write!(self.f, ", metrics=[{metrics}]")?; } else { write!(self.f, ", metrics=[]")?; @@ -574,6 +609,9 @@ impl ExecutionPlanVisitor for IndentVisitor<'_, '_> { if let Some(cats) = self.metric_categories { metrics = metrics.filter_by_categories(cats); } + if let Some(names) = self.metric_names { + metrics = metrics.filter_by_names(names); + } write!(self.f, ", metrics=[{metrics}]")?; } else { write!(self.f, ", metrics=[]")?; @@ -616,6 +654,8 @@ struct GraphvizVisitor<'a, 'b> { metric_types: &'a [MetricType], /// Optional filter by semantic category metric_categories: Option<&'a [MetricCategory]>, + /// Optional filter by metric name. + metric_names: Option<&'a [String]>, graphviz_builder: GraphvizBuilder, /// Used to record parent node ids when visiting a plan. @@ -660,6 +700,9 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> { if let Some(cats) = self.metric_categories { metrics = metrics.filter_by_categories(cats); } + if let Some(names) = self.metric_names { + metrics = metrics.filter_by_names(names); + } format!("metrics=[{metrics}]") } else { "metrics=[]".to_string() @@ -671,6 +714,9 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> { if let Some(cats) = self.metric_categories { metrics = metrics.filter_by_categories(cats); } + if let Some(names) = self.metric_names { + metrics = metrics.filter_by_names(names); + } format!("metrics=[{metrics}]") } else { "metrics=[]".to_string() @@ -729,6 +775,7 @@ struct PgJsonExecutionPlanVisitor<'a> { show_schema: bool, metric_types: &'a [MetricType], metric_categories: Option<&'a [MetricCategory]>, + metric_names: Option<&'a [String]>, objects: HashMap, parent_ids: Vec, next_id: u32, @@ -813,6 +860,12 @@ impl PgJsonExecutionPlanVisitor<'_> { metrics }; + let metrics = if let Some(names) = self.metric_names { + metrics.filter_by_names(names) + } else { + metrics + }; + // Build the Extras bucket, while extracting PG-canonical keys to the // top level. let mut extras = serde_json::Map::new(); @@ -1701,6 +1754,40 @@ mod tests { assert_eq!(root["Actual Rows"].as_u64(), Some(42)); assert_eq!(root["Actual Total Time"].as_f64(), Some(5.0)); assert_eq!(root["Extras"]["output_batches"].as_u64(), Some(7)); + + let metric_names = vec!["output_rows".to_string()]; + for rendered in [ + DisplayableExecutionPlan::with_metrics(plan.as_ref()) + .set_metric_names(metric_names.clone()) + .indent(false) + .to_string(), + DisplayableExecutionPlan::with_full_metrics(plan.as_ref()) + .set_metric_names(metric_names.clone()) + .indent(false) + .to_string(), + DisplayableExecutionPlan::with_metrics(plan.as_ref()) + .set_metric_names(metric_names.clone()) + .graphviz() + .to_string(), + DisplayableExecutionPlan::with_full_metrics(plan.as_ref()) + .set_metric_names(metric_names.clone()) + .graphviz() + .to_string(), + ] { + assert!(rendered.contains("output_rows")); + assert!(!rendered.contains("elapsed_compute")); + assert!(!rendered.contains("output_batches")); + } + + let out = DisplayableExecutionPlan::with_metrics(plan.as_ref()) + .set_metric_names(metric_names) + .pgjson(false) + .to_string(); + let value: serde_json::Value = serde_json::from_str(&out).unwrap(); + let root = value[0].get("Plan").expect("plan"); + assert_eq!(root["Actual Rows"].as_u64(), Some(42)); + assert!(root.get("Actual Total Time").is_none()); + assert!(root.get("Extras").is_none()); } #[test] From 8fbbfdcaf5c201f32fbbbd841148d46ae99fa6af Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:47:07 +0300 Subject: [PATCH 049/109] perf: optimize `array_empty` udf (#23923) ## Which issue does this PR close? N/A ## Rationale for this change I saw that `empty` implementation is inefficient. I wrote review comments for more why inefficient. there are some optimizations that do not require benchmark since they are obvious once you understand, this is one of them ## What changes are included in this PR? rewrote the function to be fast ## Are these changes tested? existing tests ## Are there any user-facing changes? no --- datafusion/functions-nested/src/empty.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/datafusion/functions-nested/src/empty.rs b/datafusion/functions-nested/src/empty.rs index 262eb4935c968..6db412d29b0d8 100644 --- a/datafusion/functions-nested/src/empty.rs +++ b/datafusion/functions-nested/src/empty.rs @@ -122,9 +122,19 @@ fn array_empty_inner(args: &[ArrayRef]) -> Result { } fn general_array_empty(array: &ArrayRef) -> Result { - let result = as_generic_list_array::(array)? - .iter() - .map(|arr| arr.map(|arr| arr.is_empty())) - .collect::(); + let result = as_generic_list_array::(array)?; + let is_empty_iter = result.offsets().lengths().map(|n| n == 0); + // SAFETY: this is safe since the iterator lengths is exact size and + // trusted - it maps over fixed known number of elements + let output_buffer = unsafe { BooleanArray::from_trusted_len_iter(is_empty_iter) }; + + let (values, _) = output_buffer.into_parts(); + + // Add the nulls + let result = BooleanArray::new( + values, + result.nulls().filter(|n| n.null_count() > 0).cloned(), + ); + Ok(Arc::new(result)) } From 53f5bf5347485e9d77ad545c5836e4172ce75e47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:11:57 +1000 Subject: [PATCH 050/109] chore(deps): bump the codeql-actions group with 2 updates (#23938) Bumps the codeql-actions group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.37.1 to 4.37.3
Release notes

Sourced from github/codeql-action/init's releases.

v4.37.3

No user facing changes.

v4.37.2

  • The new address format for the config-file input that was introduced in CodeQL Action 4.37.0 is now enabled by default. In addition to the format described there, the remote= prefix can now be used to explicitly indicate that the input refers to a remote file. All previous input formats continue to be accepted as well. #4023
  • The CodeQL Action can now make use of configured private registries in Default Setup to retrieve CodeQL configuration files from remote repositories that require authentication. This will allow customers to store their CodeQL configuration in a single repository that can then be referenced by Default Setup workflows in other repositories. We expect to roll this and other, related changes out to everyone in July. #4007
Changelog

Sourced from github/codeql-action/init's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

  • This version of the CodeQL Action adds support for the tools input for the codeql-action/init step to be specified using a github-codeql-tools repository property. This feature will gradually be rolled out following the release of this version. Once rolled out, this allows for the CodeQL CLI version that is used in GitHub-managed workflows, such as Default Setup, to be set to a custom value. For example, customers who run into issues with rate limits when a new CodeQL CLI version is released can set the value to toolcache to always use the CodeQL CLI version that is available in the runner toolcache. For Advanced Setup workflows, the value provided for tools in the workflow definition always takes precedence unless the value of the repository property starts with !. #4037

4.37.3 - 22 Jul 2026

No user facing changes.

4.37.2 - 21 Jul 2026

  • The new address format for the config-file input that was introduced in CodeQL Action 4.37.0 is now enabled by default. In addition to the format described there, the remote= prefix can now be used to explicitly indicate that the input refers to a remote file. All previous input formats continue to be accepted as well. #4023
  • The CodeQL Action can now make use of configured private registries in Default Setup to retrieve CodeQL configuration files from remote repositories that require authentication. This will allow customers to store their CodeQL configuration in a single repository that can then be referenced by Default Setup workflows in other repositories. We expect to roll this and other, related changes out to everyone in July. #4007

4.37.1 - 16 Jul 2026

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.20.6 and earlier. These versions of CodeQL were discontinued on 1 July 2026 alongside GitHub Enterprise Server 3.16, and will be unsupported by the next minor release of the CodeQL Action. #3956
  • Update default CodeQL bundle version to 2.26.1. #4019

4.37.0 - 08 Jul 2026

  • Update default CodeQL bundle version to 2.26.0. #3995
  • In addition to the existing input format, the config-file input for the codeql-action/init step will soon support a new [owner/]repo[@ref][:path] format. All components except the repository name are optional. If omitted, owner defaults to the same owner as the repository the analysis is running for, ref to main, and path to .github/codeql-action.yaml. Support for this format ships in this version of the CodeQL Action, but will only be enabled over the coming weeks. #3973

4.36.3 - 01 Jul 2026

No user facing changes.

4.36.2 - 04 Jun 2026

  • Cache CodeQL CLI version information across Actions steps. #3943
  • Reduce requests while waiting for analysis processing by using exponential backoff when polling SARIF processing status. #3937
  • Update default CodeQL bundle version to 2.25.6. #3948

4.36.1 - 02 Jun 2026

No user facing changes.

4.36.0 - 22 May 2026

  • Breaking change: Bump the minimum required CodeQL bundle version to 2.19.4. #3894
  • Add support for SHA-256 Git object IDs. #3893
  • Update default CodeQL bundle version to 2.25.5. #3926

4.35.5 - 15 May 2026

  • We have improved how the JavaScript bundles for the CodeQL Action are generated to avoid duplication across bundles and reduce the size of the repository by around 70%. This should have no effect on the runtime behaviour of the CodeQL Action. #3899

... (truncated)

Commits
  • e4fba86 Merge pull request #4031 from github/update-v4.37.3-72f6a9da0
  • fb50ab5 Update changelog for v4.37.3
  • 72f6a9d Merge pull request #4030 from github/mbg/fix/no-proxy
  • 3b5ee58 Use default request options instead of undefined
  • bfb6be4 Merge pull request #4028 from github/mergeback/v4.37.2-to-main-e0647621
  • 526ab84 Rebuild
  • d6217b9 Update changelog and version after v4.37.2
  • e064762 Merge pull request #4027 from github/update-v4.37.2-385bcdc5a
  • e0faed8 Add a couple of change notes
  • 73aad0e Update changelog for v4.37.2
  • Additional commits viewable in compare view

Updates `github/codeql-action/analyze` from 4.37.1 to 4.37.3
Release notes

Sourced from github/codeql-action/analyze's releases.

v4.37.3

No user facing changes.

v4.37.2

  • The new address format for the config-file input that was introduced in CodeQL Action 4.37.0 is now enabled by default. In addition to the format described there, the remote= prefix can now be used to explicitly indicate that the input refers to a remote file. All previous input formats continue to be accepted as well. #4023
  • The CodeQL Action can now make use of configured private registries in Default Setup to retrieve CodeQL configuration files from remote repositories that require authentication. This will allow customers to store their CodeQL configuration in a single repository that can then be referenced by Default Setup workflows in other repositories. We expect to roll this and other, related changes out to everyone in July. #4007
Changelog

Sourced from github/codeql-action/analyze's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

  • This version of the CodeQL Action adds support for the tools input for the codeql-action/init step to be specified using a github-codeql-tools repository property. This feature will gradually be rolled out following the release of this version. Once rolled out, this allows for the CodeQL CLI version that is used in GitHub-managed workflows, such as Default Setup, to be set to a custom value. For example, customers who run into issues with rate limits when a new CodeQL CLI version is released can set the value to toolcache to always use the CodeQL CLI version that is available in the runner toolcache. For Advanced Setup workflows, the value provided for tools in the workflow definition always takes precedence unless the value of the repository property starts with !. #4037

4.37.3 - 22 Jul 2026

No user facing changes.

4.37.2 - 21 Jul 2026

  • The new address format for the config-file input that was introduced in CodeQL Action 4.37.0 is now enabled by default. In addition to the format described there, the remote= prefix can now be used to explicitly indicate that the input refers to a remote file. All previous input formats continue to be accepted as well. #4023
  • The CodeQL Action can now make use of configured private registries in Default Setup to retrieve CodeQL configuration files from remote repositories that require authentication. This will allow customers to store their CodeQL configuration in a single repository that can then be referenced by Default Setup workflows in other repositories. We expect to roll this and other, related changes out to everyone in July. #4007

4.37.1 - 16 Jul 2026

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.20.6 and earlier. These versions of CodeQL were discontinued on 1 July 2026 alongside GitHub Enterprise Server 3.16, and will be unsupported by the next minor release of the CodeQL Action. #3956
  • Update default CodeQL bundle version to 2.26.1. #4019

4.37.0 - 08 Jul 2026

  • Update default CodeQL bundle version to 2.26.0. #3995
  • In addition to the existing input format, the config-file input for the codeql-action/init step will soon support a new [owner/]repo[@ref][:path] format. All components except the repository name are optional. If omitted, owner defaults to the same owner as the repository the analysis is running for, ref to main, and path to .github/codeql-action.yaml. Support for this format ships in this version of the CodeQL Action, but will only be enabled over the coming weeks. #3973

4.36.3 - 01 Jul 2026

No user facing changes.

4.36.2 - 04 Jun 2026

  • Cache CodeQL CLI version information across Actions steps. #3943
  • Reduce requests while waiting for analysis processing by using exponential backoff when polling SARIF processing status. #3937
  • Update default CodeQL bundle version to 2.25.6. #3948

4.36.1 - 02 Jun 2026

No user facing changes.

4.36.0 - 22 May 2026

  • Breaking change: Bump the minimum required CodeQL bundle version to 2.19.4. #3894
  • Add support for SHA-256 Git object IDs. #3893
  • Update default CodeQL bundle version to 2.25.5. #3926

4.35.5 - 15 May 2026

  • We have improved how the JavaScript bundles for the CodeQL Action are generated to avoid duplication across bundles and reduce the size of the repository by around 70%. This should have no effect on the runtime behaviour of the CodeQL Action. #3899

... (truncated)

Commits
  • e4fba86 Merge pull request #4031 from github/update-v4.37.3-72f6a9da0
  • fb50ab5 Update changelog for v4.37.3
  • 72f6a9d Merge pull request #4030 from github/mbg/fix/no-proxy
  • 3b5ee58 Use default request options instead of undefined
  • bfb6be4 Merge pull request #4028 from github/mergeback/v4.37.2-to-main-e0647621
  • 526ab84 Rebuild
  • d6217b9 Update changelog and version after v4.37.2
  • e064762 Merge pull request #4027 from github/update-v4.37.2-385bcdc5a
  • e0faed8 Add a couple of change notes
  • 73aad0e Update changelog for v4.37.2
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3fb5433880e57..7d10034f6987d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,11 +45,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4 + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4 + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 with: category: "/language:actions" From 0d7bfd54d547e2967185089bc4d450e658c1ac47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:12:44 +0000 Subject: [PATCH 051/109] chore(deps): bump taiki-e/install-action from 2.84.0 to 2.85.2 (#23941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.84.0 to 2.85.2.
Release notes

Sourced from taiki-e/install-action's releases.

2.85.2

  • Update prek@latest to 0.4.11.

  • Update mise@latest to 2026.7.13.

  • Update kingfisher@latest to 1.109.0.

2.85.1

  • Update vacuum@latest to 0.30.0.

  • Update uv@latest to 0.11.32.

  • Update mise@latest to 2026.7.12.

  • Update cyclonedx@latest to 0.33.1.

  • Update cargo-neat@latest to 0.5.2.

2.85.0

  • Support wild (alias: wild-linker). (#1949)

  • Support bpf-linker. (#1950)

  • Support rafn. (#1935, thanks @​DarkWanderer)

  • Update cargo-neat@latest to 0.5.1.

  • Update zizmor@latest to 1.28.0.

  • Update wasmtime@latest to 47.0.2.

  • Update uv@latest to 0.11.31.

  • Update syft@latest to 1.49.0.

2.84.1

  • Update wasmtime@latest to 47.0.1.

  • Update wasm-tools@latest to 1.254.0.

  • Update uv@latest to 0.11.30.

  • Update mise@latest to 2026.7.11.

  • Update cargo-neat@latest to 0.5.0.

  • Update cargo-crap@latest to 0.3.1.

  • Update biome@latest to 2.5.5.

Changelog

Sourced from taiki-e/install-action's changelog.

Changelog

All notable changes to this project will be documented in this file.

This project adheres to Semantic Versioning.

[Unreleased]

[2.85.2] - 2026-07-26

  • Update prek@latest to 0.4.11.

  • Update mise@latest to 2026.7.13.

  • Update kingfisher@latest to 1.109.0.

[2.85.1] - 2026-07-25

  • Update vacuum@latest to 0.30.0.

  • Update uv@latest to 0.11.32.

  • Update mise@latest to 2026.7.12.

  • Update cyclonedx@latest to 0.33.1.

  • Update cargo-neat@latest to 0.5.2.

[2.85.0] - 2026-07-23

  • Support wild (alias: wild-linker). (#1949)

  • Support bpf-linker. (#1950)

  • Support rafn. (#1935, thanks @​DarkWanderer)

  • Update cargo-neat@latest to 0.5.1.

  • Update zizmor@latest to 1.28.0.

  • Update wasmtime@latest to 47.0.2.

  • Update uv@latest to 0.11.31.

  • Update syft@latest to 1.49.0.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=taiki-e/install-action&package-manager=github_actions&previous-version=2.84.0&new-version=2.85.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/audit.yml | 2 +- .github/workflows/breaking_changes_detector.yml | 2 +- .github/workflows/dependencies.yml | 2 +- .github/workflows/dev.yml | 6 +++--- .github/workflows/docs.yaml | 2 +- .github/workflows/docs_pr.yaml | 2 +- .github/workflows/rust.yml | 8 ++++---- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e7d8942fc1198..ba77320d47760 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install cargo-audit - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-audit - name: Run audit check diff --git a/.github/workflows/breaking_changes_detector.yml b/.github/workflows/breaking_changes_detector.yml index 609981a7778b7..8972eb4404b0e 100644 --- a/.github/workflows/breaking_changes_detector.yml +++ b/.github/workflows/breaking_changes_detector.yml @@ -89,7 +89,7 @@ jobs: - name: Install cargo-semver-checks if: steps.changed_crates.outputs.packages != '' - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-semver-checks diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 599f517b66274..380909fc2cbee 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -63,7 +63,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install cargo-machete - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-machete@0.9 - name: Detect unused dependencies diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 92c6264d5c5b5..c5c2ed9582271 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install HawkEye - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: hawkeye@6.2.0 - name: Run license header check @@ -66,7 +66,7 @@ jobs: source ci/scripts/utils/tool_versions.sh echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV" - name: Install lychee - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: lychee@${{ env.LYCHEE_VERSION }} - name: Run markdown link check @@ -91,7 +91,7 @@ jobs: # it may cause checks to fail more often. # We can upgrade it manually once a while. - name: Install typos - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: typos@1.37.0 - name: Run typos check diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 71f04a2cdbaca..b0a1ab295ddc5 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -53,7 +53,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-depgraph@1.6 diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index b9b6e5d82b960..2b6c1c7d1e044 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -59,7 +59,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-depgraph@1.6 - name: Build docs html and check for warnings diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1d2c0362c888d..1f3f5269de04c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -309,7 +309,7 @@ jobs: - name: Install llvm-tools-preview run: rustup component add llvm-tools-preview - name: Install cargo-llvm-cov - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-llvm-cov - name: Rust Dependency Cache @@ -466,7 +466,7 @@ jobs: sudo apt-get update -qq sudo apt-get install -y -qq clang - name: Setup wasm-pack - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: wasm-pack - name: Run tests with headless mode @@ -697,7 +697,7 @@ jobs: with: rust-version: stable - name: Install taplo - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: taplo-cli@0.9 # if you encounter an error, try running 'taplo format' to fix the formatting automatically. @@ -782,7 +782,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Install cargo-msrv - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-msrv From 6cc7fa4c4537259738e88da25561848aa9555088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:12:58 +1000 Subject: [PATCH 052/109] chore(deps): bump actions/stale from 10.4.0 to 11.0.0 (#23942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/stale](https://github.com/actions/stale) from 10.4.0 to 11.0.0.
Release notes

Sourced from actions/stale's releases.

v11.0.0

What's Changed

Enhancement

Dependency Update

Full Changelog: https://github.com/actions/stale/compare/v10...v11.0.0

Commits
  • 4391f3d Fix 24 high severity vulnerabilities by overriding brace-expansion to 5.0.8 (...
  • eaf9131 refactor: update imports to use ES module syntax and improve test structure (...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/stale&package-manager=github_actions&previous-version=10.4.0&new-version=11.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7b0d1b9e90187..81188559d89f0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: stale-pr-message: "Thank you for your contribution. Unfortunately, this pull request is stale because it has been open 60 days with no activity. Please remove the stale label or comment or this will be closed in 7 days." days-before-pr-stale: 60 From 994fc81ac89d817ac7311fb8d507f564cdfec2d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:13:42 +1000 Subject: [PATCH 053/109] chore(deps): bump base64 from 0.22.1 to 0.23.0 (#23944) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.22.1 to 0.23.0.
Changelog

Sourced from base64's changelog.

0.23.0

  • Added more consts for preconfigured configs and engines
  • Make DecodeError::InvalidLastSymbol more clear by including the decoded value
  • Added SIMD-accelerated engines behind the default-on simd-unsafe feature: Simd picks the best instruction set at runtime (AVX2 on x86_64, NEON on aarch64) and falls back to the scalar GeneralPurpose engine, while Avx2 and Neon target one instruction set with no runtime detection and work in no_std. The engines support the standard and URL-safe alphabets.
  • Update MSRV to 1.71.0
  • Add support for custom padding symbols
Commits
  • 9e9220a v0.23.0
  • 870326e Merge pull request #306 from marshallpierce/mp/trailing-bits-docs
  • fbec5f1 Document no trailing trailing bits
  • 0a23549 Merge pull request #305 from marshallpierce/mp/edition-2021
  • f10b7e2 Update deps & edition
  • 9d21a59 Merge pull request #304 from marshallpierce/mp/custom-padding-rebase
  • f70bad2 Support custom padding symbols
  • 684d79c Merge pull request #301 from marshallpierce/mp/simd-gardening
  • 5bf66f2 Merge pull request #284 from AbeZbm/add-tests
  • d3831cf Followups to SIMD work
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=base64&package-manager=cargo&previous-version=0.22.1&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 10 ++++++++-- datafusion-examples/Cargo.toml | 2 +- datafusion/functions/Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed63f60b41519..245f02f8d3a7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -989,6 +989,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -2083,7 +2089,7 @@ dependencies = [ "arrow-flight", "arrow-schema", "async-trait", - "base64 0.22.1", + "base64 0.23.0", "bytes", "dashmap", "datafusion", @@ -2221,7 +2227,7 @@ version = "54.1.0" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.0", "blake2", "blake3", "chrono", diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index e59f7eb9483c3..6d6d917ac46ec 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -50,7 +50,7 @@ async-trait = { workspace = true } bytes = { workspace = true } dashmap = { workspace = true } # note only use main datafusion crate for examples -base64 = "0.22.1" +base64 = "0.23.0" datafusion-expr = { workspace = true } datafusion-physical-expr-adapter = { workspace = true } datafusion-proto = { workspace = true, features = ["parquet"] } diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 83f8c0f2a3299..3c88c290561bb 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -67,7 +67,7 @@ name = "datafusion_functions" [dependencies] arrow = { workspace = true } arrow-buffer = { workspace = true } -base64 = { version = "0.22", optional = true } +base64 = { version = "0.23", optional = true } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.8", optional = true } chrono = { workspace = true } From 5d9638c1de6d6aaf6fa14e96f01711ed67ecd6f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:18:36 -0400 Subject: [PATCH 054/109] chore(deps): bump astral-sh/setup-uv from 8.3.2 to 9.0.0 (#23939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 8.3.2 to 9.0.0.
Release notes

Sourced from astral-sh/setup-uv's releases.

v9.0.0 🌈 Change prune-cache default to false

Changes

This release disables the default cache cache pruning to ease the load on the PyPi infrastructure. Since users might experience more GitHub Actions cache usage which might result in higher costs this is marked as a breaking change. To read more on why we did this (now) you can read the detailed analysis and reasoning in #967

Besides this big breaking change we also have a small bugfix while building caches for linux distributions that behave a big different than the "big ones" and a speed up in version resolution by only reading the version manifest until a matching version is found saving runtime and network bandwith.

🚨 Breaking changes

🐛 Bug fixes

  • fix: fall back to distribution ID when os-release has no version field @​cxzhong (#961)

🚀 Enhancements

🧰 Maintenance

📚 Documentation

⬆️ Dependency updates

Commits
  • c771a70 chore(deps): roll up Dependabot updates (#970)
  • 2f537ca chore: update known checksums for 0.11.30 (#968)
  • 2269552 Speed up version client by partial response reads (#807)
  • 47a7f4f Change prune-cache default to false (#967)
  • 71966ef chore(deps): roll up Dependabot updates (#962)
  • f12b1f0 fix: fall back to distribution ID when os-release has no version field (#961)
  • ecd24dd chore: update known checksums for 0.11.29 (#960)
  • 6a19136 docs: update version references to v8.3.2 (#949)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=astral-sh/setup-uv&package-manager=github_actions&previous-version=8.3.2&new-version=9.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yaml | 2 +- .github/workflows/docs_pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b0a1ab295ddc5..a90b96f90bb7d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -43,7 +43,7 @@ jobs: path: asf-site - name: Setup uv - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Install dependencies run: uv sync --package datafusion-docs diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 2b6c1c7d1e044..bffb02e81f8da 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -50,7 +50,7 @@ jobs: submodules: true fetch-depth: 1 - name: Setup uv - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Install doc dependencies run: uv sync --package datafusion-docs - name: Install Graphviz From b0b9dae41095780fec5f8c0033ba48c0d81077be Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:15:41 +0300 Subject: [PATCH 055/109] chore: refactor SortMergeJoin bitwise stream to generators and simplify to be textbook like as possible (#23761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? N/A ## Rationale for this change SortMergeJoin bitwise stream implementation is very complex and hard to understand while on paper it should be pretty simple. the reason for that is we have to store state between polls (we had `boundary`) and handle the case where both can get `Poll::Pending` from child and `Poll::Ready` from child which further complicate the code ## What changes are included in this PR? 1. Move to async generators 2. Rewrote main loop to be textbook like as possible (this was entirely written by Claude Fable, sorry, I tried manually but the code was too complex to hold in my head 😅 ) ## Are these changes tested? existing tests ## Are there any user-facing changes? The join_time now includes the time to read from the async spill stream between pending which is arguable more correct since this time is part of the operator, although long waits between pending calls will be counted in the op `join_time` while the alternative is not counting the read from file and decoding... --- .../joins/sort_merge_join/bitwise_stream.rs | 1023 +++++++---------- .../src/joins/sort_merge_join/exec.rs | 4 +- .../src/joins/sort_merge_join/tests.rs | 289 ++++- 3 files changed, 679 insertions(+), 637 deletions(-) diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs index d1ca9707febf2..3716ecde284c5 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs @@ -84,8 +84,9 @@ //! //! Key groups can span batch boundaries on either side. The stream handles //! this by detecting when a group extends to the end of a batch, loading the -//! next batch, and continuing if the key matches. The [`PendingBoundary`] enum -//! preserves loop context across async `Poll::Pending` re-entries. +//! next batch, and continuing if the key matches. The generator-based stream +//! suspends in place at `await` points, so no explicit re-entry state is +//! needed. //! //! # Memory //! @@ -119,29 +120,31 @@ //! factor than the pair-materialization approach. use std::cmp::Ordering; -use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; +use crate::EmptyRecordBatchStream; use crate::joins::utils::{JoinFilter, JoinKeyComparator, compare_join_arrays}; use crate::metrics::{ - BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, + BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, Time, }; use crate::spill::spill_manager::SpillManager; -use crate::{EmptyRecordBatchStream, RecordBatchStream}; +use crate::stream::{ObservedStream, RecordBatchStreamAdapter}; use arrow::array::{Array, ArrayRef, BooleanArray, BooleanBufferBuilder, RecordBatch}; use arrow::compute::{BatchCoalescer, SortOptions, filter_record_batch, not}; use arrow::datatypes::SchemaRef; use arrow::util::bit_chunk_iterator::UnalignedBitChunk; use arrow::util::bit_util::apply_bitwise_binary_op; +use datafusion_common::instant::Instant; use datafusion_common::{ - JoinSide, JoinType, NullEquality, Result, ScalarValue, internal_err, + DataFusionError, JoinSide, JoinType, NullEquality, Result, ScalarValue, internal_err, }; use datafusion_execution::memory_pool::MemoryReservation; -use datafusion_execution::{SendableRecordBatchStream, SpillFile}; +use datafusion_execution::{ + SendableRecordBatchStream, SpillFile, TryEmitter, async_try_stream, +}; use datafusion_physical_expr_common::physical_expr::PhysicalExprRef; -use futures::{Stream, StreamExt, ready}; +use futures::StreamExt; /// Evaluates join key expressions against a batch, returning one array per key. fn evaluate_join_keys( @@ -194,26 +197,6 @@ fn find_key_group_end(cmp: &JoinKeyComparator, from: usize, len: usize) -> usize lo } -/// When an outer key group spans a batch boundary, the boundary loop emits -/// the current batch, then polls for the next. If that poll returns Pending, -/// `ready!` exits `poll_join` and we re-enter from the top on the next call. -/// Without this state, the new batch would be processed fresh by the -/// merge-scan — but inner already advanced past this key, so the matching -/// outer rows would be skipped via `Ordering::Less` and never marked. -/// -/// This enum carries the last key (as single-row sliced arrays) from the -/// previous batch so we can check whether the next batch continues the same -/// key group. Stored as `Option`: `None` means normal -/// processing. -#[derive(Debug)] -enum PendingBoundary { - /// Resuming a no-filter boundary loop. - NoFilter { saved_keys: Vec }, - /// Resuming a filtered boundary loop. Inner key data remains in the - /// buffer (or spill file) for the resumed loop. - Filtered { saved_keys: Vec }, -} - /// Sort-Merge join stream for Semi/Anti/Mark joins. /// /// Named "bitwise" because it tracks outer-row matches via a per-batch @@ -255,22 +238,6 @@ pub(crate) struct BitwiseSortMergeJoinStream { inner_key_buffer: Vec, inner_key_spill: Option>, - // Track the active spill_stream - spill_stream: Option, - // Whether the active spill stream has produced any batches yet. - spill_stream_has_data: bool, - // Prevents wiping out the buffer if we yield while evaluating the filter - inner_group_buffered: bool, - - // True when buffer_inner_key_group returned Pending after partially - // filling inner_key_buffer. On re-entry, buffer_inner_key_group - // must skip clear() and resume from poll_next_inner_batch (the - // current inner_batch was already sliced and pushed before Pending). - buffering_inner_pending: bool, - - // Boundary re-entry state — see PendingBoundary doc comment. - pending_boundary: Option, - // Join ON expressions, evaluated against each new batch to produce // the key arrays used for sorted key comparisons. on_outer: Vec, @@ -286,12 +253,18 @@ pub(crate) struct BitwiseSortMergeJoinStream { coalescer: BatchCoalescer, schema: SchemaRef, - // Metrics - join_time: crate::metrics::Time, + // Metrics — output rows/batches and end time are recorded by the + // ObservedStream wrapper in try_new, not here. input_batches: Count, input_rows: Count, - baseline_metrics: BaselineMetrics, peak_mem_used: Gauge, + /// Time spent doing the join's own work (including spill write and + /// read-back). The clock is stopped while awaiting the child inputs or + /// the consumer taking an emitted batch — see [`Self::stop_join_time`]. + join_time: Time, + /// Start of the currently running `join_time` span; `None` while the + /// clock is stopped. + join_time_start: Option, // Memory / spill — only the inner key buffer is tracked via reservation, // matching existing SMJ (which tracks only the buffered side). The outer @@ -308,14 +281,6 @@ pub(crate) struct BitwiseSortMergeJoinStream { outer_self_cmp: Option, /// Comparator for inner self-comparison (find_key_group_end on inner) inner_self_cmp: Option, - - // True once the current outer batch has been emitted. The Equal - // branch's inner loops call emit then `ready!(poll_next_outer_batch)`. - // If that poll returns Pending, poll_join re-enters from the top - // on the next poll — with outer_batch still Some and outer_offset - // past the end. The main loop's step 3 would re-emit without this - // guard. Cleared when poll_next_outer_batch loads a new batch. - batch_emitted: bool, } impl BitwiseSortMergeJoinStream { @@ -336,7 +301,7 @@ impl BitwiseSortMergeJoinStream { reservation: MemoryReservation, spill_manager: SpillManager, runtime_env: Arc, - ) -> Result { + ) -> Result { debug_assert!( matches!( join_type, @@ -362,7 +327,7 @@ impl BitwiseSortMergeJoinStream { let peak_mem_used = MetricBuilder::new(metrics).peak_memory_usage("peak_mem_used", partition); - Ok(Self { + let mut state = Self { join_type, outer, inner, @@ -375,11 +340,6 @@ impl BitwiseSortMergeJoinStream { matched: BooleanBufferBuilder::new(0), inner_key_buffer: vec![], inner_key_spill: None, - spill_stream: None, - spill_stream_has_data: false, - inner_group_buffered: false, - buffering_inner_pending: false, - pending_boundary: None, on_outer, on_inner, filter, @@ -388,12 +348,12 @@ impl BitwiseSortMergeJoinStream { outer_is_left, coalescer: BatchCoalescer::new(Arc::clone(&schema), batch_size) .with_biggest_coalesce_batch_size(Some(batch_size / 2)), - schema, - join_time, + schema: Arc::clone(&schema), input_batches, input_rows, - baseline_metrics, peak_mem_used, + join_time, + join_time_start: None, reservation, spill_manager, runtime_env, @@ -401,8 +361,39 @@ impl BitwiseSortMergeJoinStream { outer_inner_cmp: None, outer_self_cmp: None, inner_self_cmp: None, - batch_emitted: false, - }) + }; + + let stream = async_try_stream(|mut emitter| async move { + state.start_join_time(); + let result = state.join(&mut emitter).await; + state.stop_join_time(); + result + }); + // ObservedStream records the baseline metrics (output rows/batches, + // end time) exactly as the former hand-written poll_next did. + Ok(Box::pin(ObservedStream::new( + Box::pin(RecordBatchStreamAdapter::new(schema, stream)), + baseline_metrics, + None, + ))) + } + + /// Start (resume) the `join_time` clock. + fn start_join_time(&mut self) { + debug_assert!(self.join_time_start.is_none(), "join_time already running"); + self.join_time_start = Some(Instant::now()); + } + + /// Stop (pause) the `join_time` clock, accumulating the elapsed span. + /// + /// Called around awaits whose duration is not the join's own work: the + /// child input streams' `next()` and `emitter.emit()` (where the + /// consumer processes the batch). The join's own spill read-back is NOT + /// excluded — that time is join work. + fn stop_join_time(&mut self) { + if let Some(start) = self.join_time_start.take() { + self.join_time.add_elapsed(start); + } } /// Resize the memory reservation to match current tracked usage. @@ -475,23 +466,24 @@ impl BitwiseSortMergeJoinStream { fn clear_inner_key_group(&mut self) { self.inner_key_buffer.clear(); self.inner_key_spill = None; - self.spill_stream = None; - self.spill_stream_has_data = false; - self.inner_group_buffered = false; self.inner_buffer_size = 0; } - /// Poll for the next outer batch. Returns true if a batch was loaded. - fn poll_next_outer_batch(&mut self, cx: &mut Context<'_>) -> Poll> { + /// Fetch the next outer batch. Returns true if a batch was loaded. + async fn next_outer_batch(&mut self) -> Result { loop { - match ready!(self.outer.poll_next_unpin(cx)) { + // The child's execution time is its own, not join_time. + self.stop_join_time(); + let item = self.outer.next().await; + self.start_join_time(); + match item { None => { // Release the outer input pipeline's resources. let outer_schema = self.outer.schema(); self.outer = Box::pin(EmptyRecordBatchStream::new(outer_schema)); - return Poll::Ready(Ok(false)); + return Ok(false); } - Some(Err(e)) => return Poll::Ready(Err(e)), + Some(Err(e)) => return Err(e), Some(Ok(batch)) => { let batch_num_rows = batch.num_rows(); self.input_batches.add(1); @@ -505,26 +497,29 @@ impl BitwiseSortMergeJoinStream { self.outer_key_arrays = keys; self.outer_inner_cmp = None; self.outer_self_cmp = None; - self.batch_emitted = false; self.matched = BooleanBufferBuilder::new(batch_num_rows); self.matched.append_n(batch_num_rows, false); - return Poll::Ready(Ok(true)); + return Ok(true); } } } } - /// Poll for the next inner batch. Returns true if a batch was loaded. - fn poll_next_inner_batch(&mut self, cx: &mut Context<'_>) -> Poll> { + /// Fetch the next inner batch. Returns true if a batch was loaded. + async fn next_inner_batch(&mut self) -> Result { loop { - match ready!(self.inner.poll_next_unpin(cx)) { + // The child's execution time is its own, not join_time. + self.stop_join_time(); + let item = self.inner.next().await; + self.start_join_time(); + match item { None => { // Release the inner input pipeline's resources. let inner_schema = self.inner.schema(); self.inner = Box::pin(EmptyRecordBatchStream::new(inner_schema)); - return Poll::Ready(Ok(false)); + return Ok(false); } - Some(Err(e)) => return Poll::Ready(Err(e)), + Some(Err(e)) => return Err(e), Some(Ok(batch)) => { let batch_num_rows = batch.num_rows(); self.input_batches.add(1); @@ -538,22 +533,17 @@ impl BitwiseSortMergeJoinStream { self.inner_key_arrays = keys; self.outer_inner_cmp = None; self.inner_self_cmp = None; - return Poll::Ready(Ok(true)); + return Ok(true); } } } } - /// Emit the current outer batch through the coalescer, applying the - /// matched bitset as a selection mask. No-op if already emitted - /// (see `batch_emitted` field). + /// Push the current outer batch into the coalescer, applying the matched + /// bitset as a selection mask. Consumes the batch (`outer_batch` becomes + /// `None`). fn emit_outer_batch(&mut self) -> Result<()> { - if self.batch_emitted { - return Ok(()); - } - self.batch_emitted = true; - - let batch = self.outer_batch.as_ref().unwrap(); + let batch = self.outer_batch.take().unwrap(); // finish() converts the bit-packed builder directly to a // BooleanBuffer — no iteration or repacking needed. @@ -576,14 +566,14 @@ impl BitwiseSortMergeJoinStream { } JoinType::LeftSemi | JoinType::RightSemi => { let selection = BooleanArray::new(matched_buf, None); - let filtered = filter_record_batch(batch, &selection)?; + let filtered = filter_record_batch(&batch, &selection)?; if filtered.num_rows() > 0 { self.coalescer.push_batch(filtered)?; } } JoinType::LeftAnti | JoinType::RightAnti => { let selection = not(&BooleanArray::new(matched_buf, None))?; - let filtered = filter_record_batch(batch, &selection)?; + let filtered = filter_record_batch(&batch, &selection)?; if filtered.num_rows() > 0 { self.coalescer.push_batch(filtered)?; } @@ -593,165 +583,114 @@ impl BitwiseSortMergeJoinStream { Ok(()) } - /// Process a key match between outer and inner sides (no filter). - /// Sets matched bits for all outer rows sharing the current key. - fn process_key_match_no_filter(&mut self) -> Result<()> { - let outer_batch = self.outer_batch.as_ref().unwrap(); - let num_outer = outer_batch.num_rows(); + /// Mark all outer rows in the current key group as matched and advance + /// the outer cursor past the group (within the current batch). + fn mark_outer_key_group_matched(&mut self) -> Result<()> { + let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); + let from = self.outer_offset; + let group_end = find_key_group_end(self.get_outer_self_cmp()?, from, num_outer); - self.get_outer_self_cmp()?; - let outer_group_end = find_key_group_end( - self.outer_self_cmp.as_ref().unwrap(), - self.outer_offset, - num_outer, - ); - - for i in self.outer_offset..outer_group_end { + for i in from..group_end { self.matched.set_bit(i, true); } - self.outer_offset = outer_group_end; + self.outer_offset = group_end; Ok(()) } - /// Advance inner past the current key group. Returns Ok(true) if inner + /// Advance the inner cursor past the current key group. The group may + /// span multiple inner batches. Sets `inner_batch` to `None` if inner /// is exhausted. - fn advance_inner_past_key_group( - &mut self, - cx: &mut Context<'_>, - ) -> Poll> { + async fn advance_inner_past_key_group(&mut self) -> Result<()> { loop { - let inner_batch = match &self.inner_batch { - Some(b) => b, - None => return Poll::Ready(Ok(true)), + let Some(inner_batch) = &self.inner_batch else { + return Ok(()); }; let num_inner = inner_batch.num_rows(); - - self.get_inner_self_cmp()?; - let group_end = find_key_group_end( - self.inner_self_cmp.as_ref().unwrap(), - self.inner_offset, - num_inner, - ); + let from = self.inner_offset; + let group_end = + find_key_group_end(self.get_inner_self_cmp()?, from, num_inner); if group_end < num_inner { self.inner_offset = group_end; - return Poll::Ready(Ok(false)); + return Ok(()); } - // Key group extends to end of batch — need to check next batch + // Key group extends to the end of the batch — it may continue + // into the next one; save the last key so we can check. let saved_inner_keys = slice_keys(&self.inner_key_arrays, num_inner - 1); - match ready!(self.poll_next_inner_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - return Poll::Ready(Ok(true)); - } - Ok(true) => { - if keys_match( - &saved_inner_keys, - &self.inner_key_arrays, - &self.sort_options, - self.null_equality, - )? { - continue; - } else { - return Poll::Ready(Ok(false)); - } - } + if !self.next_inner_batch().await? { + self.inner_batch = None; + return Ok(()); + } + if !keys_match( + &saved_inner_keys, + &self.inner_key_arrays, + &self.sort_options, + self.null_equality, + )? { + return Ok(()); } } } - /// Buffer inner key group for filter evaluation. Collects all inner rows - /// with the current key across batch boundaries. - /// - /// If poll_next_inner_batch returns Pending, we save progress via - /// buffering_inner_pending. On re-entry (from the Equal branch in - /// poll_join), we skip clear() and the slice+push for the current - /// batch (which was already buffered before Pending), and go directly - /// to polling for the next inner batch. - fn buffer_inner_key_group(&mut self, cx: &mut Context<'_>) -> Poll> { - // On re-entry after Pending: don't clear the partially-filled - // buffer. The current inner_batch was already sliced and pushed - // before Pending, so jump to polling for the next batch. - let mut resume_from_poll = false; - if self.buffering_inner_pending { - self.buffering_inner_pending = false; - resume_from_poll = true; - } else { - self.clear_inner_key_group(); - } + /// Buffer the inner key group for filter evaluation, advancing the inner + /// cursor past the group. Collects all inner rows with the current key + /// across batch boundaries. Sets `inner_batch` to `None` if inner is + /// exhausted. + async fn buffer_inner_key_group(&mut self) -> Result<()> { + self.clear_inner_key_group(); loop { - if self.inner_batch.is_none() { - return Poll::Ready(Ok(true)); - } - let num_inner = self.inner_batch.as_ref().unwrap().num_rows(); - self.get_inner_self_cmp()?; - let group_end = find_key_group_end( - self.inner_self_cmp.as_ref().unwrap(), - self.inner_offset, - num_inner, - ); - - if !resume_from_poll { - let inner_batch = self.inner_batch.as_ref().unwrap(); - let slice = - inner_batch.slice(self.inner_offset, group_end - self.inner_offset); - self.inner_buffer_size += slice.get_array_memory_size(); - self.inner_key_buffer.push(slice); - - // Reserve memory for the newly buffered slice. If the pool - // is exhausted, spill the entire buffer to disk. - if self.try_resize_reservation().is_err() { - if self.runtime_env.disk_manager.tmp_files_enabled() { - self.spill_inner_key_buffer()?; - } else { - // Re-attempt to get the error message - self.try_resize_reservation().map_err(|e| { - datafusion_common::DataFusionError::Execution(format!( - "{e}. Disk spilling disabled." - )) - })?; - } + let Some(inner_batch) = &self.inner_batch else { + return Ok(()); + }; + let num_inner = inner_batch.num_rows(); + let from = self.inner_offset; + let group_end = + find_key_group_end(self.get_inner_self_cmp()?, from, num_inner); + + let inner_batch = self.inner_batch.as_ref().unwrap(); + let slice = inner_batch.slice(from, group_end - from); + self.inner_buffer_size += slice.get_array_memory_size(); + self.inner_key_buffer.push(slice); + + // Reserve memory for the newly buffered slice. If the pool + // is exhausted, spill the entire buffer to disk. + if self.try_resize_reservation().is_err() { + if self.runtime_env.disk_manager.tmp_files_enabled() { + self.spill_inner_key_buffer()?; + } else { + // Re-attempt to get the error message + self.try_resize_reservation().map_err(|e| { + DataFusionError::Execution(format!( + "{e}. Disk spilling disabled." + )) + })?; } + } - if group_end < num_inner { - self.inner_offset = group_end; - return Poll::Ready(Ok(false)); - } + if group_end < num_inner { + self.inner_offset = group_end; + return Ok(()); } - resume_from_poll = false; - // Key group extends to end of batch — check next + // Key group extends to the end of the batch — it may continue + // into the next one; save the last key so we can check. let saved_inner_keys = slice_keys(&self.inner_key_arrays, num_inner - 1); - // If poll returns Pending, the current batch is already - // in inner_key_buffer. - self.buffering_inner_pending = true; - match ready!(self.poll_next_inner_batch(cx)) { - Err(e) => { - self.buffering_inner_pending = false; - return Poll::Ready(Err(e)); - } - Ok(false) => { - self.buffering_inner_pending = false; - return Poll::Ready(Ok(true)); - } - Ok(true) => { - self.buffering_inner_pending = false; - if keys_match( - &saved_inner_keys, - &self.inner_key_arrays, - &self.sort_options, - self.null_equality, - )? { - continue; - } else { - return Poll::Ready(Ok(false)); - } - } + if !self.next_inner_batch().await? { + self.inner_batch = None; + return Ok(()); + } + if !keys_match( + &saved_inner_keys, + &self.inner_key_arrays, + &self.sort_options, + self.null_equality, + )? { + return Ok(()); } } } @@ -759,14 +698,8 @@ impl BitwiseSortMergeJoinStream { /// Process a key match with a filter. For each inner row in the buffered /// key group, evaluates the filter against the outer key group and ORs /// the results into the matched bitset using u64-chunked bitwise ops. - fn process_key_match_with_filter( - &mut self, - cx: &mut Context<'_>, - ) -> Poll> { - self.get_outer_self_cmp()?; - let filter = self.filter.as_ref().unwrap(); - let outer_batch = self.outer_batch.as_ref().unwrap(); - let num_outer = outer_batch.num_rows(); + async fn process_key_match_with_filter(&mut self) -> Result<()> { + let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); // buffer_inner_key_group must be called before this function debug_assert!( @@ -782,60 +715,54 @@ impl BitwiseSortMergeJoinStream { "matched vector must be sized for the current outer batch" ); - let outer_group_end = find_key_group_end( - self.outer_self_cmp.as_ref().unwrap(), - self.outer_offset, - num_outer, - ); - let outer_group_len = outer_group_end - self.outer_offset; - let outer_slice = outer_batch.slice(self.outer_offset, outer_group_len); + let outer_group_start = self.outer_offset; + let outer_group_end = + find_key_group_end(self.get_outer_self_cmp()?, outer_group_start, num_outer); + let outer_group_len = outer_group_end - outer_group_start; + + let filter = self.filter.as_ref().unwrap(); + let outer_batch = self.outer_batch.as_ref().unwrap(); + let outer_slice = outer_batch.slice(outer_group_start, outer_group_len); // Count already-matched bits using popcnt on u64 chunks (zero-copy). let mut matched_count = UnalignedBitChunk::new( self.matched.as_slice(), - self.outer_offset, + outer_group_start, outer_group_len, ) .count_ones(); // Process spilled inner batches first asynchronously. if matched_count < outer_group_len - && (self.inner_key_spill.is_some() || self.spill_stream.is_some()) + && let Some(spill_file) = &self.inner_key_spill { - if self.spill_stream.is_none() - && let Some(spill_file) = &self.inner_key_spill - { - let stream = self - .spill_manager - .read_spill_as_stream(Arc::clone(spill_file), None)?; - self.spill_stream = Some(stream); - } - + let mut spill_stream = self + .spill_manager + .read_spill_as_stream(Arc::clone(spill_file), None)?; + let mut spill_stream_has_data = false; + + // Note: the clock keeps running across the spill reads — the + // spill file is the join's own data, so reading it back is + // join work (unlike the child inputs' `next()`). while matched_count < outer_group_len { - let stream = self.spill_stream.as_mut().unwrap(); - match ready!(stream.poll_next_unpin(cx)) { + match spill_stream.next().await { Some(Ok(inner_slice)) => { - self.spill_stream_has_data = true; + spill_stream_has_data = true; matched_count = eval_filter_for_inner_slice( self.outer_is_left, filter, &outer_slice, &inner_slice, &mut self.matched, - self.outer_offset, + outer_group_start, outer_group_len, matched_count, )?; } - Some(Err(e)) => { - self.spill_stream = None; - self.spill_stream_has_data = false; - return Poll::Ready(Err(e)); - } + Some(Err(e)) => return Err(e), None => { - self.spill_stream = None; - if !self.spill_stream_has_data { - return Poll::Ready(internal_err!("Spill file was empty")); + if !spill_stream_has_data { + return internal_err!("Spill file was empty"); } break; } @@ -855,7 +782,7 @@ impl BitwiseSortMergeJoinStream { &outer_slice, inner_slice, &mut self.matched, - self.outer_offset, + outer_group_start, outer_group_len, matched_count, )?; @@ -867,357 +794,291 @@ impl BitwiseSortMergeJoinStream { self.outer_offset = outer_group_end; - self.spill_stream = None; - self.spill_stream_has_data = false; - - Poll::Ready(Ok(())) + Ok(()) } - /// Continue processing an outer key group that spans multiple outer - /// batches. Returns `true` if this outer batch was fully consumed - /// by the key group and the caller should load another. - fn resume_boundary(&mut self, cx: &mut Context<'_>) -> Poll> { - debug_assert!( - self.outer_batch.is_some(), - "caller must load outer_batch first" - ); - match self.pending_boundary.take() { - Some(PendingBoundary::NoFilter { saved_keys }) => { - let same_key = keys_match( - &saved_keys, - &self.outer_key_arrays, - &self.sort_options, - self.null_equality, - )?; - if same_key { - self.process_key_match_no_filter()?; - let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); - if self.outer_offset >= num_outer { - self.pending_boundary = Some(PendingBoundary::NoFilter { - saved_keys: slice_keys(&self.outer_key_arrays, num_outer - 1), - }); - self.emit_outer_batch()?; - self.outer_batch = None; - return Poll::Ready(Ok(true)); - } - } + /// Evaluate the filter for the buffered inner key group against the + /// outer key group. If the outer key group continues into subsequent + /// outer batches, keep evaluating there too. + async fn process_filtered_match_loop(&mut self) -> Result<()> { + loop { + self.process_key_match_with_filter().await?; + + let outer_batch = self.outer_batch.as_ref().unwrap(); + if self.outer_offset < outer_batch.num_rows() { + break; } - Some(PendingBoundary::Filtered { saved_keys }) => { - debug_assert!( - !self.inner_key_buffer.is_empty() || self.inner_key_spill.is_some(), - "Filtered pending boundary entered but no inner key data exists" - ); - let same_key = keys_match( - &saved_keys, - &self.outer_key_arrays, - &self.sort_options, - self.null_equality, - )?; - if same_key { - match self.process_key_match_with_filter(cx) { - Poll::Ready(Ok(())) => (), - Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), - Poll::Pending => { - self.pending_boundary = - Some(PendingBoundary::Filtered { saved_keys }); - return Poll::Pending; - } - } - let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); - if self.outer_offset >= num_outer { - self.pending_boundary = Some(PendingBoundary::Filtered { - saved_keys: slice_keys(&self.outer_key_arrays, num_outer - 1), - }); - self.emit_outer_batch()?; - self.outer_batch = None; - return Poll::Ready(Ok(true)); - } - } - self.clear_inner_key_group(); + + // The outer key group may continue into the next outer batch; + // save the last key so we can check. + let saved_keys = + slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1); + + self.emit_outer_batch()?; + + if !self.next_outer_batch().await? { + break; + } + if !keys_match( + &saved_keys, + &self.outer_key_arrays, + &self.sort_options, + self.null_equality, + )? { + break; } - None => {} } - Poll::Ready(Ok(false)) + + self.clear_inner_key_group(); + Ok(()) } - /// Helper to process an Equal match across potential outer batch boundaries. - fn process_filtered_match_loop(&mut self, cx: &mut Context<'_>) -> Poll> { + /// Mark the outer key group as matched. If the outer key group continues + /// into subsequent outer batches, keep marking there too. + async fn process_unfiltered_match_loop(&mut self) -> Result<()> { loop { - ready!(self.process_key_match_with_filter(cx))?; + self.mark_outer_key_group_matched()?; let outer_batch = self.outer_batch.as_ref().unwrap(); - if self.outer_offset >= outer_batch.num_rows() { - let saved_keys = - slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1); + if self.outer_offset < outer_batch.num_rows() { + return Ok(()); + } - self.emit_outer_batch()?; - self.pending_boundary = Some(PendingBoundary::Filtered { saved_keys }); + // The outer key group may continue into the next outer batch; + // save the last key so we can check. + let saved_keys = + slice_keys(&self.outer_key_arrays, outer_batch.num_rows() - 1); - // Clear stale batch before polling - self.outer_batch = None; + self.emit_outer_batch()?; - match ready!(self.poll_next_outer_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - self.pending_boundary = None; - break; - } - Ok(true) => { - let Some(PendingBoundary::Filtered { saved_keys }) = - self.pending_boundary.take() - else { - unreachable!() - }; - let same = keys_match( - &saved_keys, - &self.outer_key_arrays, - &self.sort_options, - self.null_equality, - )?; - if same { - continue; - } - break; - } - } - } else { - break; + if !self.next_outer_batch().await? { + return Ok(()); + } + if !keys_match( + &saved_keys, + &self.outer_key_arrays, + &self.sort_options, + self.null_equality, + )? { + return Ok(()); } } + } - self.clear_inner_key_group(); // This resets inner_group_buffered to false - Poll::Ready(Ok(())) + /// Keys at both cursors are equal: determine which outer rows in the key + /// group have a match. Both key groups may span batch boundaries. + async fn process_key_match(&mut self) -> Result<()> { + if self.filter.is_some() { + // Buffer the inner key group so each inner row can be evaluated + // against the outer key group, OR-ing filter results into the + // matched bitset. + self.buffer_inner_key_group().await?; + self.process_filtered_match_loop().await + } else { + // Without a filter, key equality alone means every outer row in + // the group matches; the inner rows themselves are not needed. + self.advance_inner_past_key_group().await?; + self.process_unfiltered_match_loop().await + } } - /// Main loop: drive the merge-scan to produce output batches. - fn poll_join(&mut self, cx: &mut Context<'_>) -> Poll>> { - let join_time = self.join_time.clone(); - let _timer = join_time.timer(); + /// Compare the join keys at the outer and inner cursors, returning the + /// ordering of the outer key relative to the inner key (e.g. `Greater` + /// means outer key > inner key, per the sort options). + fn compare_current_keys(&mut self) -> Result { + let (outer_idx, inner_idx) = (self.outer_offset, self.inner_offset); + Ok(self.get_outer_inner_cmp()?.compare(outer_idx, inner_idx)) + } - loop { - // 1. Ensure we have an outer batch - if self.outer_batch.is_none() { - match ready!(self.poll_next_outer_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - // Outer exhausted — flush coalescer - self.pending_boundary = None; - self.coalescer.finish_buffered_batch()?; - if let Some(batch) = self.coalescer.next_completed_batch() { - return Poll::Ready(Ok(Some(batch))); - } - return Poll::Ready(Ok(None)); - } - Ok(true) => {} // Loaded batch, move on to checks - } - } + /// Outer key is unmatched: advance the outer cursor past its key group + /// (within the current batch). If the group continues into the next + /// batch, those rows compare Less again and are skipped the same way. + fn skip_outer_key_group(&mut self) -> Result<()> { + let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); + let from = self.outer_offset; + self.outer_offset = + find_key_group_end(self.get_outer_self_cmp()?, from, num_outer); + Ok(()) + } - // Handles pausing while fetching a NEW outer batch. - if self.pending_boundary.is_some() && ready!(self.resume_boundary(cx))? { - continue; - } + /// Sync fast path for `Ordering::Greater`: skip the inner key group when + /// it ends within the current batch. Returns false — leaving all state + /// unchanged — when the group reaches the batch boundary, in which case + /// the caller must take [`Self::advance_inner_past_key_group`]. + fn try_skip_inner_key_group(&mut self) -> Result { + let num_inner = self.inner_batch.as_ref().unwrap().num_rows(); + let from = self.inner_offset; + let group_end = find_key_group_end(self.get_inner_self_cmp()?, from, num_inner); + if group_end >= num_inner { + return Ok(false); + } + self.inner_offset = group_end; + Ok(true) + } - // Handles pausing while reading the disk stream mid-batch. - if self.inner_group_buffered { - ready!(self.process_filtered_match_loop(cx))?; - continue; - } + /// Sync fast path for `Ordering::Equal` without a filter: when both key + /// groups end within their current batches (the common case — a group + /// only reaches a batch boundary once per batch), mark the outer group + /// matched and advance both cursors without any async machinery. + /// Returns false — leaving all state unchanged — when a filter is + /// present or either group reaches a batch boundary, in which case the + /// caller must take [`Self::process_key_match`]. + fn try_process_key_match(&mut self) -> Result { + if self.filter.is_some() { + return Ok(false); + } - // 2. Ensure we have an inner batch (unless inner is exhausted). - // Skip this when resuming a pending boundary — inner was already - // advanced past the key group before the boundary loop started. - if self.inner_batch.is_none() && self.pending_boundary.is_none() { - match ready!(self.poll_next_inner_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - // Inner exhausted — emit remaining outer batches. - // For semi: no more matches possible. - // For anti: all remaining outer rows are unmatched. - self.emit_outer_batch()?; - self.outer_batch = None; - - loop { - match ready!(self.poll_next_outer_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => break, - Ok(true) => { - self.emit_outer_batch()?; - self.outer_batch = None; - } - } - } + let num_inner = self.inner_batch.as_ref().unwrap().num_rows(); + let inner_from = self.inner_offset; + let inner_group_end = + find_key_group_end(self.get_inner_self_cmp()?, inner_from, num_inner); + if inner_group_end >= num_inner { + return Ok(false); + } - self.coalescer.finish_buffered_batch()?; - if let Some(batch) = self.coalescer.next_completed_batch() { - return Poll::Ready(Ok(Some(batch))); - } - return Poll::Ready(Ok(None)); - } - Ok(true) => {} - } - } + let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); + let outer_from = self.outer_offset; + let outer_group_end = + find_key_group_end(self.get_outer_self_cmp()?, outer_from, num_outer); + if outer_group_end >= num_outer { + return Ok(false); + } - // 3. Main merge-scan loop - let outer_batch = self.outer_batch.as_ref().unwrap(); - let num_outer = outer_batch.num_rows(); + for i in outer_from..outer_group_end { + self.matched.set_bit(i, true); + } + self.outer_offset = outer_group_end; + self.inner_offset = inner_group_end; + Ok(true) + } - if self.outer_offset >= num_outer { - self.emit_outer_batch()?; - self.outer_batch = None; + /// True when the outer cursor already points at an unprocessed row: the + /// sync fast path of [`Self::advance_outer_row`]. Checked inline in the + /// hot loop so the async helper (and its state machine) is only entered + /// at batch boundaries — same pattern as `sorts/merge.rs`. + fn has_current_outer_row(&self) -> bool { + self.outer_batch + .as_ref() + .is_some_and(|batch| self.outer_offset < batch.num_rows()) + } - if let Some(batch) = self.coalescer.next_completed_batch() { - return Poll::Ready(Ok(Some(batch))); - } - continue; - } + /// True when the inner cursor already points at an unprocessed row: the + /// sync fast path of [`Self::advance_inner_row`]. + fn has_current_inner_row(&self) -> bool { + self.inner_batch + .as_ref() + .is_some_and(|batch| self.inner_offset < batch.num_rows()) + } - let inner_batch = match &self.inner_batch { - Some(b) => b, - None => { + /// Ensure the outer cursor points at an unprocessed row, emitting + /// finished outer batches and loading new ones as needed. Returns false + /// when outer is exhausted. + async fn advance_outer_row( + &mut self, + emitter: &mut TryEmitter, + ) -> Result { + loop { + match &self.outer_batch { + Some(batch) if self.outer_offset < batch.num_rows() => { + return Ok(true); + } + Some(_) => { + // Current batch fully scanned — emit it and load the next. self.emit_outer_batch()?; - self.outer_batch = None; - continue; + self.emit_completed_batches(emitter).await; } - }; - let num_inner = inner_batch.num_rows(); - - if self.inner_offset >= num_inner { - match ready!(self.poll_next_inner_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - self.inner_batch = None; - continue; + None => { + if !self.next_outer_batch().await? { + return Ok(false); } - Ok(true) => continue, } } + } + } - // 4. Compare keys at current positions - self.get_outer_inner_cmp()?; - let cmp = self - .outer_inner_cmp - .as_ref() - .unwrap() - .compare(self.outer_offset, self.inner_offset); - - match cmp { - Ordering::Less => { - self.get_outer_self_cmp()?; - let group_end = find_key_group_end( - self.outer_self_cmp.as_ref().unwrap(), - self.outer_offset, - num_outer, - ); - self.outer_offset = group_end; - } + /// Ensure the inner cursor points at an unprocessed row, loading new + /// inner batches as needed. Returns false when inner is exhausted. + async fn advance_inner_row(&mut self) -> Result { + loop { + if let Some(batch) = &self.inner_batch + && self.inner_offset < batch.num_rows() + { + return Ok(true); + } + if !self.next_inner_batch().await? { + self.inner_batch = None; + return Ok(false); + } + } + } + + /// Inner is exhausted, so no further matches are possible: emit the + /// current outer batch and all remaining ones with their current matched + /// bits (semi drops unmatched rows, anti emits them, mark emits them + /// with mark=false). + async fn drain_outer(&mut self) -> Result<()> { + self.emit_outer_batch()?; + while self.next_outer_batch().await? { + self.emit_outer_batch()?; + } + Ok(()) + } + + /// Emit all completed coalescer batches to the stream consumer. + async fn emit_completed_batches( + &mut self, + emitter: &mut TryEmitter, + ) { + while let Some(batch) = self.coalescer.next_completed_batch() { + // While the emitted batch is in the consumer's hands the join + // isn't doing any work. + self.stop_join_time(); + emitter.emit(batch).await; + self.start_join_time(); + } + } + + /// Main loop: a classic merge-scan over the two sorted inputs, emitting + /// output batches as they complete. + async fn join( + &mut self, + emitter: &mut TryEmitter, + ) -> Result<()> { + // The `has_current_*` / `has_completed_batch` fast paths keep async + // state machinery out of the per-key-group hot path; the awaiting + // helpers are only entered at batch boundaries. + while self.has_current_outer_row() || self.advance_outer_row(emitter).await? { + if !(self.has_current_inner_row() || self.advance_inner_row().await?) { + self.drain_outer().await?; + break; + } + + // Each arm handles the common case synchronously (`try_*`); the + // async continuations only run when a key group reaches a batch + // boundary or a filter must be evaluated. + match self.compare_current_keys()? { + Ordering::Less => self.skip_outer_key_group()?, Ordering::Greater => { - self.get_inner_self_cmp()?; - let group_end = find_key_group_end( - self.inner_self_cmp.as_ref().unwrap(), - self.inner_offset, - num_inner, - ); - if group_end >= num_inner { - let saved_keys = - slice_keys(&self.inner_key_arrays, num_inner - 1); - match ready!(self.poll_next_inner_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - self.inner_batch = None; - continue; - } - Ok(true) => { - if keys_match( - &saved_keys, - &self.inner_key_arrays, - &self.sort_options, - self.null_equality, - )? { - match ready!(self.advance_inner_past_key_group(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(_) => continue, - } - } - continue; - } - } - } else { - self.inner_offset = group_end; + if !self.try_skip_inner_key_group()? { + self.advance_inner_past_key_group().await?; } } Ordering::Equal => { - if self.filter.is_some() { - debug_assert!(!self.inner_group_buffered); - // Buffer inner key group (may span batches) - match ready!(self.buffer_inner_key_group(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(_inner_exhausted) => { - self.inner_group_buffered = true; - } - } - // Process outer rows against buffered inner group - // (may need to handle outer batch boundary) - ready!(self.process_filtered_match_loop(cx))?; - } else { - // No filter: advance inner past key group, then - // mark all outer rows with this key as matched. - match ready!(self.advance_inner_past_key_group(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(_inner_exhausted) => {} - } - - loop { - self.process_key_match_no_filter()?; - - let num_outer = self.outer_batch.as_ref().unwrap().num_rows(); - if self.outer_offset >= num_outer { - let saved_keys = - slice_keys(&self.outer_key_arrays, num_outer - 1); - - self.emit_outer_batch()?; - self.pending_boundary = - Some(PendingBoundary::NoFilter { saved_keys }); - // Clear stale batch before polling - self.outer_batch = None; - - match ready!(self.poll_next_outer_batch(cx)) { - Err(e) => return Poll::Ready(Err(e)), - Ok(false) => { - self.pending_boundary = None; - break; - } - Ok(true) => { - let Some(PendingBoundary::NoFilter { - saved_keys, - }) = self.pending_boundary.take() - else { - unreachable!() - }; - let same_key = keys_match( - &saved_keys, - &self.outer_key_arrays, - &self.sort_options, - self.null_equality, - )?; - if same_key { - continue; - } - break; - } - } - } else { - break; - } - } + if !self.try_process_key_match()? { + self.process_key_match().await?; } } } - // Check for completed coalescer batch - if let Some(batch) = self.coalescer.next_completed_batch() { - return Poll::Ready(Ok(Some(batch))); + if self.coalescer.has_completed_batch() { + self.emit_completed_batches(emitter).await; } } + + // Flush whatever is still buffered in the coalescer. + self.coalescer.finish_buffered_batch()?; + self.emit_completed_batches(emitter).await; + Ok(()) } } @@ -1370,7 +1231,7 @@ fn evaluate_filter_for_inner_row( .as_any() .downcast_ref::() .ok_or_else(|| { - datafusion_common::DataFusionError::Internal( + DataFusionError::Internal( "Filter expression did not return BooleanArray".to_string(), ) })?; @@ -1381,21 +1242,3 @@ fn evaluate_filter_for_inner_row( Ok(bool_arr.clone()) } } - -impl Stream for BitwiseSortMergeJoinStream { - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let poll = self.poll_join(cx).map(|result| result.transpose()); - self.baseline_metrics.record_poll(poll) - } -} - -impl RecordBatchStream for BitwiseSortMergeJoinStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs index 00cac069aae5d..3b597323b2e7b 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs @@ -528,7 +528,7 @@ impl ExecutionPlan for SortMergeJoinExec { | JoinType::LeftMark | JoinType::RightMark ) { - Ok(Box::pin(BitwiseSortMergeJoinStream::try_new( + BitwiseSortMergeJoinStream::try_new( Arc::clone(&self.schema), self.sort_options.clone(), self.null_equality, @@ -544,7 +544,7 @@ impl ExecutionPlan for SortMergeJoinExec { reservation, spill_manager, context.runtime_env(), - )?)) + ) } else { Ok(Box::pin(MaterializingSortMergeJoinStream::try_new( Arc::clone(&self.schema), diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs index 64dadcb123eb7..313818cf6c6b5 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs @@ -27,6 +27,7 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use std::time::Duration; use super::bitwise_stream::BitwiseSortMergeJoinStream; use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn}; @@ -51,6 +52,7 @@ use arrow_ord::sort::SortColumn; use arrow_schema::SchemaRef; use bytes::Bytes; use datafusion_common::JoinType::*; +use datafusion_common::instant::Instant; use datafusion_common::{ JoinSide, internal_err, test_util::{batches_to_sort_string, batches_to_string}, @@ -3814,7 +3816,7 @@ async fn consume_stream_until_finish_barrier_reached( let mut after_finish_barrier_reached = vec![]; let mut background_task = JoinSet::new(); - let mut start_time_since_last_ready = datafusion_common::instant::Instant::now(); + let mut start_time_since_last_ready = Instant::now(); loop { let next_item = output_stream.next(); @@ -3834,7 +3836,7 @@ async fn consume_stream_until_finish_barrier_reached( } else { output_batched.push(batch); } - start_time_since_last_ready = datafusion_common::instant::Instant::now(); + start_time_since_last_ready = Instant::now(); } Poll::Ready(Some(Err(e))) => return Err(e), Poll::Ready(None) if !switch_to_finish_barrier => { @@ -3861,9 +3863,7 @@ async fn consume_stream_until_finish_barrier_reached( } // Make sure the test doesn't run forever - if start_time_since_last_ready.elapsed() - > std::time::Duration::from_secs(5) - { + if start_time_since_last_ready.elapsed() > Duration::from_secs(5) { return internal_err!( "Stream should have emitted data by now, but it's still pending. Output batches so far: {}", output_batched.len() @@ -4032,7 +4032,7 @@ fn columns(schema: &Schema) -> Vec { // ==================== BitwiseSortMergeJoinStream direct tests ==================== // // These tests construct a BitwiseSortMergeJoinStream directly (bypassing exec) -// to exercise async re-entry and spill edge cases using PendingStream. +// to exercise waiting on inputs and spill edge cases using PendingStream. /// Create test memory/spill resources for stream-level tests. fn test_stream_resources( @@ -4112,18 +4112,212 @@ impl RecordBatchStream for PendingStream { } /// Helper: collect all output from a BitwiseSortMergeJoinStream. -async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result> { - common::collect(Box::pin(stream)).await +async fn collect_stream(stream: SendableRecordBatchStream) -> Result> { + common::collect(stream).await +} + +// ==================== join_time metric tests ==================== +// +// These verify that `join_time` measures only the join's own work: waiting +// for either child input or for the consumer to take an emitted batch must +// not be counted. + +/// Stream that sleeps `delay` before yielding each batch, to simulate a +/// slow input. +fn delayed_stream( + batches: Vec, + delay: Duration, +) -> SendableRecordBatchStream { + let schema = batches[0].schema(); + Box::pin(crate::stream::RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches.into_iter().map(Ok)).then(move |item| async move { + tokio::time::sleep(delay).await; + item + }), + )) +} + +/// Three 2-row batches with unique matching keys. +fn join_time_batches() -> Vec { + vec![ + build_table_i32( + ("a1", &vec![0, 1]), + ("b1", &vec![1, 2]), + ("c1", &vec![7, 8]), + ), + build_table_i32( + ("a1", &vec![2, 3]), + ("b1", &vec![3, 4]), + ("c1", &vec![7, 8]), + ), + build_table_i32( + ("a1", &vec![4, 5]), + ("b1", &vec![5, 6]), + ("c1", &vec![7, 8]), + ), + ] +} + +/// Build a no-filter LeftSemi bitwise stream over the given input streams. +/// The small batch size makes each outer batch surface as its own output +/// batch, so a slow consumer test sees multiple emits. +fn join_time_test_join( + outer: SendableRecordBatchStream, + inner: SendableRecordBatchStream, +) -> (SendableRecordBatchStream, ExecutionPlanMetricsSet) { + let metrics = ExecutionPlanMetricsSet::new(); + let outer_schema = outer.schema(); + let (reservation, spill_manager, runtime_env) = + test_stream_resources(inner.schema(), &metrics); + let stream = BitwiseSortMergeJoinStream::try_new( + outer_schema, + vec![SortOptions::default()], + NullEquality::NullEqualsNothing, + outer, + inner, + vec![Arc::new(Column::new("b1", 1)) as PhysicalExprRef], + vec![Arc::new(Column::new("b1", 1)) as PhysicalExprRef], + None, + LeftSemi, + 2, + 0, + &metrics, + reservation, + spill_manager, + runtime_env, + ) + .unwrap(); + (stream, metrics) } -/// Reproduces the buffer_inner_key_group re-entry bug: +fn join_time_of(metrics: &ExecutionPlanMetricsSet) -> Duration { + Duration::from_nanos( + metrics + .clone_inner() + .sum_by_name("join_time") + .map(|m| m.as_usize()) + .unwrap_or(0) as u64, + ) +} + +/// Run a join with the given injected `delay`, retrying with 4x the delay +/// (up to 3 attempts) when `join_time < delay` fails. /// -/// When buffer_inner_key_group buffers inner rows across batch boundaries -/// and poll_next_inner_batch returns Pending mid-way, the ready! macro -/// exits poll_join. On re-entry, the merge-scan reaches Equal again and -/// calls buffer_inner_key_group a second time -- which starts with -/// clear(), destroying the partially collected inner rows. Previously -/// consumed batches are gone, so re-buffering misses them. +/// This de-flakes the check without masking real bugs: a genuine exclusion +/// bug makes `join_time` absorb the injected waits, so it scales with the +/// delay and fails at every escalation level. Only a fixed-size disturbance +/// (e.g. the OS preempting the test thread while the join_time clock is +/// running) is filtered out, since it cannot grow 4x with the delay. +/// +/// `run` returns `(join_time, wall)` for one join execution. Deterministic +/// invariants (row counts, wall-time lower bounds) stay as asserts inside +/// `run` — deliberately: a panic there fails the test immediately without +/// retrying, since those cannot flake and escalation would only mask a real +/// bug. Likewise `Err` from `run` (join execution failure) propagates +/// immediately. Only the preemption-sensitive `join_time` check is retried. +async fn check_join_time_excluded(mut run: F) -> Result<()> +where + F: FnMut(Duration) -> Fut, + Fut: Future>, +{ + let mut delay = Duration::from_millis(50); + for attempt in 0..3 { + let (join_time, wall) = run(delay).await?; + if join_time < delay { + return Ok(()); + } + assert!( + attempt < 2, + "join_time ({join_time:?}) should be well below the injected \ + delay ({delay:?}) even after escalating retries; wall {wall:?}" + ); + delay *= 4; + } + unreachable!() +} + +/// join_time must not include time spent waiting for the outer input. +#[tokio::test] +async fn join_time_excludes_outer_input_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let outer = delayed_stream(join_time_batches(), delay); + let inner = delayed_stream(join_time_batches(), Duration::ZERO); + let (stream, metrics) = join_time_test_join(outer, inner); + + let start = Instant::now(); + let batches = collect_stream(stream).await?; + let wall = start.elapsed(); + + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 6, "all outer rows should match"); + assert!( + wall >= delay * 3, + "outer delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + +/// join_time must not include time spent waiting for the inner input. +#[tokio::test] +async fn join_time_excludes_inner_input_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let outer = delayed_stream(join_time_batches(), Duration::ZERO); + let inner = delayed_stream(join_time_batches(), delay); + let (stream, metrics) = join_time_test_join(outer, inner); + + let start = Instant::now(); + let batches = collect_stream(stream).await?; + let wall = start.elapsed(); + + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 6, "all outer rows should match"); + assert!( + wall >= delay * 3, + "inner delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + +/// join_time must not include time the consumer spends holding an emitted +/// batch (the generator is suspended inside `emitter.emit` meanwhile). +#[tokio::test] +async fn join_time_excludes_consumer_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let outer = delayed_stream(join_time_batches(), Duration::ZERO); + let inner = delayed_stream(join_time_batches(), Duration::ZERO); + let (mut stream, metrics) = join_time_test_join(outer, inner); + + let start = Instant::now(); + let mut output_batches = 0u32; + while let Some(batch) = stream.next().await { + batch?; + output_batches += 1; + // Simulate a slow consumer between emitted batches. + tokio::time::sleep(delay).await; + } + let wall = start.elapsed(); + + assert!( + output_batches >= 3, + "expected multiple emitted batches, got {output_batches}" + ); + assert!( + wall >= delay * output_batches, + "consumer delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + +/// An inner key group spanning multiple inner batches must survive the inner +/// input returning Pending mid-way: inner rows delivered before the Pending +/// still take part in the filter evaluation. /// /// Setup: /// - Inner: 3 single-row batches, all with key=1, filter values c2=[10, 20, 30] @@ -4131,8 +4325,7 @@ async fn collect_stream(stream: BitwiseSortMergeJoinStream) -> Result Result<()> { let left_schema = Arc::new(Schema::new(vec![ @@ -4249,22 +4442,17 @@ async fn filter_buffer_pending_loses_inner_rows() -> Result<()> { Ok(()) } -/// Reproduces the no-filter boundary Pending re-entry bug: -/// -/// When an outer key group spans a batch boundary, the no-filter path -/// emits the current batch, then polls for the next outer batch. If -/// poll returns Pending, poll_join exits. On re-entry, without the -/// PendingBoundary fix, the new batch is processed fresh by the -/// merge-scan. Since inner already advanced past this key, the outer -/// rows with the matching key are skipped via Ordering::Less. +/// A matched outer key group spanning a batch boundary must survive the outer +/// input returning Pending at that boundary: the rows continuing the key group +/// still count as matched, even though the inner side has already advanced +/// past the key. /// /// Setup: /// - Outer: 2 single-row batches, both with key=1 (key group spans boundary) /// - Inner: 1 row with key=1 /// - Pending injected on outer before 2nd batch /// -/// Without fix: only first outer row emitted (second lost on re-entry) -/// With fix: both outer rows emitted +/// Expected: both outer rows emitted #[tokio::test] async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> { let left_schema = Arc::new(Schema::new(vec![ @@ -4353,9 +4541,8 @@ async fn no_filter_boundary_pending_loses_outer_rows() -> Result<()> { /// /// The outer input has an unmatched prefix row followed by a matching key /// group that continues in the next batch. Both rows with key=1 should be -/// treated as matched. Returning `Pending` before the second batch forces -/// `poll_join` to return and later resume from its top-level state, rather -/// than continuing the same in-progress boundary loop. +/// treated as matched. Returning `Pending` before the second batch makes the +/// join wait for the continuation while the key group is still open. #[tokio::test] async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> { let left_schema = Arc::new(Schema::new(vec![ @@ -4447,8 +4634,8 @@ async fn no_filter_boundary_pending_with_unmatched_prefix() -> Result<()> { Ok(()) } -/// Tests the filtered boundary Pending re-entry: outer key group spans -/// batches with a filter, and poll_next_outer_batch returns Pending. +/// Same as the no-filter boundary case, with a filter: the outer key group +/// spans batches and the outer input returns Pending at the boundary. /// /// Setup: /// - Outer: 2 single-row batches, both key=1, c1=[10, 20] @@ -4656,6 +4843,21 @@ async fn bitwise_spill_with_filter() -> Result<()> { metrics.spilled_rows().unwrap() > 0, "expected spilled_rows > 0 for {join_type:?}, batch_size={batch_size}" ); + let join_time = metrics + .sum_by_name("join_time") + .map(|m| m.as_usize()) + .unwrap_or(0); + assert!( + join_time > 0, + "expected join_time > 0 for {join_type:?}, batch_size={batch_size}" + ); + let output_rows = metrics.output_rows().unwrap_or(0); + let collected_rows: usize = spilled_result.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + output_rows, collected_rows, + "output_rows metric should match collected rows for \ + {join_type:?}, batch_size={batch_size}" + ); // Run without spilling and compare results let task_ctx_no_spill = Arc::new( @@ -4690,22 +4892,19 @@ async fn bitwise_spill_with_filter() -> Result<()> { Ok(()) } -/// Reproduces a bug where `resume_boundary` for the Filtered pending case -/// only checks `inner_key_buffer.is_empty()` but ignores `inner_key_spill`. -/// After spilling, the in-memory buffer is cleared while the spill file -/// holds the data. If the outer key group spans a batch boundary, the -/// second outer batch's rows are never evaluated against the inner group. +/// Once the inner key group has spilled, an outer key group spanning a batch +/// boundary must still be evaluated against the spilled inner rows — the +/// second outer batch's rows must not be treated as having no inner group to +/// match against. /// /// Setup: /// - Outer: 2 single-row batches, both key=1, c1=[10, 10] /// - Inner: 1 batch with many rows all key=1 (enough to trigger spill) /// - Filter: c1 == c2 (matches when c2=10) /// - Memory limit: tiny (100 bytes) to force spilling -/// - Pending before 2nd outer batch to trigger boundary re-entry +/// - Pending before 2nd outer batch, while the key group is still open /// /// Expected: both outer rows match (semi=2 rows, anti=0 rows) -/// Bug: second outer row is skipped because resume_boundary sees empty -/// inner_key_buffer and skips re-evaluation. #[tokio::test] async fn spill_filtered_boundary_loses_outer_rows() -> Result<()> { let left_schema = Arc::new(Schema::new(vec![ @@ -5253,8 +5452,8 @@ async fn materializing_spill_pending_stream() -> Result<()> { "expected spill_count > 0 for {join_type:?}" ); - // Compare against a no-spill run to make sure the Pending - // re-entry path didn't corrupt or drop any data. + // Compare against a no-spill run to make sure waiting on the + // spill reads didn't corrupt or drop any data. let task_ctx_no_spill = Arc::new(TaskContext::default()); let join_no_spill = join_with_options( Arc::clone(&left), @@ -5277,9 +5476,9 @@ async fn materializing_spill_pending_stream() -> Result<()> { } /// Bitwise-side (Semi/Anti) coverage: identical to `bitwise_spill_with_filter`, -/// but every spill read goes through `PendingSpillFile`, forcing -/// `process_key_match_with_filter`'s spilled-batch loop to actually hit and -/// resume from `Poll::Pending`. +/// but every spill read goes through `PendingSpillFile`, so reading the +/// spilled inner rows back must actually hit and recover from `Poll::Pending` +/// mid-read. #[tokio::test] async fn bitwise_spill_pending_stream() -> Result<()> { let left = build_table( From 73b8ad387d325eeeb72718a0dc3454cad3fba338 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jul 2026 06:23:08 +0800 Subject: [PATCH 056/109] refactor: address review feedback on percentile_cont(DISTINCT) accumulator (#23946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Rationale for this change Follow-up to post-merge review feedback from @neilconway on #23913. ## What changes are included in this PR? - Use the fast foldhash `RandomState` for the distinct-value count map instead of the standard library's default SipHash (the shared `GenericDistinctBuffer` already does this; the merged fix regressed to SipHash on this hot path). - Use `estimate_memory_size` in `size()` instead of a hand-rolled capacity calculation. - Adopt the null-free fast path in `update_batch`/`retract_batch` (skip per-element validity checks when the input has no nulls), mirroring `GenericDistinctBuffer`. - Add `ORDER BY` to the sliding-window regression test so its row order is deterministic. ## Are these changes tested? Yes — existing percentile unit tests and the full `aggregate.slt` pass. ## Are there any user-facing changes? No. --- .../src/percentile_cont.rs | 56 +++++++++++++++---- .../sqllogictest/test_files/aggregate.slt | 43 +++++++++++++- 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs index 4b5e892fb5cdf..baf1978ea24af 100644 --- a/datafusion/functions-aggregate/src/percentile_cont.rs +++ b/datafusion/functions-aggregate/src/percentile_cont.rs @@ -32,8 +32,10 @@ use arrow::{ use num_traits::AsPrimitive; use arrow::array::ArrowNativeTypeOp; +use datafusion_common::hash_utils::RandomState; use datafusion_common::internal_err; use datafusion_common::types::{NativeType, logical_float64}; +use datafusion_common::utils::memory::estimate_memory_size; use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator; use crate::min_max::{max_udaf, min_udaf}; @@ -672,7 +674,11 @@ where #[derive(Debug)] struct DistinctPercentileContAccumulator { /// Distinct value -> number of in-window rows carrying it. - counts: HashMap, usize>, + /// + /// Uses the same fast (foldhash) `RandomState` as the shared + /// `GenericDistinctBuffer` rather than the standard library's default + /// SipHash, which is considerably slower for this hot path. + counts: HashMap, usize, RandomState>, percentile: f64, } @@ -709,8 +715,15 @@ where // `values` may carry extra argument columns (e.g. the percentile // literal); only the first column holds the aggregated values. let arr = values[0].as_primitive::(); - for value in arr.iter().flatten() { - *self.counts.entry(Hashable(value)).or_default() += 1; + if arr.null_count() > 0 { + for value in arr.iter().flatten() { + *self.counts.entry(Hashable(value)).or_default() += 1; + } + } else { + // Fast path: no nulls, so skip the per-element validity check. + for value in arr.values().iter() { + *self.counts.entry(Hashable(*value)).or_default() += 1; + } } Ok(()) } @@ -733,9 +746,11 @@ where } fn size(&self) -> usize { - size_of_val(self) - + self.counts.capacity() - * (size_of::>() + size_of::()) + estimate_memory_size::<(Hashable, usize)>( + self.counts.capacity(), + size_of_val(self), + ) + .unwrap() } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { @@ -744,12 +759,31 @@ where } let arr = values[0].as_primitive::(); - for value in arr.iter().flatten() { - if let Some(count) = self.counts.get_mut(&Hashable(value)) { - *count -= 1; - if *count == 0 { - self.counts.remove(&Hashable(value)); + let mut decrement = |value: T::Native| { + match self.counts.get_mut(&Hashable(value)) { + Some(count) => { + *count -= 1; + if *count == 0 { + self.counts.remove(&Hashable(value)); + } + Ok(()) } + // Retracting a value that isn't tracked means the accumulator + // state has diverged from the window frame; continuing would + // silently produce wrong results, so surface it as an error. + None => internal_err!( + "percentile_cont(DISTINCT) retract_batch: retracted a value not present in the window" + ), + } + }; + if arr.null_count() > 0 { + for value in arr.iter().flatten() { + decrement(value)?; + } + } else { + // Fast path: no nulls, so skip the per-element validity check. + for value in arr.values().iter() { + decrement(*value)?; } } Ok(()) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index f6380c9a5f06d..26b8a78f3921a 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1122,7 +1122,8 @@ SELECT percentile_cont(DISTINCT x, 0.5) FROM distinct_pct; query IR SELECT id, percentile_cont(DISTINCT x, 0.5) OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) -FROM distinct_pct; +FROM distinct_pct +ORDER BY id; ---- 1 5 2 5 @@ -1131,6 +1132,46 @@ FROM distinct_pct; statement ok DROP TABLE distinct_pct; +# Regression: grouped percentile_cont(DISTINCT ...) forces two-phase +# (Partial + FinalPartitioned) aggregation, exercising the distinct +# accumulator's state()/merge_batch() paths. Duplicate values within a +# group must be de-duplicated across the per-partition merge. +# Group 1 distinct {1,5,9} -> median 5; group 2 distinct {3,7} -> median 5. +statement ok +CREATE TABLE grp_distinct_pct(g INT, x DOUBLE) AS VALUES + (1, 5), (1, 5), (1, 9), (1, 1), + (2, 7), (2, 7), (2, 3); + +query IR +SELECT g, percentile_cont(DISTINCT x, 0.5) FROM grp_distinct_pct GROUP BY g ORDER BY g; +---- +1 5 +2 5 + +statement ok +DROP TABLE grp_distinct_pct; + +# Regression: sliding-window percentile_cont(DISTINCT ...) over data with +# NULLs exercises the null_count() > 0 slow path in BOTH update_batch (a NULL +# row enters the frame) and retract_batch (a NULL row leaves the frame as the +# window slides). NULLs are ignored; distinct dedups the non-null values. +statement ok +CREATE TABLE distinct_pct_nulls(id INT, x DOUBLE) AS VALUES + (1, 5), (2, NULL), (3, 9), (4, 5); + +query IR +SELECT id, percentile_cont(DISTINCT x, 0.5) + OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) +FROM distinct_pct_nulls ORDER BY id; +---- +1 5 +2 5 +3 9 +4 7 + +statement ok +DROP TABLE distinct_pct_nulls; + query RT select approx_median(arrow_cast(col_f32, 'Float16')), arrow_typeof(approx_median(arrow_cast(col_f32, 'Float16'))) from median_table; ---- From cdc412455af0af373e03a5a867c8c387fd1e2824 Mon Sep 17 00:00:00 2001 From: Peter Lee Date: Wed, 29 Jul 2026 07:27:31 +0800 Subject: [PATCH 057/109] fix: last value accumulator merge indexing (#23905) ## Which issue does this PR close? - Related to apache/datafusion-comet#4131. ## Rationale for this change `TrivialLastValueAccumulator::merge_batch` filtered out unset states but then read row `0` from the remaining value array. As a result, merging multiple `LAST_VALUE` partial states returned the first valid state instead of the last one. ## What changes are included in this PR? - Select the final valid state row when merging unordered `LAST_VALUE` states. - Extend the existing merge test to verify the evaluated result. ## Are these changes tested? Yes: ```shell cargo test -p datafusion-functions-aggregate test_first_last_state_after_merge --- datafusion/functions-aggregate/src/first_last.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs index e2da7ec753aa5..ea45e42e84f33 100644 --- a/datafusion/functions-aggregate/src/first_last.rs +++ b/datafusion/functions-aggregate/src/first_last.rs @@ -1193,7 +1193,7 @@ impl Accumulator for TrivialLastValueAccumulator { if let Some(last) = filtered_states.last() && !last.is_empty() { - self.last = ScalarValue::try_from_array(last, 0)?; + self.last = ScalarValue::try_from_array(last, last.len() - 1)?; self.is_set = true; } Ok(()) @@ -1525,10 +1525,24 @@ mod tests { let merged_state = last_accumulator.state()?; assert_eq!(merged_state.len(), state1.len()); + assert_eq!(last_accumulator.evaluate()?, ScalarValue::Int64(Some(10))); Ok(()) } + #[test] + fn test_trivial_last_value_merge_all_flags_false() -> Result<()> { + let mut acc = TrivialLastValueAccumulator::try_new(&DataType::Int64, false)?; + let states: Vec = vec![ + Arc::new(Int64Array::from(vec![None, None])), + Arc::new(BooleanArray::from(vec![false, false])), + ]; + + acc.merge_batch(&states)?; + assert_eq!(acc.evaluate()?, ScalarValue::Int64(None)); + Ok(()) + } + #[test] fn test_first_group_acc() -> Result<()> { let schema = Arc::new(Schema::new(vec![ From d61d31a8fae9f6b572d896b539ea410728c1aa6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:42:04 -0700 Subject: [PATCH 058/109] chore(deps): bump syn from 2.0.119 to 3.0.2 (#23945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [syn](https://github.com/dtolnay/syn) from 2.0.119 to 3.0.2.
Release notes

Sourced from syn's releases.

3.0.2

3.0.1

3.0.0

This release contains adjustments to the syntax tree to account for ongoing Rust language development from the 3 years since syn 2.0.0 and to anticipate some in-flight Rust language RFCs.

These include: default values in fields, pinned type sugar, raw lifetimes, generator blocks and functions, unnamed enum variants, attributes in tuple types and tuple patterns, named arguments in parenthesized generic argument lists, lightweight clones, const traits, const function pointers, mutability restricted fields, supertrait auto implementation, final associated functions, trait implementability restrictions, const blocks in path arguments, item-level const blocks, return type notation, never patterns, function delegation, mutable by-reference bindings, in-place initialization, field projections, explicitly dyn-compatible traits, view types, file-level frontmatter, generic const arguments, guard patterns, lazy type aliases, explicitly safe foreign items, super let, unsafe fields, pattern types, heterogeneous try-blocks, function contracts, async function trait bounds, static closure coroutine syntax, unsafe binder types, move expressions, for-await loops, and postfix keywords.

Breaking changes

Modifiers

To reserve more room for language evolution, there are 10 new non-exhaustive structs in the syntax tree having the following commonality:

  • Name ending in Modifiers. {BlockModifiers, ClosureModifiers, ConstModifiers, FieldModifiers, FnModifiers, ImplModifiers, LocalModifiers, TraitBoundModifiers, TraitModifiers, TypeModifiers}

  • Each implements Default. The default value is guaranteed to comprise no tokens.

  • Non-exhaustive. Can only be instantiated by Syn's parser or by creating and then mutating ▁▁Modifiers::default().

  • Does not implement Parse. When parsing, they are parsed by the enclosing syntax tree node.

  • Does not implement ToTokens. In some cases the syntax that these nodes might hold in the future is not necessarily contiguous tokens.

  • Provides .require_empty() -> Result<()> which returns a meaningfully spanned error if the modifiers are different from the empty default. This enables a caller to reject syntax it does not recognize without knowing what that syntax may be.

Types

  • Type::BareFn has been renamed to Type::FnPtr to mirror the compiler's terminology. Together with this, BareVariadic is renamed to FnPtrVariadic.

  • The mutually exclusive const_token and mutability fields of Type::Ptr have been unified into an enum of type PointerMutability, which was already previously used by Expr::RawAddr.

  • Every Type variant now holds attributes, which can represent the attributes of element types inside a tuple type, or attributes for a function return type.

  • BareFnArg is renamed to NamedArg and is used in ParenthesizedGenericArguments, in addition to the existing use in Type::FnPtr.

Expressions

  • In Expr::Closure, the fields or1_token and or2_token have been renamed to inputs_begin and inputs_end to indicate the beginning and ending | token of the closure inputs.

... (truncated)

Commits
  • 88ee7be Release 3.0.2
  • 587bc20 Merge pull request #2070 from dtolnay/emptyrange
  • 96801f7 Allow Error::new_range at empty cursor range
  • 9dc16c9 Merge pull request #2069 from dtolnay/prevspan
  • 1db76b7 Align on using impl trait across all Error constructors
  • bfa1ebf Make Cursor::prev_span public
  • c6ac5e5 Merge pull request #2068 from dtolnay/newrange
  • 1436454 Add Error::new_range constructor taking Range<Cursor>
  • 123c148 Release 3.0.1
  • bc11ddd Merge pull request #2067 from dtolnay/fastpeek
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- datafusion/macros/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 245f02f8d3a7c..b0e591cb0f78d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2357,7 +2357,7 @@ version = "54.1.0" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.119", + "syn 3.0.2", ] [[package]] diff --git a/datafusion/macros/Cargo.toml b/datafusion/macros/Cargo.toml index 91f1dde62aaac..d5ab6a8fff624 100644 --- a/datafusion/macros/Cargo.toml +++ b/datafusion/macros/Cargo.toml @@ -46,4 +46,4 @@ proc-macro = true [dependencies] datafusion-doc = { workspace = true } quote = "1.0.44" -syn = { version = "2.0.117", features = ["full"] } +syn = { version = "3.0.2", features = ["full"] } From ef94a85b85bfc19042cad8c732ad1d78f07b34a5 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Tue, 28 Jul 2026 17:52:28 -0700 Subject: [PATCH 059/109] feat: support `ansi` for `elt` (#23928) Part of https://github.com/apache/datafusion/issues/23929 ## Rationale for this change Spark's `elt` errors on out-of-range indices when ANSI is on, and returns `NULL` when off. DataFusion previously always returned `NULL` (there was a `TODO` for this in the source). ## What changes are included in this PR? - `elt.rs`: read `enable_ansi_mode` from `ScalarFunctionArgs::config_options`; raise `"The index N is out of bounds. The array has M elements."` in ANSI mode, keep `NULL` behavior otherwise. `NULL` index always returns `NULL`. - `elt.slt`: add missing ANSI-off cases (index 0, negative, large, `NULL` index, `NULL` value, vectorized) and a full ANSI-on section with `statement error` assertions. ## Are these changes tested? Yes. New Rust unit tests and sqllogictest cases cover both modes. ## Are there any user-facing changes? With `enable_ansi_mode = true`, `elt` on invalid indices now errors instead of returning `NULL`. Default behavior unchanged. No API changes. --- datafusion/spark/src/function/string/elt.rs | 46 +++--- .../test_files/spark/string/elt.slt | 140 ++++++++++++++++++ 2 files changed, 163 insertions(+), 23 deletions(-) diff --git a/datafusion/spark/src/function/string/elt.rs b/datafusion/spark/src/function/string/elt.rs index e58faf0c40f93..b88477a7720f3 100644 --- a/datafusion/spark/src/function/string/elt.rs +++ b/datafusion/spark/src/function/string/elt.rs @@ -24,7 +24,7 @@ use arrow::compute::{can_cast_types, cast}; use arrow::datatypes::DataType::{Int64, Utf8}; use arrow::datatypes::{DataType, Int64Type}; use datafusion_common::cast::as_string_array; -use datafusion_common::{DataFusionError, Result, plan_datafusion_err}; +use datafusion_common::{DataFusionError, Result, exec_err, plan_datafusion_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -63,7 +63,11 @@ impl ScalarUDFImpl for SparkElt { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - make_scalar_function(elt, vec![])(&args.args) + let enable_ansi_mode = args.config_options.execution.enable_ansi_mode; + make_scalar_function( + move |arrays: &[ArrayRef]| elt(arrays, enable_ansi_mode), + vec![], + )(&args.args) } fn coerce_types(&self, arg_types: &[DataType]) -> Result> { @@ -80,18 +84,13 @@ impl ScalarUDFImpl for SparkElt { "ELT index must be Int64 (or castable to Int64), got {idx_dt:?}" ))); } - let mut coerced = Vec::with_capacity(arg_types.len()); - coerced.push(Int64); - - for _ in 1..length { - coerced.push(Utf8); - } - + let mut coerced = vec![Utf8; length]; + coerced[0] = Int64; Ok(coerced) } } -fn elt(args: &[ArrayRef]) -> Result { +fn elt(args: &[ArrayRef], enable_ansi_mode: bool) -> Result { let n_rows = args[0].len(); let idx: &PrimitiveArray = @@ -103,11 +102,10 @@ fn elt(args: &[ArrayRef]) -> Result { })?; let num_values = args.len() - 1; - let mut cols: Vec> = Vec::with_capacity(num_values); + let mut cols: Vec = Vec::with_capacity(num_values); for a in args.iter().skip(1) { let casted = cast(a, &Utf8)?; - let sa = as_string_array(&casted)?; - cols.push(Arc::new(sa.clone())); + cols.push(as_string_array(&casted)?.clone()); } let mut builder = StringBuilder::new(); @@ -120,10 +118,12 @@ fn elt(args: &[ArrayRef]) -> Result { let index = idx.value(i); - // TODO: if spark.sql.ansi.enabled is true, - // throw ArrayIndexOutOfBoundsException for invalid indices; - // if false, return NULL instead (current behavior). if index < 1 || (index as usize) > num_values { + if enable_ansi_mode { + return exec_err!( + "The index {index} is out of bounds. The array has {num_values} elements." + ); + } builder.append_null(); continue; } @@ -146,13 +146,13 @@ mod tests { use super::*; use arrow::array::Int64Array; - fn run_elt_arrays(arrs: Vec) -> Result> { - let arr = elt(&arrs)?; - let string_array = arr - .as_any() - .downcast_ref::() - .ok_or_else(|| DataFusionError::Internal("expected Utf8".into()))?; - Ok(Arc::new(string_array.clone())) + fn run_elt_arrays(arrs: Vec) -> Result { + run_elt_arrays_with(arrs, false) + } + + fn run_elt_arrays_with(arrs: Vec, ansi: bool) -> Result { + let arr = elt(&arrs, ansi)?; + Ok(as_string_array(&arr)?.clone()) } #[test] diff --git a/datafusion/sqllogictest/test_files/spark/string/elt.slt b/datafusion/sqllogictest/test_files/spark/string/elt.slt index 12917d17e1e47..9f0348324aadc 100644 --- a/datafusion/sqllogictest/test_files/spark/string/elt.slt +++ b/datafusion/sqllogictest/test_files/spark/string/elt.slt @@ -59,3 +59,143 @@ query T SELECT elt(1, 10, null) ---- 10 + +######################################## +# ANSI mode = false (default): invalid indices return NULL +######################################## + +# Index 0 -> NULL (Spark returns NULL when ANSI is off) +query T +SELECT elt(0::int, 'a', 'b'); +---- +NULL + +# Negative index -> NULL +query T +SELECT elt(-1::int, 'a', 'b'); +---- +NULL + +# Index far beyond the input list -> NULL +query T +SELECT elt(100::int, 'a', 'b', 'c'); +---- +NULL + +# NULL index -> NULL regardless of mode +query T +SELECT elt(NULL::int, 'a', 'b'); +---- +NULL + +# NULL value at the selected index -> NULL +query T +SELECT elt(2::int, 'a', NULL); +---- +NULL + +# Three-argument list, pick middle element +query T +SELECT elt(2::int, 'scala', 'java', 'python'); +---- +java + +# Three-argument list, pick last element +query T +SELECT elt(3::int, 'scala', 'java', 'python'); +---- +python + +# Mixed types get cast to string (Spark returns string) +query T +SELECT elt(2::int, 1, 2, 3); +---- +2 + +# Vectorized: mix of valid, out-of-range, and NULL indices in ANSI-off mode +statement ok +CREATE TABLE elt_rows(idx INT, a STRING, b STRING, c STRING) AS VALUES + (1, 'a1', 'b1', 'c1'), + (2, 'a2', 'b2', 'c2'), + (3, 'a3', 'b3', 'c3'), + (0, 'a4', 'b4', 'c4'), + (-1, 'a5', 'b5', 'c5'), + (4, 'a6', 'b6', 'c6'), + (NULL, 'a7', 'b7', 'c7'); + +query T +SELECT elt(idx, a, b, c) FROM elt_rows ORDER BY a; +---- +a1 +b2 +c3 +NULL +NULL +NULL +NULL + +statement ok +DROP TABLE elt_rows; + +######################################## +# ANSI mode = true: invalid indices raise ArrayIndexOutOfBoundsException +######################################## + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +# Valid indices still work +query T +SELECT elt(1::int, 'scala', 'java'); +---- +scala + +query T +SELECT elt(2::int, 'scala', 'java'); +---- +java + +# NULL index still returns NULL (matches Spark: no error when index itself is NULL) +query T +SELECT elt(NULL::int, 'a', 'b'); +---- +NULL + +# NULL value at valid index still returns NULL (only invalid indices error) +query T +SELECT elt(1::int, NULL, 'b'); +---- +NULL + +# Out-of-range positive index errors +statement error DataFusion error: Execution error: The index 3 is out of bounds\. The array has 2 elements\. +SELECT elt(3::int, 'scala', 'java'); + +# Zero index errors +statement error DataFusion error: Execution error: The index 0 is out of bounds\. The array has 2 elements\. +SELECT elt(0::int, 'scala', 'java'); + +# Negative index errors +statement error DataFusion error: Execution error: The index -1 is out of bounds\. The array has 2 elements\. +SELECT elt(-1::int, 'scala', 'java'); + +# Large positive index errors +statement error DataFusion error: Execution error: The index 100 is out of bounds\. The array has 3 elements\. +SELECT elt(100::int, 'a', 'b', 'c'); + +# Vectorized: a batch that contains any invalid index errors in ANSI mode +statement ok +CREATE TABLE elt_ansi(idx INT, a STRING, b STRING) AS VALUES + (1, 'a1', 'b1'), + (2, 'a2', 'b2'), + (3, 'a3', 'b3'); + +statement error DataFusion error: Execution error: The index 3 is out of bounds\. The array has 2 elements\. +SELECT elt(idx, a, b) FROM elt_ansi; + +statement ok +DROP TABLE elt_ansi; + +# Reset ANSI mode +statement ok +set datafusion.execution.enable_ansi_mode = false; From 74eebbb94514eea80b2b4a7745047879fc898735 Mon Sep 17 00:00:00 2001 From: Jeffrey Vo Date: Wed, 29 Jul 2026 13:27:06 +0900 Subject: [PATCH 060/109] chore: remove unused `header` file (#23958) Leftover from arrow, but not needed anyway - see https://github.com/apache/arrow/pull/45526 --- header | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 header diff --git a/header b/header deleted file mode 100644 index 70665d1a26295..0000000000000 --- a/header +++ /dev/null @@ -1,16 +0,0 @@ -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - From 68d587468073c1fc84ffe00db66f35d506dcc828 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 29 Jul 2026 14:24:41 +0800 Subject: [PATCH 061/109] feat(physical-plan): generic Rows-backed GroupColumn keeps mixed schemas on the column-wise path (#23523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #22715 (nested type coverage in `GroupValuesColumn` EPIC) - Alternative to #23128 (per-type approach) — implements the direction @alamb proposed in - Step toward the terminal goal of retiring `GroupValuesRows` entirely (#23404) ## Rationale for this change Today `GroupValuesColumn` is **all-or-nothing**: a single nested column in the GROUP BY key (\`Struct\`, \`List\`, \`FixedSizeList\`, …) makes \`supported_schema\` return \`false\` and drops the *entire* aggregation onto the row-wise \`GroupValuesRows\` fallback — even when every other column would have qualified for the column-wise fast path. For a \`GROUP BY int_col, struct_col\` shape, the \`int_col\` pays the row-encoded storage cost for no reason. ## What changes are included in this PR? Add \`RowsGroupColumn\`: a generic \`GroupColumn\` backed by a single-field \`RowConverter\`, wired in as the nested-type dispatch arm of \`group_column_supported_type\` / \`make_group_column\`. Native columns keep their type-specialized builders; the nested column pays row-encoding only for its one column. Gated to \`data_type.is_nested()\` so intentionally excluded scalar types (Float16, Decimal256) stay on \`GroupValuesRows\` and the \`group_column_supported_type\` ⇔ \`make_group_column\` invariant holds. ## Impact Memory, measured with 4000 groups of \`8 × Int64 + 1 × FixedSizeList\` in \`mixed_schema_column_path_uses_less_memory_than_rows_fallback\`: | | Bytes | vs baseline | |-------------------------------------------------------|----------|-------------| | \`GroupValuesRows\` (today's fallback) | 1096 KB | 100% | | \`GroupValuesColumn\` + \`RowsGroupColumn\` fallback | 594 KB | **54.2%** | Speed: not benchmarked as a headline result — the wins come from native columns keeping their type-specialized \`equal_to\`/\`append_val\` fast paths instead of falling back to byte-encoded row comparisons. ## Are these changes tested? Yes: - Unit tests inside \`row_backed\`: FSL / Struct roundtrip, \`take_n\`, \`supports_type\` matches \`RowConverter::supports_fields\`. - \`mixed_schema_column_path_uses_less_memory_than_rows_fallback\` (mod.rs): the 54.2% memory claim + identical group assignment vs \`GroupValuesRows\`. - \`nested_float_edge_cases_match_rows_fallback\`: nested \`-0.0\` / \`NaN\` produce the same groupings as \`GroupValuesRows\` (the correctness invariant to watch, since hashing runs on the raw column and equality runs on the row bytes). - \`multi_batch_and_emit_first_matches_rows_fallback\`: multi-batch streaming intern + \`EmitTo::First\` + \`take_n\`. All 39 tests in \`aggregates::group_values\` pass. ## Are there any user-facing changes? No — internal aggregation representation only. Same query results, lower memory footprint on mixed-schema GROUP BY keys. ## Follow-ups (out of scope) - Add coverage for any type \`RowConverter\` cannot encode (currently arrow-rs 59.x handles Map fine; \`supports_type\` delegates to \`RowConverter::supports_fields\` so it auto-tracks upstream). - Retire \`GroupValuesRows\` entirely once coverage is complete (#23404). --- .../group_values/multi_group_by/mod.rs | 268 +++++ .../group_values/multi_group_by/row_backed.rs | 1014 +++++++++++++++++ .../src/aggregates/group_values/row.rs | 98 +- 3 files changed, 1371 insertions(+), 9 deletions(-) create mode 100644 datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index f275d777c3279..5163948bd594a 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -21,6 +21,7 @@ mod boolean; mod bytes; pub mod bytes_view; pub mod primitive; +pub mod row_backed; use std::mem::{self, size_of}; @@ -28,6 +29,7 @@ use crate::aggregates::group_values::GroupValues; use crate::aggregates::group_values::multi_group_by::{ boolean::BooleanGroupValueBuilder, bytes::ByteGroupValueBuilder, bytes_view::ByteViewGroupValueBuilder, primitive::PrimitiveGroupValueBuilder, + row_backed::RowsGroupColumn, }; use arrow::array::{Array, ArrayRef, BooleanBufferBuilder}; use arrow::compute::cast; @@ -923,6 +925,15 @@ macro_rules! instantiate_primitive { /// builder for. The `group_column_supported_type_matches_make_group_column` /// test below pins this biconditional. fn group_column_supported_type(data_type: &DataType) -> bool { + // Nested types (Struct / List / LargeList / FixedSizeList, recursively) have + // no type-specialized `GroupColumn`; they are handled by the generic + // row-backed fallback in `make_group_column` whenever arrow's row format can + // encode them. Gate the fallback to nested types so intentionally-excluded + // scalar types (e.g. Float16, Decimal256) stay on `GroupValuesRows` and the + // `group_column_supported_type` ⇔ `make_group_column` invariant holds. + if data_type.is_nested() { + return RowsGroupColumn::supports_type(data_type); + } matches!( *data_type, DataType::Int8 @@ -1067,6 +1078,14 @@ fn make_group_column(field: &Field) -> Result> { v.push(Box::new(BooleanGroupValueBuilder::::new())); } } + // Generic fallback for nested types (Struct / List / LargeList / + // FixedSizeList, recursively) that lack a type-specialized builder but + // can be encoded by arrow's row format. This is what lets a mixed + // schema keep the column-wise fast path for its native columns instead + // of dropping the whole key onto `GroupValuesRows`. + ref dt if dt.is_nested() && RowsGroupColumn::supports_type(dt) => { + v.push(Box::new(RowsGroupColumn::try_new(dt.clone())?)); + } _ => return not_impl_err!("{data_type} not supported in GroupValuesColumn"), } debug_assert_eq!( @@ -1273,6 +1292,255 @@ mod tests { GroupIndexView, group_column_supported_type, make_group_column, supported_schema, }; + /// A mixed group-by key of several native columns plus one nested column + /// that has no type-specialized `GroupColumn`. + /// + /// Before the generic row-backed fallback, `supported_schema` returned + /// `false` for this schema, so the *entire* key dropped to the row-wise + /// `GroupValuesRows`. Now only the nested column pays the row-encoding + /// cost; the native columns keep their compact column-wise storage. This + /// test proves both that (a) the results are identical and (b) the + /// column-wise path now uses less memory than the all-rows fallback. + #[test] + fn mixed_schema_column_path_uses_less_memory_than_rows_fallback() { + use crate::aggregates::group_values::GroupValuesRows; + use arrow::array::{FixedSizeListArray, Int64Array}; + use arrow::datatypes::Int64Type; + + // 8 native Int64 columns + 1 FixedSizeList ("embedding"). + let fsl_field = Arc::new(Field::new("item", DataType::Int64, true)); + let mut fields: Vec = (0..8) + .map(|i| Field::new(format!("k{i}"), DataType::Int64, false)) + .collect(); + fields.push(Field::new( + "emb", + DataType::FixedSizeList(Arc::clone(&fsl_field), 4), + true, + )); + let schema: SchemaRef = Arc::new(Schema::new(fields)); + + // The whole schema must now be eligible for the column-wise path. + assert!( + supported_schema(schema.as_ref()), + "mixed native + nested schema should be column-supported now" + ); + + // Build `n_groups` distinct rows (each row is its own group). + let n_groups = 4000usize; + let mut cols: Vec = (0..8) + .map(|c| { + let vals: Vec = + (0..n_groups).map(|r| (r as i64) * 8 + c as i64).collect(); + Arc::new(Int64Array::from(vals)) as ArrayRef + }) + .collect(); + let emb: Vec>>> = (0..n_groups) + .map(|r| { + Some(vec![ + Some(r as i64), + Some(r as i64 + 1), + Some(r as i64 + 2), + Some(r as i64 + 3), + ]) + }) + .collect(); + cols.push( + Arc::new(FixedSizeListArray::from_iter_primitive::( + emb, 4, + )) as ArrayRef, + ); + + // Intern the same data into both implementations. + let mut column_path = GroupValuesColumn::::try_new(Arc::clone(&schema)) + .expect("column path"); + let mut rows_path = + GroupValuesRows::try_new(Arc::clone(&schema)).expect("rows path"); + + let mut g1 = vec![]; + let mut g2 = vec![]; + column_path.intern(&cols, &mut g1).unwrap(); + rows_path.intern(&cols, &mut g2).unwrap(); + + // (a) Correctness: same number of groups and identical group assignment. + assert_eq!(column_path.len(), n_groups); + assert_eq!(rows_path.len(), n_groups); + assert_eq!(g1, g2, "group assignment must match the rows fallback"); + + // (b) Memory: the column-wise path stores the 8 native columns compactly + // and only row-encodes the nested one, so it should be smaller than + // encoding every column into rows. + // + // The delta is only printed here — a hard `column_size < rows_size` + // assert would be brittle to future Arrow row-format or memory- + // accounting changes without reflecting a grouping-correctness + // regression. Track the memory improvement via benchmarks instead. + let column_size = column_path.size(); + let rows_size = rows_path.size(); + println!( + "mixed-schema group values size: column-wise = {column_size} bytes, \ + all-rows fallback = {rows_size} bytes \ + ({:.1}% of fallback)", + 100.0 * column_size as f64 / rows_size as f64 + ); + + // Emitted values must be equal too (compare via the rows fallback which + // is the established reference implementation). + let out_col = column_path.emit(EmitTo::All).unwrap(); + let out_row = rows_path.emit(EmitTo::All).unwrap(); + assert_eq!(out_col.len(), out_row.len()); + for (a, b) in out_col.iter().zip(out_row.iter()) { + assert_eq!(a.as_ref(), b.as_ref()); + } + } + + /// Relabel a group-index vector so labels are assigned in order of first + /// appearance. Two vectors are equivalent groupings iff their canonical + /// forms are equal — this ignores the (opaque, non-semantic) difference in + /// group-index numbering between the vectorized column path and the + /// sequential rows fallback. + /// + /// The [`GroupValues`] trait only guarantees that equal keys receive the + /// same group-id and that new keys receive a fresh id; the order in which + /// new ids are handed out is deliberately not part of the contract, and + /// can differ between correct implementations (e.g. because of internal + /// hash-map ordering). Canonicalizing before comparison is what lets us + /// assert equivalence across implementations. + fn canonical_grouping(groups: &[usize]) -> Vec { + let mut map = HashMap::new(); + let mut next = 0usize; + groups + .iter() + .map(|&g| { + *map.entry(g).or_insert_with(|| { + let v = next; + next += 1; + v + }) + }) + .collect() + } + + /// The generic row-backed column must be behavior-preserving: for the + /// nested columns it now handles, `GroupValuesColumn` must induce the same + /// grouping (partition of rows) as the established `GroupValuesRows` + /// fallback — including the float `-0.0` / `+0.0` / `NaN` edge cases decided + /// jointly by hashing and the row format. + #[test] + fn nested_float_edge_cases_match_rows_fallback() { + use crate::aggregates::group_values::GroupValuesRows; + use arrow::array::{FixedSizeListArray, Float64Array}; + + let item = Arc::new(Field::new("item", DataType::Float64, true)); + let schema: SchemaRef = Arc::new(Schema::new(vec![Field::new( + "emb", + DataType::FixedSizeList(Arc::clone(&item), 2), + true, + )])); + assert!(supported_schema(schema.as_ref())); + + // Rows exercising +0.0 vs -0.0, two NaN bit patterns, and inner nulls. + let nan = f64::NAN; + let other_nan = f64::from_bits(0x7ff8_0000_0000_0001); + let values = Float64Array::from(vec![ + Some(0.0), + Some(1.0), // [ +0.0, 1.0 ] + Some(-0.0), + Some(1.0), // [ -0.0, 1.0 ] + Some(nan), + Some(2.0), // [ NaN, 2.0 ] + Some(other_nan), + Some(2.0), // [ NaN', 2.0 ] + Some(0.0), + Some(1.0), // [ +0.0, 1.0 ] (dup of row 0) + ]); + let field_ref = Arc::new(Field::new("item", DataType::Float64, true)); + let input: ArrayRef = Arc::new(FixedSizeListArray::new( + field_ref, + 2, + Arc::new(values), + None, + )); + + let cols = vec![input]; + + let mut column_path = + GroupValuesColumn::::try_new(Arc::clone(&schema)).unwrap(); + let mut rows_path = GroupValuesRows::try_new(Arc::clone(&schema)).unwrap(); + + let mut g1 = vec![]; + let mut g2 = vec![]; + column_path.intern(&cols, &mut g1).unwrap(); + rows_path.intern(&cols, &mut g2).unwrap(); + + assert_eq!( + canonical_grouping(&g1), + canonical_grouping(&g2), + "column-wise path must induce the same grouping as the rows fallback \ + on float edge cases (got column={g1:?}, rows={g2:?})" + ); + assert_eq!(column_path.len(), rows_path.len()); + } + + /// Equivalence across multiple `intern` batches and `EmitTo::First(n)`. + #[test] + fn multi_batch_and_emit_first_matches_rows_fallback() { + use crate::aggregates::group_values::GroupValuesRows; + use arrow::array::{FixedSizeListArray, Int32Array}; + use arrow::datatypes::Int32Type; + + let item = Arc::new(Field::new("item", DataType::Int32, true)); + let schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("emb", DataType::FixedSizeList(Arc::clone(&item), 2), true), + ])); + + let make_batch = |base: i32| -> Vec { + let k = Arc::new(Int32Array::from(vec![base, base + 1, base])) as ArrayRef; + let emb: Vec>>> = vec![ + Some(vec![Some(base), Some(base)]), + Some(vec![Some(base + 1), None]), + Some(vec![Some(base), Some(base)]), // dup of row 0 + ]; + let emb = Arc::new( + FixedSizeListArray::from_iter_primitive::(emb, 2), + ) as ArrayRef; + vec![k, emb] + }; + + let mut column_path = + GroupValuesColumn::::try_new(Arc::clone(&schema)).unwrap(); + let mut rows_path = GroupValuesRows::try_new(Arc::clone(&schema)).unwrap(); + + for base in [0, 10, 0] { + let cols = make_batch(base); + let (mut a, mut b) = (vec![], vec![]); + column_path.intern(&cols, &mut a).unwrap(); + rows_path.intern(&cols, &mut b).unwrap(); + // Same grouping (partition), even if the opaque group-index labels + // differ between the vectorized and sequential paths. + assert_eq!( + canonical_grouping(&a), + canonical_grouping(&b), + "grouping must match for batch base={base}" + ); + } + + let total_groups = column_path.len(); + assert_eq!(total_groups, rows_path.len()); + + // `EmitTo::First(n)` then `EmitTo::All` on the nested column path must + // work and together emit exactly `total_groups` rows. (Cross-path value + // equality is covered by `mixed_schema_...` and the row_backed unit + // tests; group-index ordering differs here so we check counts.) + let col_first = column_path.emit(EmitTo::First(2)).unwrap(); + assert_eq!(col_first[0].len(), 2); + let col_rest = column_path.emit(EmitTo::All).unwrap(); + assert_eq!(col_first[0].len() + col_rest[0].len(), total_groups); + // Column count / schema preserved on both emits. + assert_eq!(col_first.len(), schema.fields().len()); + assert_eq!(col_rest.len(), schema.fields().len()); + } + /// CRITICAL invariant: if `group_column_supported_type(t)` returns true /// the dispatcher must accept that type at intern time, and conversely /// if `group_column_supported_type(t)` returns false the planner must diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs new file mode 100644 index 0000000000000..29beb3bd66229 --- /dev/null +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs @@ -0,0 +1,1014 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A generic [`GroupColumn`] backed by the arrow row format. +//! +//! Unlike the type-specialized builders in this module (primitive, byte, +//! boolean, ...), [`RowsGroupColumn`] works for *any* data type that arrow's +//! [`RowConverter`] can encode — including nested types such as `Struct`, +//! `List`, `LargeList` and `FixedSizeList`. It stores one group value per row +//! in a single-column [`Rows`] buffer and compares group keys by their encoded +//! bytes. +//! +//! # Why this exists +//! +//! [`GroupValuesColumn`] can only be used when *every* column of the group-by +//! key has a [`GroupColumn`] implementation; otherwise the whole aggregation +//! falls back to the row-wise [`GroupValuesRows`], which is materially slower +//! and heavier for the columns that *would* have qualified for the column-wise +//! fast path. By providing a generic fallback `GroupColumn`, a schema like +//! `GROUP BY int_col, struct_col` keeps `int_col` on its fast native builder +//! and only pays the row-encoding cost on `struct_col`, instead of dragging both +//! columns onto `GroupValuesRows`. +//! +//! # Relationship to hashing +//! +//! This column does not hash anything itself: [`GroupValuesColumn`] hashes the +//! raw input columns via `create_hashes`, which already supports nested types. +//! Equality is decided here by comparing arrow-row bytes. For the two to agree +//! on group identity, values that this column considers equal must hash equal — +//! see the float `-0.0` / `NaN` note on [`RowsGroupColumn`]. +//! +//! [`GroupValuesColumn`]: crate::aggregates::group_values::multi_group_by::GroupValuesColumn +//! [`GroupValuesRows`]: crate::aggregates::group_values::GroupValuesRows + +use crate::aggregates::group_values::multi_group_by::GroupColumn; +use crate::aggregates::group_values::row::encode_array_if_necessary; + +use arrow::array::{Array, ArrayRef, BooleanBufferBuilder}; +use arrow::datatypes::DataType; +use arrow::row::{RowConverter, Rows, SortField}; +use datafusion_common::{DataFusionError, Result}; + +/// A [`GroupColumn`] that stores group values for a single column in the arrow +/// [row format], backed by a single-field [`RowConverter`]. +/// +/// # NULL semantics +/// +/// The [`GroupColumn`] contract treats two NULLs as equal. The row format +/// encodes NULL with a distinct sentinel, so `null`-row bytes compare equal to +/// each other and unequal to any non-null row — matching the contract without +/// special-casing. +/// +/// # Float `-0.0` / `NaN` +/// +/// Equality here is byte equality under arrow's IEEE-754 *totalOrder* row +/// encoding, which treats `-0.0` and `+0.0` as distinct and canonicalizes +/// `NaN`. Because hashing is performed separately (on the raw input array), a +/// caller must ensure the two agree — e.g. by normalizing `-0.0 → +0.0` on the +/// input columns before hashing when a float leaf is present (as +/// [`GroupValuesRows`] does). See the module docs. +/// +/// [row format]: arrow::row +/// [`GroupValuesRows`]: crate::aggregates::group_values::GroupValuesRows +pub struct RowsGroupColumn { + /// Single-field row converter for this column's data type. + row_converter: RowConverter, + /// Accumulated group values in row format; `group_values.row(i)` is the + /// group value for group index `i`. + group_values: Rows, + /// The column's expected output type. The row format decodes dictionary / + /// run-end encoded values to their plain value type, so emitted arrays are + /// re-encoded to this type in `build` / `take_n` (mirroring + /// `GroupValuesRows::emit`). + output_type: DataType, +} + +/// Walk `data_type`'s subtree and return `true` if it contains a +/// [`DataType::FixedSizeList`] whose descendant tree includes any +/// [`DataType::Dictionary`]. +/// +/// Two-state recursion: once we cross a `FixedSizeList`, `inside_fsl` +/// stays true for every descendant, so a `Dictionary` anywhere below +/// counts. Above that boundary, encountering a `Dictionary` is fine — +/// only nested containers propagate the risk. +/// +/// TODO: this guard works around +/// (`decode_fixed_size_list` panics instead of applying the +/// dictionary-flatten `corrected_type` step). Fixed upstream by +/// (merged 2026-07-24, not +/// yet in a release as of arrow 59.1.0). Once DataFusion upgrades to an +/// arrow release containing that fix, `FixedSizeList` will +/// decode like the other list-likes (flattened child, re-encoded by +/// `encode_array_if_necessary`'s existing `FixedSizeList` arm) — remove +/// this guard and its `supports_type` rejection at that point. +fn contains_fsl_with_dictionary(data_type: &DataType) -> bool { + fn walk(dt: &DataType, inside_fsl: bool) -> bool { + match dt { + DataType::Dictionary(_, _) => inside_fsl, + DataType::FixedSizeList(f, _) => walk(f.data_type(), true), + DataType::List(f) + | DataType::LargeList(f) + | DataType::ListView(f) + | DataType::LargeListView(f) => walk(f.data_type(), inside_fsl), + DataType::Map(f, _) => walk(f.data_type(), inside_fsl), + DataType::Struct(fs) => fs.iter().any(|f| walk(f.data_type(), inside_fsl)), + DataType::RunEndEncoded(_, values) => walk(values.data_type(), inside_fsl), + DataType::Union(fs, _) => { + fs.iter().any(|(_, f)| walk(f.data_type(), inside_fsl)) + } + _ => false, + } + } + walk(data_type, false) +} + +/// Return `true` if `data_type` contains a [`DataType::Union`] or +/// [`DataType::RunEndEncoded`] anywhere in its subtree. +/// +/// These two nested variants can round-trip through `RowConverter` in +/// principle, but their arrow-row decoders have not been validated by +/// this crate's test matrix against the full range of leaf types (dict, +/// nested, etc.). Before this PR both were handled by `GroupValuesRows` +/// (they were not `is_nested`-eligible for `GroupValuesColumn`), so +/// reject them here to preserve the pre-PR routing rather than route +/// untested shapes through `RowsGroupColumn`. When we grow explicit +/// round-trip tests for these types, this blacklist can be removed. +fn contains_union_or_run_end_encoded(data_type: &DataType) -> bool { + match data_type { + DataType::Union(_, _) | DataType::RunEndEncoded(_, _) => true, + DataType::List(f) + | DataType::LargeList(f) + | DataType::ListView(f) + | DataType::LargeListView(f) + | DataType::FixedSizeList(f, _) => { + contains_union_or_run_end_encoded(f.data_type()) + } + DataType::Map(f, _) => contains_union_or_run_end_encoded(f.data_type()), + DataType::Struct(fs) => fs + .iter() + .any(|f| contains_union_or_run_end_encoded(f.data_type())), + _ => false, + } +} + +impl RowsGroupColumn { + /// Returns whether `data_type` can be handled by this generic column. + /// + /// This is stricter than [`RowConverter::supports_fields`]: the row + /// format also has to survive the `build` / `take_n` reverse trip + /// through [`RowConverter::convert_rows`], and arrow's + /// `decode_fixed_size_list` (arrow-row 59.1.0) skips the + /// dictionary-flatten correction that the other list-like decoders + /// apply, so any `FixedSizeList` containing a `Dictionary` leaf + /// panics on emit with `"FixedSizeListArray expected data type + /// Dictionary(...) got for \"item\""`. + /// + /// Reject those shapes here so `make_group_column` falls back to + /// `GroupValuesRows`. The other list-likes (`List`, `LargeList`, + /// `ListView`, `LargeListView`, `Map`) do carry the correction, so + /// they decode without panicking — but the correction *flattens* any + /// dictionary child to its value type, so `build` / `take_n` must + /// re-encode the emitted array back to `output_type` via + /// `encode_array_if_necessary` (which has a reconstruction arm for + /// each of these containers). + /// + /// Additionally, `Union` and `RunEndEncoded` are rejected because + /// they were routed to `GroupValuesRows` before this column existed + /// and their arrow-row round-trip has not been covered by this + /// crate's tests yet. Keeping them on the pre-PR path avoids + /// introducing an untested code path for those types. + pub fn supports_type(data_type: &DataType) -> bool { + if contains_fsl_with_dictionary(data_type) { + return false; + } + if contains_union_or_run_end_encoded(data_type) { + return false; + } + RowConverter::supports_fields(&[SortField::new(data_type.clone())]) + } + + /// Create an empty [`RowsGroupColumn`] for `data_type`. + pub fn try_new(data_type: DataType) -> Result { + let row_converter = RowConverter::new(vec![SortField::new(data_type.clone())])?; + let group_values = row_converter.empty_rows(0, 0); + Ok(Self { + row_converter, + group_values, + output_type: data_type, + }) + } + + /// Materialize `rows` into a single array of `self.output_type`, re-applying + /// dictionary / run-end encoding the row format strips on decode. + fn rows_to_array<'a>( + &self, + rows: impl IntoIterator>, + ) -> ArrayRef { + let mut arrays = self + .row_converter + .convert_rows(rows) + .expect("row conversion during emit"); + debug_assert_eq!(arrays.len(), 1, "single-field row converter"); + let array = arrays.swap_remove(0); + encode_array_if_necessary(&array, &self.output_type) + .expect("dictionary re-encode during emit") + } + + /// Encode a whole incoming column into the row format. + fn convert(&self, array: &ArrayRef) -> Result { + self.row_converter + .convert_columns(std::slice::from_ref(array)) + .map_err(DataFusionError::from) + } +} + +impl GroupColumn for RowsGroupColumn { + fn equal_to(&self, lhs_row: usize, array: &ArrayRef, rhs_row: usize) -> bool { + // Scalar path (hash-collision remainder / streaming). Encode just the + // single incoming row rather than the whole column. The vectorized + // methods below encode the batch once; this path is expected to be rare. + let incoming = self + .convert(&array.slice(rhs_row, 1)) + .expect("row conversion during equal_to"); + self.group_values.row(lhs_row) == incoming.row(0) + } + + fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> { + let incoming = self.convert(&array.slice(row, 1))?; + self.group_values.push(incoming.row(0)); + Ok(()) + } + + fn vectorized_equal_to( + &self, + lhs_rows: &[usize], + array: &ArrayRef, + rhs_rows: &[usize], + equal_to_results: &mut BooleanBufferBuilder, + ) { + // Encode the incoming column once for the whole batch. + let incoming = self + .convert(array) + .expect("row conversion during vectorized_equal_to"); + for (idx, (&lhs_row, &rhs_row)) in + lhs_rows.iter().zip(rhs_rows.iter()).enumerate() + { + // Preserve the AND-accumulate contract: skip rows already false. + if !equal_to_results.get_bit(idx) { + continue; + } + if self.group_values.row(lhs_row) != incoming.row(rhs_row) { + equal_to_results.set_bit(idx, false); + } + } + } + + fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()> { + // Encode the incoming column once, then push the selected rows. + let incoming = self.convert(array)?; + for &row in rows { + self.group_values.push(incoming.row(row)); + } + Ok(()) + } + + fn len(&self) -> usize { + self.group_values.num_rows() + } + + fn size(&self) -> usize { + self.row_converter.size() + self.group_values.size() + } + + fn build(self: Box) -> ArrayRef { + self.rows_to_array(&self.group_values) + } + + fn take_n(&mut self, n: usize) -> ArrayRef { + debug_assert!(n <= self.group_values.num_rows()); + + // Materialize the first `n` group rows. + let output = self.rows_to_array(self.group_values.iter().take(n)); + + // Shift the remaining rows to the front by rebuilding the buffer. + // TODO: mirror the arrow-rs efficiency TODO in `GroupValuesRows::emit`. + let mut remaining = self.row_converter.empty_rows(0, 0); + for row in self.group_values.iter().skip(n) { + remaining.push(row); + } + self.group_values = remaining; + + output + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use arrow::array::{Array, ArrayRef, FixedSizeListArray, Int32Array, StructArray}; + use arrow::datatypes::{DataType, Field, Int32Type}; + use std::sync::Arc; + + fn fsl_i32(data: Vec>>>, list_len: i32) -> ArrayRef { + Arc::new(FixedSizeListArray::from_iter_primitive::( + data, list_len, + )) + } + + /// The generic column must agree with a per-row reference for equality, + /// including inner-null and outer-null rows, on a `FixedSizeList`. + #[test] + fn fsl_append_equal_to_build_roundtrip() { + let dt = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Int32, true)), + 2, + ); + let mut col = Box::new(RowsGroupColumn::try_new(dt).unwrap()); + + // group values: [1,2], null-outer, [3, null-inner] + let input = fsl_i32( + vec![ + Some(vec![Some(1), Some(2)]), + None, + Some(vec![Some(3), None]), + ], + 2, + ); + + col.vectorized_append(&input, &[0, 1, 2]).unwrap(); + assert_eq!(col.len(), 3); + + // Probe with a fresh batch: row0 == group0, row1 (null) == group1, + // row2 differs from group0, row3 (inner null) == group2. + let probe = fsl_i32( + vec![ + Some(vec![Some(1), Some(2)]), // == g0 + None, // == g1 + Some(vec![Some(9), Some(9)]), // != g0 + Some(vec![Some(3), None]), // == g2 + ], + 2, + ); + + assert!(col.equal_to(0, &probe, 0)); + assert!(col.equal_to(1, &probe, 1)); + assert!(!col.equal_to(0, &probe, 2)); + assert!(col.equal_to(2, &probe, 3)); + + // Vectorized equal_to should match the scalar reference. + let mut results = BooleanBufferBuilder::new(3); + results.append_n(3, true); + col.vectorized_equal_to(&[0, 1, 2], &probe, &[0, 1, 3], &mut results); + assert!(results.get_bit(0)); + assert!(results.get_bit(1)); + assert!(results.get_bit(2)); + + // build() must reproduce the original group values. + let out = col.build(); + let out = out.as_any().downcast_ref::().unwrap(); + assert_eq!(out.len(), 3); + assert!(out.is_null(1)); + assert!(!out.is_null(0)); + } + + /// `take_n` must emit the first `n` rows and shift the rest to the front. + #[test] + fn fsl_take_n_shifts_remaining() { + let dt = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Int32, true)), + 1, + ); + let mut col = RowsGroupColumn::try_new(dt).unwrap(); + + let input = fsl_i32( + vec![ + Some(vec![Some(10)]), + Some(vec![Some(20)]), + Some(vec![Some(30)]), + ], + 1, + ); + col.vectorized_append(&input, &[0, 1, 2]).unwrap(); + + let first = col.take_n(1); + let first = first.as_any().downcast_ref::().unwrap(); + let first_vals = first + .value(0) + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + assert_eq!(first_vals.value(0), 10); + assert_eq!(col.len(), 2); + + // Remaining 20, 30 should now be at indices 0, 1. + let rest = Box::new(col).build(); + let rest = rest.as_any().downcast_ref::().unwrap(); + assert_eq!(rest.len(), 2); + let g0 = rest + .value(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!(g0, 20); + } + + /// Works for `Struct` too — proves the column is type-generic. + #[test] + fn struct_roundtrip() { + let dt = DataType::Struct(vec![Field::new("a", DataType::Int32, true)].into()); + let mut col = RowsGroupColumn::try_new(dt).unwrap(); + + let a: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2)])); + let input: ArrayRef = Arc::new(StructArray::new( + vec![Field::new("a", DataType::Int32, true)].into(), + vec![a], + None, + )); + col.vectorized_append(&input, &[0, 1]).unwrap(); + assert_eq!(col.len(), 2); + assert!(col.equal_to(0, &input, 0)); + assert!(!col.equal_to(0, &input, 1)); + } + + #[test] + fn supports_type_matches_row_converter_impl() { + assert!(RowsGroupColumn::supports_type(&DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Int32, true)), + 3 + ))); + assert!(RowsGroupColumn::supports_type(&DataType::Struct( + vec![Field::new("a", DataType::Int32, true)].into() + ))); + // Whether Map is encodable depends on the arrow-rs version. + // Just assert that our `supports_type` agrees with arrow's + // `RowConverter::supports_fields` — either both accept it or both + // reject it. Both are correct wrt the invariant. + let map_field = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("keys", DataType::Int32, false), + Field::new("values", DataType::Int32, true), + ] + .into(), + ), + false, + )); + let map_dt = DataType::Map(map_field, false); + let arrow_supports = + RowConverter::supports_fields(&[SortField::new(map_dt.clone())]); + assert_eq!(RowsGroupColumn::supports_type(&map_dt), arrow_supports); + } + + /// Regression test for the nested-container recursion in + /// [`crate::aggregates::group_values::row::encode_array_if_necessary`]. + /// `RowConverter` flattens dictionary values on the way in, so a + /// `List>` schema round-trips with `Utf8` values + /// unless the helper re-encodes the leaf. Without that recursion, + /// `build()` would emit an array whose data type does not match the + /// group column's declared type. + #[test] + fn build_preserves_list_of_dictionary_schema() { + use arrow::array::{DictionaryArray, ListArray, StringArray}; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::Int32Type; + + let dict_dt = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let item_field = Arc::new(Field::new("item", dict_dt.clone(), true)); + let outer_dt = DataType::List(Arc::clone(&item_field)); + + // Skip if this arrow-rs version rejects the nesting — the invariant we + // care about is `output().data_type() == declared type` conditional on + // supports_type saying yes. + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + + // Build List> of one row = ["a", "b"]. + let values = Arc::new(StringArray::from(vec!["a", "b"])); + let keys = Int32Array::from(vec![0, 1]); + let dict = DictionaryArray::::try_new(keys, values).unwrap(); + let offsets = OffsetBuffer::from_lengths([2]); + let list = + ListArray::try_new(Arc::clone(&item_field), offsets, Arc::new(dict), None) + .unwrap(); + let input: ArrayRef = Arc::new(list); + + col.vectorized_append(&input, &[0]).unwrap(); + let built = col.build(); + assert_eq!( + built.data_type(), + &outer_dt, + "build() must return the declared List data type, \ + not the RowConverter-flattened List", + ); + } + + // ---- FSL rejection ---------------------------------------- + // + // arrow-row 59.1.0's `decode_fixed_size_list` skips the + // dict-flatten correction that the generic `decode` path applies + // to `List` / `LargeList` / `ListView` / `LargeListView` / `Map`, + // so any `FixedSizeList` containing a `Dictionary` leaf panics on + // emit. `supports_type` must reject those shapes so + // `GroupValuesRows` fallback handles them instead. These tests pin + // the current shape of that black-list. + + fn dict_utf8() -> DataType { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) + } + + fn fsl_of(inner: DataType) -> DataType { + DataType::FixedSizeList(Arc::new(Field::new("item", inner, true)), 2) + } + + #[test] + fn supports_type_rejects_fixed_size_list_of_dict() { + // Direct case: `FixedSizeList>`. + assert!(!RowsGroupColumn::supports_type(&fsl_of(dict_utf8()))); + } + + #[test] + fn supports_type_rejects_fsl_with_dict_nested_in_struct() { + // The dict is one level deep under a struct that is itself the + // FSL element. arrow-row still panics because `convert_raw` + // returns the struct with a decoded (Utf8) field while the + // FSL builder expects the declared struct-with-dict shape. + let struct_dt = DataType::Struct(vec![Field::new("d", dict_utf8(), true)].into()); + assert!(!RowsGroupColumn::supports_type(&fsl_of(struct_dt))); + } + + #[test] + fn supports_type_rejects_fsl_with_dict_nested_in_list() { + // `FixedSizeList>` — the inner `List` handles + // dicts correctly on its own, but the outer FSL wrapper still + // panics with the mismatched declared child type. + let list_of_dict = + DataType::List(Arc::new(Field::new("item", dict_utf8(), true))); + assert!(!RowsGroupColumn::supports_type(&fsl_of(list_of_dict))); + } + + #[test] + fn supports_type_rejects_fsl_hidden_under_outer_list() { + // Sibling positioning: the outer container is a `List` (which is + // fine on its own), but its child is a `FixedSizeList`. + // The panic surface is at the inner FSL layer regardless of what + // wraps it, so this must still be rejected. + let outer = + DataType::List(Arc::new(Field::new("item", fsl_of(dict_utf8()), true))); + assert!(!RowsGroupColumn::supports_type(&outer)); + } + + #[test] + fn supports_type_rejects_fsl_hidden_under_outer_struct() { + // Same, but the outer wrapper is a struct. + let outer = + DataType::Struct(vec![Field::new("f", fsl_of(dict_utf8()), true)].into()); + assert!(!RowsGroupColumn::supports_type(&outer)); + } + + // ---- FSL without dicts is still fine ---------------------------- + + #[test] + fn supports_type_accepts_fsl_of_primitive() { + // Sanity: a plain FSL must not get caught by the + // dict-under-FSL blacklist. + assert!(RowsGroupColumn::supports_type(&fsl_of(DataType::Int32))); + } + + #[test] + fn supports_type_accepts_fsl_of_struct_without_dict() { + // FSL of struct where the struct's fields are all primitives. + let struct_dt = + DataType::Struct(vec![Field::new("a", DataType::Int32, true)].into()); + assert!(RowsGroupColumn::supports_type(&fsl_of(struct_dt))); + } + + // ---- Positive round-trip tests for non-FSL list-likes ----------- + // + // The other list-like decoders in arrow-row 59.1.0 + // (`GenericListArrayOrMap` path) apply the corrected_type fix, so + // `List`, `LargeList`, `ListView`, `LargeListView` + // and `Map<..., Dict>` all round-trip cleanly. These tests pin + // that they are (a) accepted by `supports_type` and (b) actually + // survive `vectorized_append` + `build()` without panicking, so a + // future arrow-rs regression there is caught here rather than in + // production. + + #[test] + fn supports_type_accepts_large_list_of_dict() { + let dt = DataType::LargeList(Arc::new(Field::new("item", dict_utf8(), true))); + assert!(RowsGroupColumn::supports_type(&dt)); + } + + #[test] + fn supports_type_accepts_list_view_of_dict() { + let dt = DataType::ListView(Arc::new(Field::new("item", dict_utf8(), true))); + assert!(RowsGroupColumn::supports_type(&dt)); + } + + #[test] + fn supports_type_accepts_large_list_view_of_dict() { + let dt = DataType::LargeListView(Arc::new(Field::new("item", dict_utf8(), true))); + assert!(RowsGroupColumn::supports_type(&dt)); + } + + #[test] + fn supports_type_map_agrees_with_row_converter() { + // Map>. Whether arrow-row supports Map + // depends on the version; either way, our `supports_type` must + // agree with `RowConverter::supports_fields` — otherwise we'd + // pick a strategy the converter can't back. + let entries = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("keys", DataType::Int32, false), + Field::new("values", dict_utf8(), true), + ] + .into(), + ), + false, + )); + let map_dt = DataType::Map(entries, false); + let arrow_supports = + RowConverter::supports_fields(&[SortField::new(map_dt.clone())]); + assert_eq!(RowsGroupColumn::supports_type(&map_dt), arrow_supports); + } + + /// End-to-end regression: `LargeList>` must + /// actually survive `vectorized_append` + `build()` on the current + /// arrow-rs version, not just be accepted by `supports_type`. + #[test] + fn build_preserves_large_list_of_dictionary_schema() { + use arrow::array::{DictionaryArray, LargeListArray, StringArray}; + use arrow::buffer::OffsetBuffer; + + let item_field = Arc::new(Field::new("item", dict_utf8(), true)); + let outer_dt = DataType::LargeList(Arc::clone(&item_field)); + + // Skip if this arrow-rs version rejects the nesting (defensive: + // the invariant we care about is `output().data_type() == declared` + // conditional on `supports_type` saying yes). + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + + let values = Arc::new(StringArray::from(vec!["a", "b"])); + let keys = Int32Array::from(vec![0, 1]); + let dict = DictionaryArray::::try_new(keys, values).unwrap(); + let offsets = OffsetBuffer::::from_lengths([2]); + let list = LargeListArray::try_new( + Arc::clone(&item_field), + offsets, + Arc::new(dict), + None, + ) + .unwrap(); + + col.vectorized_append(&(Arc::new(list) as ArrayRef), &[0]) + .unwrap(); + let built = col.build(); + assert_eq!( + built.data_type(), + &outer_dt, + "LargeList: build() must preserve the declared type", + ); + } + + /// Build a two-row `ListView>` array with rows + /// `["a", "b"]` and `["c"]` — the shape from the review reproducer: + /// `arrow_cast(a, 'ListView(Dictionary(Int32, Utf8))')`. + fn list_view_of_dict_input() -> (DataType, ArrayRef) { + use arrow::array::{DictionaryArray, ListViewArray, StringArray}; + use arrow::buffer::ScalarBuffer; + + let item_field = Arc::new(Field::new("item", dict_utf8(), true)); + let outer_dt = DataType::ListView(Arc::clone(&item_field)); + + let values = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let keys = Int32Array::from(vec![0, 1, 2]); + let dict = DictionaryArray::::try_new(keys, values).unwrap(); + let offsets = ScalarBuffer::::from(vec![0, 2]); + let sizes = ScalarBuffer::::from(vec![2, 1]); + let list = ListViewArray::try_new( + Arc::clone(&item_field), + offsets, + sizes, + Arc::new(dict), + None, + ) + .unwrap(); + (outer_dt, Arc::new(list) as ArrayRef) + } + + /// `ListView`: arrow-row's `decode_list_view` flattens the + /// dictionary child (`corrected_type`), so `build` must re-encode + /// the emitted array back to the declared type. Regression for the + /// review reproducer that failed with + /// `expected ListView(Dictionary(Int32, Utf8)) but found ListView(Utf8)`. + #[test] + fn build_preserves_list_view_of_dictionary_schema() { + let (outer_dt, input) = list_view_of_dict_input(); + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + col.vectorized_append(&input, &[0, 1]).unwrap(); + assert_eq!(col.len(), 2); + + let built = col.build(); + assert_eq!( + built.data_type(), + &outer_dt, + "ListView: build() must return the declared type, \ + not the RowConverter-flattened ListView", + ); + assert_eq!(built.len(), 2); + } + + /// Same regression through the `take_n` path (used by + /// `EmitTo::First(n)`), including the type of the *remaining* + /// values emitted by a subsequent `build`. + #[test] + fn take_n_preserves_list_view_of_dictionary_schema() { + let (outer_dt, input) = list_view_of_dict_input(); + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + col.vectorized_append(&input, &[0, 1]).unwrap(); + + let taken = col.take_n(1); + assert_eq!( + taken.data_type(), + &outer_dt, + "ListView: take_n() must return the declared type", + ); + assert_eq!(taken.len(), 1); + + let rest = col.build(); + assert_eq!( + rest.data_type(), + &outer_dt, + "ListView: build() after take_n must also preserve the type", + ); + assert_eq!(rest.len(), 1); + } + + /// `LargeListView` fails the same way as `ListView` + /// per the review; cover both `build` and `take_n`. + #[test] + fn build_and_take_n_preserve_large_list_view_of_dictionary_schema() { + use arrow::array::{DictionaryArray, LargeListViewArray, StringArray}; + use arrow::buffer::ScalarBuffer; + + let item_field = Arc::new(Field::new("item", dict_utf8(), true)); + let outer_dt = DataType::LargeListView(Arc::clone(&item_field)); + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let values = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let keys = Int32Array::from(vec![0, 1, 2]); + let dict = DictionaryArray::::try_new(keys, values).unwrap(); + let offsets = ScalarBuffer::::from(vec![0, 2]); + let sizes = ScalarBuffer::::from(vec![2, 1]); + let list = LargeListViewArray::try_new( + Arc::clone(&item_field), + offsets, + sizes, + Arc::new(dict), + None, + ) + .unwrap(); + let input: ArrayRef = Arc::new(list); + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + col.vectorized_append(&input, &[0, 1]).unwrap(); + + let taken = col.take_n(1); + assert_eq!( + taken.data_type(), + &outer_dt, + "LargeListView: take_n() must return the declared type", + ); + + let rest = col.build(); + assert_eq!( + rest.data_type(), + &outer_dt, + "LargeListView: build() must return the declared type", + ); + assert_eq!(rest.len(), 1); + } + + /// Group-identity must survive the dictionary flatten + re-encode + /// round trip: appending the same logical list twice (with distinct + /// dictionary key mappings) must map to one group, a different list + /// to another. Mirrors the review reproducer's GROUP BY semantics + /// (2 distinct groups from 3 input rows). + #[test] + fn list_view_of_dict_groups_by_logical_value() { + use arrow::array::{DictionaryArray, ListViewArray, StringArray}; + use arrow::buffer::ScalarBuffer; + + let item_field = Arc::new(Field::new("item", dict_utf8(), true)); + let outer_dt = DataType::ListView(Arc::clone(&item_field)); + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + // Rows: ["a","b"], ["a","b"], ["c"] → 2 distinct groups. + let values = Arc::new(StringArray::from(vec!["a", "b", "a", "b", "c"])); + let keys = Int32Array::from(vec![0, 1, 2, 3, 4]); + let dict = DictionaryArray::::try_new(keys, values).unwrap(); + let offsets = ScalarBuffer::::from(vec![0, 2, 4]); + let sizes = ScalarBuffer::::from(vec![2, 2, 1]); + let list = ListViewArray::try_new( + Arc::clone(&item_field), + offsets, + sizes, + Arc::new(dict), + None, + ) + .unwrap(); + let input: ArrayRef = Arc::new(list); + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + // Append row 0 as group 0. + col.vectorized_append(&input, &[0]).unwrap(); + // Row 1 must compare equal to group 0 (same logical value). + assert!( + col.equal_to(0, &input, 1), + "identical logical lists must be equal regardless of dict keys", + ); + // Row 2 must not. + assert!( + !col.equal_to(0, &input, 2), + "different logical lists must not be equal", + ); + + col.vectorized_append(&input, &[2]).unwrap(); + assert_eq!(col.len(), 2, "3 input rows → 2 distinct groups"); + + let built = col.build(); + assert_eq!(built.data_type(), &outer_dt); + assert_eq!(built.len(), 2); + } + + /// End-to-end regression for `Map>` when + /// arrow-row supports it. Same intent as the LargeList test. + #[test] + fn build_preserves_map_of_dictionary_schema() { + use arrow::array::{ + DictionaryArray, Int32Array, MapArray, StringArray, StructArray, + }; + use arrow::buffer::OffsetBuffer; + + let key_field = Arc::new(Field::new("keys", DataType::Int32, false)); + let value_field = Arc::new(Field::new("values", dict_utf8(), true)); + let entries_field = Arc::new(Field::new( + "entries", + DataType::Struct(vec![(*key_field).clone(), (*value_field).clone()].into()), + false, + )); + let outer_dt = DataType::Map(Arc::clone(&entries_field), false); + + if !RowsGroupColumn::supports_type(&outer_dt) { + return; + } + + let mut col = Box::new(RowsGroupColumn::try_new(outer_dt.clone()).unwrap()); + + // One map entry: {1 -> "a"}. + let keys = Arc::new(Int32Array::from(vec![1])) as ArrayRef; + let values_arr = Arc::new(StringArray::from(vec!["a"])); + let value_keys = Int32Array::from(vec![0]); + let value_dict = + DictionaryArray::::try_new(value_keys, values_arr).unwrap(); + let entries = StructArray::try_new( + vec![(*key_field).clone(), (*value_field).clone()].into(), + vec![keys, Arc::new(value_dict)], + None, + ) + .unwrap(); + let offsets = OffsetBuffer::::from_lengths([1]); + let map = + MapArray::try_new(Arc::clone(&entries_field), offsets, entries, None, false) + .unwrap(); + + col.vectorized_append(&(Arc::new(map) as ArrayRef), &[0]) + .unwrap(); + let built = col.build(); + assert_eq!( + built.data_type(), + &outer_dt, + "Map<..., Dict>: build() must preserve the declared type", + ); + } + + // ---- Union / RunEndEncoded defensive rejection ----------------- + // + // Before this PR both types were routed to `GroupValuesRows` + // (`group_column_supported_type` didn't have a nested branch). This + // PR added `is_nested`-based dispatch to `RowsGroupColumn`, which + // would opt them in — but the arrow-row round-trip for these two + // families hasn't been covered by our tests. Reject them here so + // the pre-PR routing is preserved; drop the blacklist when the + // round-trip matrix grows to include them. + + #[test] + fn supports_type_rejects_union() { + use arrow::datatypes::UnionFields; + + let fields = UnionFields::try_new( + vec![0_i8, 1_i8], + vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, true), + ], + ) + .unwrap(); + let dt = DataType::Union(fields, arrow::datatypes::UnionMode::Dense); + assert!( + !RowsGroupColumn::supports_type(&dt), + "Union must fall back to GroupValuesRows until arrow-row \ + round-trip is covered by our tests", + ); + } + + #[test] + fn supports_type_rejects_run_end_encoded_with_nested_values() { + // REE with `is_nested() = true` (nested values) is what this PR + // could otherwise opt into RowsGroupColumn; keep it on + // GroupValuesRows. + let list_of_i32 = + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + let dt = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new("values", list_of_i32, true)), + ); + assert!(!RowsGroupColumn::supports_type(&dt)); + } + + #[test] + fn supports_type_rejects_run_end_encoded_with_scalar_values() { + // REE with scalar values is `is_nested() == false`, so + // `group_column_supported_type` never routes it to us via the + // nested branch anyway — but pin the invariant explicitly so a + // future refactor doesn't accidentally opt it in. + let dt = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new("values", DataType::Utf8, true)), + ); + assert!(!RowsGroupColumn::supports_type(&dt)); + } + + #[test] + fn supports_type_rejects_ree_hidden_under_outer_wrapper() { + // REE buried under a struct or list: still rejected because + // the wrapper's decoder recurses through the REE branch we + // haven't validated. + let ree = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new("values", DataType::Utf8, true)), + ); + let outer = DataType::Struct(vec![Field::new("f", ree, true)].into()); + assert!(!RowsGroupColumn::supports_type(&outer)); + } + + #[test] + fn supports_type_accepts_plain_list_and_struct_still() { + // Sanity: the defensive Union/REE blacklist must not accidentally + // catch the well-tested list-likes / structs that this column + // exists to serve. + let list_of_int = + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + assert!(RowsGroupColumn::supports_type(&list_of_int)); + + let struct_of_prims = DataType::Struct( + vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, true), + ] + .into(), + ); + assert!(RowsGroupColumn::supports_type(&struct_of_prims)); + } +} diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rs index 4976a098ecee5..cbd7a609c5caa 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/row.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs @@ -17,7 +17,8 @@ use crate::aggregates::group_values::GroupValues; use arrow::array::{ - Array, ArrayRef, ListArray, PrimitiveArray, RunArray, StructArray, + Array, ArrayRef, FixedSizeListArray, LargeListArray, LargeListViewArray, ListArray, + ListViewArray, MapArray, PrimitiveArray, RunArray, StructArray, downcast_run_end_index, }; use arrow::compute::cast; @@ -247,7 +248,7 @@ impl GroupValues for GroupValuesRows { // https://github.com/apache/datafusion/issues/7647 for (field, array) in self.schema.fields.iter().zip(&mut output) { let expected = field.data_type(); - *array = dictionary_encode_if_necessary(array, expected)?; + *array = encode_array_if_necessary(array, expected)?; } self.group_values = Some(group_values); @@ -267,7 +268,17 @@ impl GroupValues for GroupValuesRows { } } -fn dictionary_encode_if_necessary( +/// Re-apply dictionary / run-end encoding to `array` so it matches `expected`. +/// +/// Arrow's [`RowConverter`] flattens dictionary and run-end-encoded values to +/// their plain value type during row encoding (at [`RowConverter::append`]), +/// so any group-value array produced from the row format is in that plain +/// type and must be re-encoded to match the schema's expected type before +/// being returned. Shared with the generic row-backed `GroupColumn`. +/// +/// [`RowConverter`]: arrow::row::RowConverter +/// [`RowConverter::append`]: arrow::row::RowConverter::append +pub(crate) fn encode_array_if_necessary( array: &ArrayRef, expected: &DataType, ) -> Result { @@ -278,7 +289,7 @@ fn dictionary_encode_if_necessary( .iter() .zip(struct_array.columns()) .map(|(expected_field, column)| { - dictionary_encode_if_necessary(column, expected_field.data_type()) + encode_array_if_necessary(column, expected_field.data_type()) }) .collect::>>()?; @@ -294,13 +305,82 @@ fn dictionary_encode_if_necessary( Ok(Arc::new(ListArray::try_new( Arc::::clone(expected_field), list.offsets().clone(), - dictionary_encode_if_necessary( - list.values(), - expected_field.data_type(), - )?, + encode_array_if_necessary(list.values(), expected_field.data_type())?, + list.nulls().cloned(), + )?)) + } + (DataType::LargeList(expected_field), &DataType::LargeList(_)) => { + let list = array.as_any().downcast_ref::().unwrap(); + + Ok(Arc::new(LargeListArray::try_new( + Arc::::clone(expected_field), + list.offsets().clone(), + encode_array_if_necessary(list.values(), expected_field.data_type())?, + list.nulls().cloned(), + )?)) + } + (DataType::ListView(expected_field), &DataType::ListView(_)) => { + // arrow-row's `decode_list_view` applies the dictionary-flatten + // `corrected_type` to the child, so a `ListView>` + // decodes as `ListView` and the child must be + // re-encoded here (same as `List` above, plus the `sizes` + // buffer that view-lists carry). + let list = array.as_any().downcast_ref::().unwrap(); + + Ok(Arc::new(ListViewArray::try_new( + Arc::::clone(expected_field), + list.offsets().clone(), + list.sizes().clone(), + encode_array_if_necessary(list.values(), expected_field.data_type())?, + list.nulls().cloned(), + )?)) + } + (DataType::LargeListView(expected_field), &DataType::LargeListView(_)) => { + let list = array.as_any().downcast_ref::().unwrap(); + + Ok(Arc::new(LargeListViewArray::try_new( + Arc::::clone(expected_field), + list.offsets().clone(), + list.sizes().clone(), + encode_array_if_necessary(list.values(), expected_field.data_type())?, + list.nulls().cloned(), + )?)) + } + ( + DataType::FixedSizeList(expected_field, expected_size), + &DataType::FixedSizeList(_, _), + ) => { + let list = array.as_any().downcast_ref::().unwrap(); + + Ok(Arc::new(FixedSizeListArray::try_new( + Arc::::clone(expected_field), + *expected_size, + encode_array_if_necessary(list.values(), expected_field.data_type())?, list.nulls().cloned(), )?)) } + (DataType::Map(expected_entries_field, ordered), &DataType::Map(_, _)) => { + let map = array.as_any().downcast_ref::().unwrap(); + // Re-encode the entries `StructArray` (which holds key/value + // columns) against the expected entries field's struct type. + let entries_as_ref: ArrayRef = Arc::new(map.entries().clone()); + let entries = encode_array_if_necessary( + &entries_as_ref, + expected_entries_field.data_type(), + )?; + let entries = entries + .as_any() + .downcast_ref::() + .expect("Map entries recurse must yield a StructArray") + .clone(); + Ok(Arc::new(MapArray::try_new( + Arc::::clone(expected_entries_field), + map.offsets().clone(), + entries, + map.nulls().cloned(), + *ordered, + )?)) + } (DataType::Dictionary(_, _), _) => Ok(cast(array.as_ref(), expected)?), ( DataType::RunEndEncoded(run_ends_field, expected_values_field), @@ -312,7 +392,7 @@ fn dictionary_encode_if_necessary( .as_any() .downcast_ref::>() .unwrap(); - let values = dictionary_encode_if_necessary( + let values = encode_array_if_necessary( &(Arc::clone(run_array.values()) as ArrayRef), expected_values_field.data_type(), )?; From cd28203e3e499cdf2ccfb53ef1e15ab856f4e51c Mon Sep 17 00:00:00 2001 From: Fred Thomas <1321800+fred1268@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:42:28 +0200 Subject: [PATCH 062/109] perf: `array_agg()` performance improvements (#23716) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23715. ## Rationale for this change Queries using `array_agg(DISTINCT col)` were significantly slower than expected. Profiling revealed that `DistinctArrayAggAccumulator::update_batch` was allocating a heap-owned `String` on every single input row — even for rows whose value was already present in the accumulator. For a typical low-cardinality workload (e.g. a column of ~25 database names across thousands of rows), this meant paying the full allocation cost for every duplicate, which dominated the runtime. ## What changes are included in this PR? This PR applies the same deduplication strategy already used by `AggregateExec` for `GROUP BY`: duplicate rows now cost only a hash probe with no heap allocation, and new distinct values are appended to a single shared buffer rather than allocated individually. The fix applies to all column types, not just strings, and `retract_batch` support (required for sliding window frames such as `ROWS BETWEEN N PRECEDING AND CURRENT ROW`) is fully preserved. ## Are these changes tested? Four unit tests were added to `DistinctArrayAggAccumulator` — one each for `Utf8`, `Int64`, `Float64`, and `Date32` — to pin the deduplication contract across the most common column types and serve as a regression guard for future changes. The existing sliding window sqllogictest suite (`array_agg_sliding_window.slt`) covers `retract_batch` correctness end-to-end and passes unchanged. Two `update_batch` micro-benchmarks were added to measure the before/after on realistic data: one with low cardinality (~25 distinct database names in 8 192 rows, modelling the common production case) and one with high cardinality (~7 800 distinct values, modelling the worst case where almost every row is new). Results on an 8 192-row batch: | Benchmark | Before | After | Speedup | |---|---|---|---| | Low cardinality (~25 distinct DB names) | 648.6 µs | 189.4 µs | **3.42×** | | High cardinality (~7 800 distinct values) | 1078.3 µs | 269.4 µs | **4.00×** | ## Are there any user-facing changes? No user-facing changes No breaking changes to public APIs --------- Co-authored-by: Andrew Lamb --- Cargo.lock | 1 + datafusion/functions-aggregate/Cargo.toml | 1 + .../functions-aggregate/src/array_agg.rs | 408 ++++++++++++++---- 3 files changed, 330 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0e591cb0f78d..a41734e064d4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2270,6 +2270,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", "rand 0.9.4", diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml index e75afc8f0b4bc..3ac882e9c6855 100644 --- a/datafusion/functions-aggregate/Cargo.toml +++ b/datafusion/functions-aggregate/Cargo.toml @@ -51,6 +51,7 @@ datafusion-macros = { workspace = true } datafusion-physical-expr = { workspace = true } datafusion-physical-expr-common = { workspace = true } half = { workspace = true } +hashbrown = { workspace = true } log = { workspace = true } num-traits = { workspace = true } diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index 4b0a9d3ddadca..cfacd771968c2 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -18,7 +18,7 @@ //! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`] use std::cmp::Ordering; -use std::collections::{HashMap, VecDeque}; +use std::collections::VecDeque; use std::mem::{size_of, size_of_val, take}; use std::sync::Arc; @@ -27,10 +27,13 @@ use arrow::array::{ UInt32Array, new_empty_array, }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; -use arrow::compute::{SortOptions, filter}; +use arrow::compute::{SortOptions, cast, filter}; use arrow::datatypes::{DataType, Field, FieldRef, Fields}; +use arrow::row::{OwnedRow, Row, RowConverter, Rows, SortField}; use datafusion_common::cast::as_list_array; +use datafusion_common::hash_utils::{RandomState, create_hashes}; +use datafusion_common::utils::proxy::HashTableAllocExt; use datafusion_common::utils::{ SingleRowListArrayBuilder, compare_rows, get_row_at_idx, take_function_args, }; @@ -49,6 +52,7 @@ use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity; use datafusion_functions_aggregate_common::utils::ordering_fields; use datafusion_macros::user_doc; use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; +use hashbrown::hash_table::HashTable; make_udaf_expr_and_func!( ArrayAgg, @@ -808,17 +812,67 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator { } } +/// Resources that are allocated lazily on the first `update_batch` call, +/// once the concrete runtime Arrow type is known. +/// +/// Grouping all three fields together makes the "either all present or all +/// absent" invariant explicit in the type system, replacing the scattered +/// `.expect()` calls that would otherwise be needed. +#[derive(Debug)] +struct DistinctState { + /// Converts Arrow arrays to/from the comparable row format. + converter: RowConverter, + /// One owned encoded row per live distinct value, indexed by group index. + /// Compacted via swap-remove on eviction so there are never dead slots. + group_rows: Vec, + /// Live refcount per group index. `counts[i]` is how many times the value + /// at `group_rows[i]` is currently present in the window frame. + counts: Vec, + /// Hash of the encoded row at group index `i`, kept in sync with + /// `group_rows` and `counts`. Needed to patch the map on swap-remove + /// eviction without re-encoding the moved row. + row_hashes: Vec, + /// Temporary buffer for encoding an incoming batch; reused across calls. + rows_buffer: Rows, +} + #[derive(Debug)] pub struct DistinctArrayAggAccumulator { - // Value → live refcount. Multiset state lets `retract_batch` correctly - // drop a duplicate occurrence while keeping the key alive if other - // copies remain in the current window frame. - values: HashMap, + /// Lazily allocated on the first `update_batch`; `None` until then. + state: Option, + /// Hash table storing `(hash, group_index)`. Only contains live entries + /// (those whose count is > 0). Evicted on `retract_batch` when count + /// drops to zero. + map: HashTable<(u64, usize)>, + /// Heap size of `map` in bytes, tracked for `size()` reporting. + map_size: usize, + /// Reused buffer for batch hashes. + hashes_buffer: Vec, + /// Random state used by `create_hashes`. + random_state: RandomState, datatype: DataType, sort_options: Option, ignore_nulls: bool, } +/// Returns `true` if `dt` is, or recursively contains, a `Dictionary` type. +/// +/// `RowConverter` always decodes to the physical (non-dictionary) type, so a +/// cast back to the declared logical type is required when this is true. +fn datatype_contains_dictionary(dt: &DataType) -> bool { + match dt { + DataType::Dictionary(_, _) => true, + DataType::List(f) + | DataType::LargeList(f) + | DataType::FixedSizeList(f, _) + | DataType::Map(f, _) => datatype_contains_dictionary(f.data_type()), + DataType::Struct(fields) => fields + .iter() + .any(|f| datatype_contains_dictionary(f.data_type())), + _ => false, + } +} + impl DistinctArrayAggAccumulator { pub fn try_new( datatype: &DataType, @@ -826,12 +880,37 @@ impl DistinctArrayAggAccumulator { ignore_nulls: bool, ) -> Result { Ok(Self { - values: HashMap::new(), + state: None, + map: HashTable::new(), + map_size: 0, + hashes_buffer: Vec::new(), + random_state: RandomState::default(), datatype: datatype.clone(), sort_options, ignore_nulls, }) } + + /// Lazily initialises the `DistinctState` on the first call, using the + /// actual runtime column type. + fn ensure_state(&mut self, data_type: &DataType) -> Result<()> { + if self.state.is_none() { + let sort_field = match self.sort_options { + Some(opts) => SortField::new_with_options(data_type.clone(), opts), + None => SortField::new(data_type.clone()), + }; + let converter = RowConverter::new(vec![sort_field])?; + let rows_buffer = converter.empty_rows(0, 0); + self.state = Some(DistinctState { + converter, + group_rows: Vec::new(), + counts: Vec::new(), + row_hashes: Vec::new(), + rows_buffer, + }); + } + Ok(()) + } } impl Accumulator for DistinctArrayAggAccumulator { @@ -845,22 +924,76 @@ impl Accumulator for DistinctArrayAggAccumulator { } let val = &values[0]; - let nulls = if self.ignore_nulls { - val.logical_nulls() + + // Filter nulls out upfront when ignore_nulls is set so they are + // never inserted into the dedup state. + let filtered; + let col: &ArrayRef = if self.ignore_nulls { + if let Some(nulls) = val.logical_nulls() { + if nulls.null_count() > 0 { + let mask: BooleanArray = nulls.iter().map(Some).collect(); + filtered = filter(val.as_ref(), &mask)?; + &filtered + } else { + val + } + } else { + val + } } else { - None + val }; - let nulls = nulls.as_ref(); - if nulls.is_none_or(|nulls| nulls.null_count() < val.len()) { - for i in 0..val.len() { - if nulls.is_none_or(|nulls| nulls.is_valid(i)) { - let key = ScalarValue::try_from_array(val, i)?.compacted(); - *self.values.entry(key).or_insert(0) += 1; + if col.is_empty() { + return Ok(()); + } + + self.ensure_state(col.data_type())?; + + // Encode the entire incoming batch into rows_buffer in one pass. + let DistinctState { + converter, + group_rows, + counts, + row_hashes, + rows_buffer, + } = self.state.as_mut().unwrap(); + rows_buffer.clear(); + converter.append(rows_buffer, std::slice::from_ref(col))?; + + // Pre-compute all hashes for the batch in one SIMD-friendly pass. + self.hashes_buffer.clear(); + self.hashes_buffer.resize(col.len(), 0); + create_hashes( + std::slice::from_ref(col), + &self.random_state, + &mut self.hashes_buffer, + )?; + + for (row_idx, &hash) in self.hashes_buffer.iter().enumerate() { + let row = rows_buffer.row(row_idx); + let entry = self.map.find_mut(hash, |&(h, group_idx)| { + h == hash && group_rows[group_idx].row() == row + }); + match entry { + Some((_, group_idx)) => { + // Already known: just increment the live refcount. + counts[*group_idx] += 1; + } + None => { + // New distinct value: own the encoded row, record it. + let new_group_idx = group_rows.len(); + group_rows.push(row.owned()); + counts.push(1); + row_hashes.push(hash); + self.map.insert_accounted( + (hash, new_group_idx), + |&(h, _)| h, + &mut self.map_size, + ); } } } - Ok(()) } @@ -871,12 +1004,7 @@ impl Accumulator for DistinctArrayAggAccumulator { assert_eq_or_internal_err!(states.len(), 1, "expects single state"); - // The DISTINCT state schema is `List` — partial accumulators - // ship the set of values they saw, not multiplicities. Re-ingesting - // each element here makes the merged counts represent "partitions - // that emitted this value," which is fine because `evaluate` only - // reads keys. Refcount semantics for retract are only valid within - // a single accumulator instance (window execution). + // The DISTINCT state is `List`. states[0] .as_list::() .iter() @@ -885,38 +1013,52 @@ impl Accumulator for DistinctArrayAggAccumulator { } fn evaluate(&mut self) -> Result { - let mut values: Vec = self.values.keys().cloned().collect(); - if values.is_empty() { + if self.map.is_empty() { return Ok(ScalarValue::new_null_list(self.datatype.clone(), true, 1)); } - if let Some(opts) = self.sort_options { - let mut delayed_cmp_err = Ok(()); - values.sort_by(|a, b| { - if a.is_null() { - return match opts.nulls_first { - true => Ordering::Less, - false => Ordering::Greater, - }; - } - if b.is_null() { - return match opts.nulls_first { - true => Ordering::Greater, - false => Ordering::Less, - }; - } - match opts.descending { - true => b.try_cmp(a), - false => a.try_cmp(b), - } - .unwrap_or_else(|err| { - delayed_cmp_err = Err(err); - Ordering::Equal - }) - }); - delayed_cmp_err?; + let DistinctState { + converter, + group_rows, + .. + } = self + .state + .as_ref() + .expect("state must be set when map is non-empty"); + + // Collect the group indices of all live entries. + let mut live_indices: Vec = + self.map.iter().map(|&(_, group_idx)| group_idx).collect(); + + // If ORDER BY was specified, the RowConverter bakes the sort direction + // into the row bytes, so lexicographic sort gives the correct order. + if self.sort_options.is_some() { + live_indices + .sort_unstable_by(|&a, &b| group_rows[a].row().cmp(&group_rows[b].row())); + } + + // Decode the selected rows back into an Arrow array. + let rows: Vec> = + live_indices.iter().map(|&i| group_rows[i].row()).collect(); + let arrays = converter.convert_rows(rows)?; + + // `convert_rows` always returns the physical (non-dictionary) type. + // Cast back to the declared logical type when they differ AND the + // declared type contains a Dictionary somewhere (directly or nested + // inside a Struct, List, etc.) — that is the only case where + // RowConverter strips the logical type. + let decoded = if arrays[0].data_type() != &self.datatype + && datatype_contains_dictionary(&self.datatype) + { + cast(arrays[0].as_ref(), &self.datatype)? + } else { + Arc::clone(&arrays[0]) }; + let values: Vec = (0..decoded.len()) + .map(|i| ScalarValue::try_from_array(decoded.as_ref(), i)) + .collect::>()?; + let arr = ScalarValue::new_list(&values, &self.datatype, true); Ok(ScalarValue::List(arr)) } @@ -929,33 +1071,94 @@ impl Accumulator for DistinctArrayAggAccumulator { assert_eq_or_internal_err!(values.len(), 1, "expects single batch"); let val = &values[0]; - let nulls = if self.ignore_nulls { - val.logical_nulls() + + // Mirror the null-filtering logic from update_batch so we only + // retract values that were actually inserted. + let filtered; + let col: &ArrayRef = if self.ignore_nulls { + if let Some(nulls) = val.logical_nulls() { + if nulls.null_count() > 0 { + let mask: BooleanArray = nulls.iter().map(Some).collect(); + filtered = filter(val.as_ref(), &mask)?; + &filtered + } else { + val + } + } else { + val + } } else { - None + val }; - let nulls = nulls.as_ref(); - for i in 0..val.len() { - if nulls.is_some_and(|nulls| !nulls.is_valid(i)) { - continue; - } - let key = ScalarValue::try_from_array(val, i)?; - match self.values.get_mut(&key) { - Some(count) => { - *count -= 1; - if *count == 0 { - self.values.remove(&key); - } - } - None => { + if col.is_empty() { + return Ok(()); + } + + let DistinctState { + converter, + group_rows, + counts, + row_hashes, + rows_buffer, + } = self + .state + .as_mut() + .expect("retract_batch called before update_batch"); + + rows_buffer.clear(); + converter.append(rows_buffer, std::slice::from_ref(col))?; + + self.hashes_buffer.clear(); + self.hashes_buffer.resize(col.len(), 0); + create_hashes( + std::slice::from_ref(col), + &self.random_state, + &mut self.hashes_buffer, + )?; + + for (row_idx, &hash) in self.hashes_buffer.iter().enumerate() { + let row = rows_buffer.row(row_idx); + match self.map.find_entry(hash, |&(h, group_idx)| { + h == hash && group_rows[group_idx].row() == row + }) { + Err(_) => { return internal_err!( - "DistinctArrayAggAccumulator::retract_batch: value not present in state" + "DistinctArrayAggAccumulator::retract_batch: \ + value not present in state" ); } + Ok(occupied) => { + let (_, dead_idx) = *occupied.get(); + counts[dead_idx] -= 1; + if counts[dead_idx] == 0 { + occupied.remove(); + // Compact via swap-remove: move the last slot into the + // dead slot so group_rows / counts / row_hashes stay + // dense with no dead entries. + let last_idx = group_rows.len() - 1; + if dead_idx != last_idx { + // Patch the map entry that points to last_idx so + // it points to dead_idx instead. + let last_hash = row_hashes[last_idx]; + self.map + .find_mut(last_hash, |&(_, idx)| idx == last_idx) + .ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "DistinctArrayAggAccumulator: map is missing \ + group index {last_idx} during swap-remove \ + compaction" + ) + })? + .1 = dead_idx; + } + group_rows.swap_remove(dead_idx); + counts.swap_remove(dead_idx); + row_hashes.swap_remove(dead_idx); + } + } } } - Ok(()) } @@ -964,12 +1167,26 @@ impl Accumulator for DistinctArrayAggAccumulator { } fn size(&self) -> usize { - size_of_val(self) + ScalarValue::size_of_hashmap(&self.values) - - size_of_val(&self.values) + size_of_val(self) + + self + .state + .as_ref() + .map(|s| { + s.group_rows + .iter() + .map(|r| r.row().data().len()) + .sum::() + + s.group_rows.capacity() * size_of::() + + s.counts.capacity() * size_of::() + + s.row_hashes.capacity() * size_of::() + + s.rows_buffer.size() + + s.converter.size() + }) + .unwrap_or(0) + + self.map_size + + self.hashes_buffer.capacity() * size_of::() + self.datatype.size() - size_of_val(&self.datatype) - - size_of_val(&self.sort_options) - + size_of::>() } } @@ -1541,8 +1758,7 @@ mod tests { acc2.update_batch(&[string_list_data([vec!["e", "f", "g"]])])?; acc1 = merge(acc1, acc2)?; - // without compaction, the size is 16684 - assert_eq!(acc1.size(), 1684); + assert_eq!(acc1.size(), 2274); Ok(()) } @@ -2678,6 +2894,39 @@ mod tests { Ok(()) } + #[test] + fn distinct_array_agg_dictionary_preserves_type() -> Result<()> { + use arrow::array::{DictionaryArray, Int32Array, StringArray}; + + // Dictionary(Int32, Utf8) input with duplicates. + let keys = Int32Array::from(vec![0, 1, 0, 2, 1]); // "a", "b", "a", "c", "b" + let values = StringArray::from(vec!["a", "b", "c"]); + let dict: ArrayRef = Arc::new(DictionaryArray::new(keys, Arc::new(values))); + + let datatype = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + let mut acc = DistinctArrayAggAccumulator::try_new(&datatype, None, false)?; + acc.update_batch(&[dict])?; + + let result = acc.evaluate()?; + let ScalarValue::List(arr) = &result else { + panic!("expected ScalarValue::List, got {result:?}"); + }; + + // The element type of the returned list must stay Dictionary(Int32, Utf8), + // not be silently widened to Utf8. + assert_eq!( + arr.values().data_type(), + &datatype, + "element type must be Dictionary(Int32, Utf8), got {}", + arr.values().data_type() + ); + + // There should be exactly 3 distinct values. + assert_eq!(arr.value(0).len(), 3); + Ok(()) + } + #[test] fn distinct_array_agg_date32_deduplicates() -> Result<()> { use arrow::array::Date32Array; @@ -2713,10 +2962,9 @@ mod tests { fn distinct_retract_memory_is_bounded() -> Result<()> { use arrow::array::Int64Array; - // Emulates `ROWS BETWEEN CURRENT ROW AND CURRENT ROW`: every row enters - // the frame and immediately leaves it again. Only `CARDINALITY` distinct - // values are ever seen and the live set never holds more than one of - // them + // Emulates a sliding window where each value enters and immediately + // leaves. Only CARDINALITY distinct values are ever live at once; + // memory must not grow with the number of rows processed. const CARDINALITY: i64 = 10; const WARMUP_ROWS: i64 = 1_000; const EXTRA_ROWS: i64 = 20_000; @@ -2748,7 +2996,7 @@ mod tests { WARMUP_ROWS + EXTRA_ROWS ); - // Everything was retracted, so nothing is left in the frame. + // Everything was retracted so evaluate must return null. let result = acc.evaluate()?; assert!( matches!(&result, ScalarValue::List(arr) if arr.is_null(0)), From 5f7feda66b1f460e9655ba33e749ae0fa8602553 Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:39:45 +0800 Subject: [PATCH 063/109] fix: accept LargeUtf8 and Utf8View patterns in SIMILAR TO planning (#23735) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23732 ## Rationale for this change Follow-up to #23704, which made the `TypeCoercion` analyzer coerce `SIMILAR TO` operands to a common string type. The SQL planner still rejects `LargeUtf8` and `Utf8View` **patterns** before the analyzer ever runs: ```sql SELECT s FROM test, patterns WHERE s SIMILAR TO arrow_cast(pat, 'Utf8View'); -- error: Invalid pattern in SIMILAR TO expression ``` Now that the analyzer coerces both operands, this planner restriction is unnecessary (and inconsistent with `LIKE`, whose planner path has no such check). ## What changes are included in this PR? - `datafusion/sql/src/expr/mod.rs`: the pattern-type check in `sql_similarto_to_expr` now accepts `Utf8` / `LargeUtf8` / `Utf8View` / `Null` instead of only `Utf8` / `Null`. - `datafusion/sqllogictest/test_files/type_coercion.slt`: regression tests with non-literal `Utf8View` and `LargeUtf8` patterns (plus a `NOT SIMILAR TO` case), next to the existing #22886 regression tests. The planner relaxation and tests were originally part of #22887; this PR lands the remaining piece of it. ## Are these changes tested? Yes — new sqllogictest cases in `type_coercion.slt`. The existing planner rejection of non-string patterns (`SELECT 'a' SIMILAR TO 1`) still errors as before. ## Are there any user-facing changes? Yes: `SIMILAR TO` queries with non-literal `LargeUtf8` / `Utf8View` patterns that previously failed during SQL planning with `Invalid pattern in SIMILAR TO expression` now plan and execute successfully. No breaking API changes. --- datafusion/sql/src/expr/mod.rs | 4 -- .../sqllogictest/test_files/type_coercion.slt | 46 +++++++++++++++++-- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index c00dcb82ff3a9..c2e4822f76b99 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -1008,10 +1008,6 @@ impl SqlToRel<'_, S> { planner_context: &mut PlannerContext, ) -> Result { let pattern = self.sql_expr_to_logical_expr(pattern, schema, planner_context)?; - let pattern_type = pattern.get_type(schema)?; - if pattern_type != DataType::Utf8 && pattern_type != DataType::Null { - return plan_err!("Invalid pattern in SIMILAR TO expression"); - } let escape_char = match escape_char.map(|v| v.value) { Some(Value::SingleQuotedString(char)) if char.len() == 1 => { Some(char.chars().next().unwrap()) diff --git a/datafusion/sqllogictest/test_files/type_coercion.slt b/datafusion/sqllogictest/test_files/type_coercion.slt index 7ec0f5f1dba30..6a56fc2407a94 100644 --- a/datafusion/sqllogictest/test_files/type_coercion.slt +++ b/datafusion/sqllogictest/test_files/type_coercion.slt @@ -304,7 +304,9 @@ query error does not support zero arguments SELECT * FROM (SELECT 1) WHERE TRY_CAST(STARTS_WITH() AS INT) = 1; ################################################################### -## SIMILAR TO type coercion (https://github.com/apache/datafusion/issues/22886) +## SIMILAR TO type coercion +## https://github.com/apache/datafusion/issues/22886 +## https://github.com/apache/datafusion/issues/23732 ################################################################### # NULL pattern is coerced to a typed NULL and evaluates to NULL instead of panicking @@ -349,6 +351,44 @@ SELECT arrow_cast(t.s, 'Dictionary(Int32, Utf8)') SIMILAR TO p.pat FROM t CROSS ---- true +# non-scalar string-like patterns are coerced by the analyzer +query B +SELECT t.s SIMILAR TO arrow_cast(p.pat, 'Utf8View') FROM t CROSS JOIN p; +---- +true + +query B +SELECT t.s SIMILAR TO arrow_cast(p.pat, 'LargeUtf8') FROM t CROSS JOIN p; +---- +true + +query B +SELECT t.s NOT SIMILAR TO arrow_cast(p.pat, 'Utf8View') FROM t CROSS JOIN p; +---- +false + +query B +SELECT t.s SIMILAR TO arrow_cast(p.pat, 'Dictionary(Int32, Utf8)') FROM t CROSS JOIN p; +---- +true + +# NULL patterns (literal or Null-typed non-scalar) evaluate to NULL +query B +SELECT t.s SIMILAR TO NULL FROM t; +---- +NULL + +statement ok +CREATE TABLE pn AS SELECT NULL AS pat; + +query B +SELECT t.s SIMILAR TO pn.pat FROM t CROSS JOIN pn; +---- +NULL + +statement ok +DROP TABLE pn; + statement ok DROP TABLE t; @@ -359,6 +399,6 @@ DROP TABLE p; query error There isn't a common type to coerce Int64 and Utf8 in SIMILAR TO expression SELECT 1 SIMILAR TO 'a'; -# a non-string pattern is rejected even earlier, during SQL planning -query error Invalid pattern in SIMILAR TO expression +# a non-string pattern is rejected by the analyzer +query error There isn't a common type to coerce Utf8 and Int64 in SIMILAR TO expression SELECT 'a' SIMILAR TO 1; From 1727b7d709630767032dea6d0eb1e43ca8e6ef04 Mon Sep 17 00:00:00 2001 From: Jeffrey Vo Date: Wed, 29 Jul 2026 21:34:54 +0900 Subject: [PATCH 064/109] Fill in missing utf8view support in function type coercion (#23916) ## Which issue does this PR close? - Closes #13363 ## Rationale for this change Fill in some missing support for utf8view (and date64) in function type coercion. Main affected functions are UDFs that use exact/uniform type signature; for example in `date_bin` it couldnt coerce utf8view to its input types (even though utf8 worked) ## What changes are included in this PR? Add support for casting to/from utf8view in some arms. ## Are these changes tested? Yes ## Are there any user-facing changes? No --- datafusion/expr/src/type_coercion/functions.rs | 8 +++----- .../sqllogictest/test_files/datetime/timestamps.slt | 6 ++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs index 37a1c10159fe8..ec3ab6f441827 100644 --- a/datafusion/expr/src/type_coercion/functions.rs +++ b/datafusion/expr/src/type_coercion/functions.rs @@ -1198,13 +1198,11 @@ fn coerced_from<'a>( ) => Some(type_into.clone()), ( Timestamp(TimeUnit::Nanosecond, None), - Null | Timestamp(_, None) | Date32 | Utf8 | LargeUtf8, + Null | Timestamp(_, None) | Date32 | Date64 | Utf8 | LargeUtf8 | Utf8View, ) => Some(type_into.clone()), - (Interval(_), Null | Utf8 | LargeUtf8) => Some(type_into.clone()), - // We can go into a Utf8View from a Utf8 or LargeUtf8 - (Utf8View, Utf8 | LargeUtf8 | Null) => Some(type_into.clone()), + (Interval(_), Null | Utf8 | LargeUtf8 | Utf8View) => Some(type_into.clone()), // Any type can be coerced into strings - (Utf8 | LargeUtf8, _) => Some(type_into.clone()), + (Utf8 | LargeUtf8 | Utf8View, _) => Some(type_into.clone()), // We can go into a BinaryView from a Binary or LargeBinary (BinaryView, Binary | LargeBinary | Null) => Some(type_into.clone()), (Null, _) if can_cast_types(type_from, type_into) => Some(type_into.clone()), diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 9ac00e72b47e6..002111d3f252a 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -1264,6 +1264,12 @@ SELECT DATE_BIN('5 month', '2022-01-01T00:00:00Z'); ---- 2021-09-01T00:00:00 +# test with utf8view +query P +SELECT DATE_BIN(arrow_cast('5 month', 'Utf8View'), '2022-01-01T00:00:00Z'); +---- +2021-09-01T00:00:00 + # month interval with default start time query P SELECT DATE_BIN('1 month', '2022-01-01 00:00:00Z'); From 043d97fb2a1ec2576f7456b53030cf009f9278c7 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Wed, 29 Jul 2026 09:22:54 -0400 Subject: [PATCH 065/109] perf: Optimize hashing, null-free fast path for `percentile_cont`, `median` (#23954) ## Which issue does this PR close? - Closes #23953 ## Rationale for this change This PR makes three improvements to non-distinct `PercentileContAccumulator` and `MedianAccumulator`, inspired by recent work on `percentile_cont(DISTINCT)` (#23946): 1. Switch from SipHash to foldhash for internal hash maps 2. Add a null-free fast path to `update_batch` and `retract_batch` 3. Raise an error if we attempt to retract an unknown value in `retract_batch`, rather than silently ignoring it. Benchmarks: percentile_cont no_nulls window=256 -61.1% (249.0 -> 96.6 us) percentile_cont with_nulls window=256 -59.4% (232.7 -> 94.5 us) percentile_cont no_nulls window=4096 -50.9% (756.3 -> 370.4 us) percentile_cont with_nulls window=4096 -48.2% (552.2 -> 286.2 us) percentile_cont no_nulls window=16384 -47.3% (2.311 -> 1.218 ms) percentile_cont with_nulls window=16384 -42.0% (1.465 -> 0.851 ms) median no_nulls window=256 -62.8% (247.0 -> 92.0 us) median with_nulls window=256 -59.8% (228.8 -> 92.0 us) median no_nulls window=4096 -40.0% (411.4 -> 246.6 us) median with_nulls window=4096 -39.4% (364.4 -> 221.0 us) median no_nulls window=16384 -31.5% (1.107 -> 0.759 ms) median with_nulls window=16384 -32.9% (0.947 -> 0.637 ms) ## What changes are included in this PR? See above. ## Are these changes tested? Yes: existing tests pass, new tests added for the null-free fast path and the improved error handling. ## Are there any user-facing changes? No. --- datafusion/functions-aggregate/src/median.rs | 88 +++++++++++++++++-- .../src/percentile_cont.rs | 80 +++++++++++++++-- 2 files changed, 155 insertions(+), 13 deletions(-) diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs index 7a399f73ec8e2..81a3c076dffbe 100644 --- a/datafusion/functions-aggregate/src/median.rs +++ b/datafusion/functions-aggregate/src/median.rs @@ -39,10 +39,11 @@ use arrow::datatypes::{ ArrowNativeType, ArrowPrimitiveType, Decimal32Type, Decimal64Type, FieldRef, }; +use datafusion_common::hash_utils::RandomState; use datafusion_common::types::{NativeType, logical_float64}; use datafusion_common::{ DataFusionError, Result, ScalarValue, assert_eq_or_internal_err, exec_datafusion_err, - internal_datafusion_err, + internal_datafusion_err, internal_err, }; use datafusion_expr::function::StateFieldsArgs; use datafusion_expr::{ @@ -288,7 +289,12 @@ impl Accumulator for MedianAccumulator { "failed to reserve {additional} values for median accumulator: {e}" ) })?; - self.all_values.extend(values.iter().flatten()); + if values.null_count() > 0 { + self.all_values.extend(values.iter().flatten()); + } else { + // Fast path: no nulls, so the values buffer can be appended wholesale. + self.all_values.extend_from_slice(values.values()); + } Ok(()) } @@ -310,11 +316,19 @@ impl Accumulator for MedianAccumulator { } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - let mut to_remove: HashMap, usize> = HashMap::new(); + let mut to_remove: HashMap, usize, RandomState> = + HashMap::default(); let arr = values[0].as_primitive::(); - for value in arr.iter().flatten() { - *to_remove.entry(Hashable(value)).or_default() += 1; + if arr.null_count() > 0 { + for value in arr.iter().flatten() { + *to_remove.entry(Hashable(value)).or_default() += 1; + } + } else { + // Fast path: no nulls, so skip the per-element validity check. + for value in arr.values().iter() { + *to_remove.entry(Hashable(*value)).or_default() += 1; + } } let mut i = 0; @@ -335,6 +349,15 @@ impl Accumulator for MedianAccumulator { i += 1; } } + + // Retracting values that are not tracked means the accumulator state + // has diverged from the window frame; continuing would silently + // produce wrong results, so surface it as an error. + if !to_remove.is_empty() { + return internal_err!( + "median retract_batch: retracted value(s) not present in the window" + ); + } Ok(()) } @@ -627,3 +650,58 @@ fn calculate_median(values: &mut [T::Native]) -> Option MedianAccumulator { + MedianAccumulator { + data_type: DataType::Float64, + all_values: vec![], + } + } + + #[test] + fn retract_batch_errors_on_untracked_value() { + let mut acc = median_accumulator(); + let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0])); + acc.update_batch(std::slice::from_ref(&values)).unwrap(); + + let retract: ArrayRef = Arc::new(Float64Array::from(vec![3.0])); + let err = acc + .retract_batch(std::slice::from_ref(&retract)) + .unwrap_err() + .to_string(); + assert!( + err.contains("not present in the window"), + "unexpected error: {err}" + ); + } + + #[test] + fn update_batch_with_and_without_nulls_agree() { + // The null-free fast path must accumulate the same values as the + // general path. + let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])); + let sparse: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + None, + Some(2.0), + None, + Some(3.0), + ])); + + let mut dense_acc = median_accumulator(); + dense_acc + .update_batch(std::slice::from_ref(&dense)) + .unwrap(); + let mut sparse_acc = median_accumulator(); + sparse_acc + .update_batch(std::slice::from_ref(&sparse)) + .unwrap(); + + assert_eq!(dense_acc.all_values, sparse_acc.all_values); + } +} diff --git a/datafusion/functions-aggregate/src/percentile_cont.rs b/datafusion/functions-aggregate/src/percentile_cont.rs index baf1978ea24af..3a98900bbb446 100644 --- a/datafusion/functions-aggregate/src/percentile_cont.rs +++ b/datafusion/functions-aggregate/src/percentile_cont.rs @@ -429,7 +429,12 @@ where "failed to reserve {additional} values for percentile_cont accumulator: {e}" ) })?; - self.all_values.extend(values.iter().flatten()); + if values.null_count() > 0 { + self.all_values.extend(values.iter().flatten()); + } else { + // Fast path: no nulls, so the values buffer can be appended wholesale. + self.all_values.extend_from_slice(values.values()); + } Ok(()) } @@ -449,11 +454,19 @@ where } fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { - let mut to_remove: HashMap, usize> = HashMap::new(); + let mut to_remove: HashMap, usize, RandomState> = + HashMap::default(); let arr = values[0].as_primitive::(); - for value in arr.iter().flatten() { - *to_remove.entry(Hashable(value)).or_default() += 1; + if arr.null_count() > 0 { + for value in arr.iter().flatten() { + *to_remove.entry(Hashable(value)).or_default() += 1; + } + } else { + // Fast path: no nulls, so skip the per-element validity check. + for value in arr.values().iter() { + *to_remove.entry(Hashable(*value)).or_default() += 1; + } } let mut i = 0; @@ -474,6 +487,15 @@ where i += 1; } } + + // Retracting values that are not tracked means the accumulator state + // has diverged from the window frame; continuing would silently + // produce wrong results, so surface it as an error. + if !to_remove.is_empty() { + return internal_err!( + "percentile_cont retract_batch: retracted value(s) not present in the window" + ); + } Ok(()) } @@ -876,18 +898,60 @@ where #[cfg(test)] mod tests { - use super::calculate_percentile; + use super::*; + use arrow::array::Float64Array; use half::f16; + #[test] + fn retract_batch_errors_on_untracked_value() { + let mut acc = PercentileContAccumulator::::new(0.5); + let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0])); + acc.update_batch(std::slice::from_ref(&values)).unwrap(); + + let retract: ArrayRef = Arc::new(Float64Array::from(vec![3.0])); + let err = acc + .retract_batch(std::slice::from_ref(&retract)) + .unwrap_err() + .to_string(); + assert!( + err.contains("not present in the window"), + "unexpected error: {err}" + ); + } + + #[test] + fn update_batch_with_and_without_nulls_agree() { + // The null-free fast path must accumulate the same values as the + // general path. + let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])); + let sparse: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + None, + Some(2.0), + None, + Some(3.0), + ])); + + let mut dense_acc = PercentileContAccumulator::::new(0.5); + dense_acc + .update_batch(std::slice::from_ref(&dense)) + .unwrap(); + let mut sparse_acc = PercentileContAccumulator::::new(0.5); + sparse_acc + .update_batch(std::slice::from_ref(&sparse)) + .unwrap(); + + assert_eq!(dense_acc.all_values, sparse_acc.all_values); + } + #[test] fn f16_interpolation_does_not_overflow_to_nan() { // Regression test for https://github.com/apache/datafusion/issues/18945 // Interpolating between 0 and the max finite f16 value previously overflowed // intermediate f16 computations and produced NaN. let mut values = vec![f16::from_f32(0.0), f16::from_f32(65504.0)]; - let result = - calculate_percentile::(&mut values, 0.5) - .expect("non-empty input"); + let result = calculate_percentile::(&mut values, 0.5) + .expect("non-empty input"); let result_f = result.to_f32(); assert!( !result_f.is_nan(), From 5941552e463a26584367dd3a0a123c6f1afe35be Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 29 Jul 2026 22:35:11 +0800 Subject: [PATCH 066/109] perf: null-free fast path for COUNT(DISTINCT) primitive accumulator (#23956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23955. ## Rationale for this change `PrimitiveDistinctCountAccumulator::update_batch` does a per-element validity check on every row even when the array has no nulls. Its own `merge_batch` already inserts wholesale from the values buffer, so the same fast path applies to `update_batch`. Inspired by the null-free fast paths added to `percentile_cont`/`median` in #23954. ## What changes are included in this PR? - When `null_count() == 0`, insert directly from the values buffer (`self.values.extend(arr.values().iter().copied())`) instead of the per-element `Option` check. - Unit test asserting the fast and general paths agree (a dense array and a null-containing array with the same non-null values yield the same distinct count). ## Benchmarks `count_distinct` benchmark, null-free 8192-row batches: ``` count_distinct i64 80% distinct -67.0% (54.9 -> 18.1 us) count_distinct i64 99% distinct -66.0% (54.9 -> 18.7 us) count_distinct u32 80% distinct -66.9% (53.9 -> 17.8 us) count_distinct u32 99% distinct -65.3% (53.3 -> 18.5 us) count_distinct i32 80% distinct -66.9% (54.1 -> 17.9 us) count_distinct i32 99% distinct -66.2% (53.2 -> 18.0 us) ``` The bitmap-backed cases (u8/i8/u16/i16) and the grouped-accumulator cases don't use this path and are unchanged, as expected. ## Are these changes tested? Yes — new unit test; existing count-distinct tests pass. ## Are there any user-facing changes? No. --- .../src/aggregate/count_distinct/native.rs | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/native.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/native.rs index c7b466d4f0e0c..00c1a47b9eafb 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/native.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/native.rs @@ -26,6 +26,7 @@ use std::hash::Hash; use std::mem::size_of_val; use std::sync::Arc; +use arrow::array::Array; use arrow::array::ArrayRef; use arrow::array::BooleanArray; use arrow::array::PrimitiveArray; @@ -86,11 +87,15 @@ where } let arr = as_primitive_array::(&values[0])?; - arr.iter().for_each(|value| { - if let Some(value) = value { + if arr.null_count() == 0 { + // Fast path: no nulls, so skip the per-element validity check and + // insert directly from the values buffer (mirrors `merge_batch`). + self.values.extend(arr.values().iter().copied()); + } else { + arr.iter().flatten().for_each(|value| { self.values.insert(value); - } - }); + }); + } Ok(()) } @@ -617,3 +622,42 @@ impl Accumulator for BooleanDistinctCountAccumulator { size_of_val(self) } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int64Array; + use arrow::datatypes::Int64Type; + + #[test] + fn update_batch_null_free_fast_path_agrees_with_general_path() { + // The null-free fast path must produce the same distinct set as the + // general (validity-checking) path. + let dense: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3, 2, 1])); + let sparse: ArrayRef = Arc::new(Int64Array::from(vec![ + Some(1), + None, + Some(2), + None, + Some(3), + Some(2), + Some(1), + ])); + + let mut dense_acc = + PrimitiveDistinctCountAccumulator::::new(&DataType::Int64); + dense_acc + .update_batch(std::slice::from_ref(&dense)) + .unwrap(); + + let mut sparse_acc = + PrimitiveDistinctCountAccumulator::::new(&DataType::Int64); + sparse_acc + .update_batch(std::slice::from_ref(&sparse)) + .unwrap(); + + // Both should count the 3 distinct non-null values {1, 2, 3}. + assert_eq!(dense_acc.evaluate().unwrap(), ScalarValue::Int64(Some(3))); + assert_eq!(sparse_acc.evaluate().unwrap(), ScalarValue::Int64(Some(3))); + } +} From 1955d5af75c229fb7131eb433fd2284549dab9ec Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Wed, 29 Jul 2026 12:07:28 -0400 Subject: [PATCH 067/109] chore: refactor `VarianceAccumulator`, add tests and benchmark (#23977) ## Which issue does this PR close? - N/A ## Rationale for this change This PR adds a benchmark for `VarianceAccumulator::update_batch`, and `retract_batch`, adds some unit tests, and refactors the `retract_batch` code to add a helper. ## What changes are included in this PR? See above. ## Are these changes tested? Yes, new tests added. ## Are there any user-facing changes? No. --- datafusion/functions-aggregate/Cargo.toml | 4 + .../functions-aggregate/benches/variance.rs | 83 ++++++++++++++ .../functions-aggregate/src/variance.rs | 104 +++++++++++++++--- 3 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 datafusion/functions-aggregate/benches/variance.rs diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml index 3ac882e9c6855..5abea16e2cc81 100644 --- a/datafusion/functions-aggregate/Cargo.toml +++ b/datafusion/functions-aggregate/Cargo.toml @@ -100,5 +100,9 @@ harness = false name = "sliding_max" harness = false +[[bench]] +name = "variance" +harness = false + [features] force_hash_collisions = ["datafusion-common/force_hash_collisions"] diff --git a/datafusion/functions-aggregate/benches/variance.rs b/datafusion/functions-aggregate/benches/variance.rs new file mode 100644 index 0000000000000..ef55bf32b8843 --- /dev/null +++ b/datafusion/functions-aggregate/benches/variance.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hint::black_box; +use std::sync::Arc; + +use arrow::array::{ArrayRef, Float64Array}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use datafusion_expr::Accumulator; +use datafusion_functions_aggregate::variance::VarianceAccumulator; +use datafusion_functions_aggregate_common::stats::StatsType; + +const BATCH_SIZE: usize = 8192; + +fn batch_array(null_stride: Option) -> ArrayRef { + let values = (0..BATCH_SIZE) + .map(|idx| { + if null_stride.is_some_and(|stride| idx % stride == 0) { + None + } else { + Some(idx as f64) + } + }) + .collect::>(); + Arc::new(Float64Array::from(values)) as ArrayRef +} + +fn update_bench(c: &mut Criterion, name: &str, batch: &ArrayRef) { + c.bench_function(name, |b| { + b.iter(|| { + let mut acc = VarianceAccumulator::try_new(StatsType::Sample).unwrap(); + acc.update_batch(std::slice::from_ref(batch)).unwrap(); + black_box(acc.evaluate().unwrap()) + }) + }); +} + +fn retract_bench(c: &mut Criterion, name: &str, batch: &ArrayRef) { + c.bench_function(name, |b| { + b.iter_batched( + || { + let mut acc = VarianceAccumulator::try_new(StatsType::Sample).unwrap(); + // Accumulate two batches so that retracting one leaves the + // accumulator with rows remaining, as in a sliding window. + acc.update_batch(std::slice::from_ref(batch)).unwrap(); + acc.update_batch(std::slice::from_ref(batch)).unwrap(); + acc + }, + |mut acc| { + acc.retract_batch(std::slice::from_ref(batch)).unwrap(); + black_box(acc.evaluate().unwrap()) + }, + BatchSize::SmallInput, + ) + }); +} + +fn variance_benchmark(c: &mut Criterion) { + let no_nulls = batch_array(None); + let with_nulls = batch_array(Some(10)); + + update_bench(c, "variance update_batch f64 no_nulls", &no_nulls); + update_bench(c, "variance update_batch f64 with_nulls", &with_nulls); + retract_bench(c, "variance retract_batch f64 no_nulls", &no_nulls); + retract_bench(c, "variance retract_batch f64 with_nulls", &with_nulls); +} + +criterion_group!(benches, variance_benchmark); +criterion_main!(benches); diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs index df652731ff4f4..b8e52f849a7cc 100644 --- a/datafusion/functions-aggregate/src/variance.rs +++ b/datafusion/functions-aggregate/src/variance.rs @@ -326,6 +326,23 @@ fn update(count: u64, mean: f64, m2: f64, value: f64) -> (u64, f64, f64) { (new_count, new_mean, new_m2) } +/// Inverse of [`update`]: removes a previously accumulated value. Retracting +/// from a state with one or zero values resets the state to empty. +#[inline] +fn retract(count: u64, mean: f64, m2: f64, value: f64) -> (u64, f64, f64) { + if count <= 1 { + return (0, 0.0, 0.0); + } + + let new_count = count - 1; + let delta1 = mean - value; + let new_mean = delta1 / new_count as f64 + mean; + let delta2 = new_mean - value; + let new_m2 = m2 - delta1 * delta2; + + (new_count, new_mean, new_m2) +} + impl Accumulator for VarianceAccumulator { fn state(&mut self) -> Result> { Ok(vec![ @@ -348,22 +365,8 @@ impl Accumulator for VarianceAccumulator { fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { let arr = as_float64_array(&values[0])?; for value in arr.iter().flatten() { - if self.count <= 1 { - self.count = 0; - self.mean = 0.0; - self.m2 = 0.0; - continue; - } - - let new_count = self.count - 1; - let delta1 = self.mean - value; - let new_mean = delta1 / new_count as f64 + self.mean; - let delta2 = new_mean - value; - let new_m2 = self.m2 - delta1 * delta2; - - self.count -= 1; - self.mean = new_mean; - self.m2 = new_m2; + (self.count, self.mean, self.m2) = + retract(self.count, self.mean, self.m2, value) } Ok(()) @@ -691,6 +694,75 @@ mod tests { use super::*; + #[test] + fn update_batch_ignores_nulls() -> Result<()> { + // An array with nulls must accumulate the same values as a dense + // array of its non-null values. + let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])); + let sparse: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + None, + Some(2.0), + Some(3.0), + None, + Some(4.0), + ])); + + let mut dense_acc = VarianceAccumulator::try_new(StatsType::Sample)?; + dense_acc.update_batch(std::slice::from_ref(&dense))?; + let mut sparse_acc = VarianceAccumulator::try_new(StatsType::Sample)?; + sparse_acc.update_batch(std::slice::from_ref(&sparse))?; + + // Sample variance of {1, 2, 3, 4} is 5/3 (all steps are exact in f64). + assert_eq!(dense_acc.evaluate()?, ScalarValue::Float64(Some(5.0 / 3.0))); + assert_eq!(dense_acc.evaluate()?, sparse_acc.evaluate()?); + Ok(()) + } + + #[test] + fn retract_batch_ignores_nulls() -> Result<()> { + let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0])); + let dense_retract: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0])); + let sparse_retract: ArrayRef = + Arc::new(Float64Array::from(vec![Some(1.0), None, Some(2.0)])); + + let mut dense_acc = VarianceAccumulator::try_new(StatsType::Sample)?; + dense_acc.update_batch(std::slice::from_ref(&values))?; + dense_acc.retract_batch(std::slice::from_ref(&dense_retract))?; + let mut sparse_acc = VarianceAccumulator::try_new(StatsType::Sample)?; + sparse_acc.update_batch(std::slice::from_ref(&values))?; + sparse_acc.retract_batch(std::slice::from_ref(&sparse_retract))?; + + // Sample variance of the remaining {3, 4} is 0.5 (all steps are exact + // in f64). + assert_eq!(dense_acc.evaluate()?, ScalarValue::Float64(Some(0.5))); + assert_eq!(dense_acc.evaluate()?, sparse_acc.evaluate()?); + Ok(()) + } + + #[test] + fn retract_batch_resets_when_underflowing() -> Result<()> { + // Retracting more values than were accumulated resets to the empty + // state, with or without nulls in the retracted batch. + let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0])); + let dense_retract: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])); + let sparse_retract: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + None, + Some(2.0), + Some(3.0), + ])); + + for retract in [&dense_retract, &sparse_retract] { + let mut acc = VarianceAccumulator::try_new(StatsType::Sample)?; + acc.update_batch(std::slice::from_ref(&values))?; + acc.retract_batch(std::slice::from_ref(retract))?; + assert_eq!(acc.get_count(), 0); + assert_eq!(acc.evaluate()?, ScalarValue::Float64(None)); + } + Ok(()) + } + #[test] fn test_groups_accumulator_merge_empty_states() -> Result<()> { let state_1 = vec![ From 2fb472a2173baeab28f6b498e0aa63ebd893ee77 Mon Sep 17 00:00:00 2001 From: yoongbok lee <7127607+U0001F3A2@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:37:14 +0900 Subject: [PATCH 068/109] test: add regression coverage and docs for NULL format handling (#23669) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - None — follow-up to #23640 and #23641. ## Rationale for this change #23641 already shipped the implementation for NULL string and format arguments. Per the review request on this PR, this follow-up keeps only the documentation and regression coverage needed to prevent a recurrence. ## What changes are included in this PR? - Documents that NULL format entries are skipped and an all-NULL format list returns NULL for `to_date`, all `to_timestamp*` variants, and `to_unixtime`. - Adds an Arrow validity-buffer regression test where a NULL format slot retains parseable backing bytes and must not win over a later valid format. - Adds SQL logic tests for NULL-format error propagation and validation-before-NULL-input ordering. ## Are these changes tested? - `cargo fmt --all` - `cargo test --locked -p datafusion-functions` - `cargo test --locked --profile ci --test sqllogictests -- dates.slt` - `cargo clippy --locked -p datafusion-functions --lib --tests -- -D warnings` - `./ci/scripts/doc_prettier_check.sh --write --allow-dirty` - Mutation check: removing the shared NULL guard makes the retained-bytes regression test fail; restored code passes. ## Are there any user-facing changes? No behavior change. The generated function documentation now describes behavior already shipped in #23641. Co-authored-by: Claude Co-authored-by: Andrew Lamb --- datafusion/functions/src/datetime/to_date.rs | 42 ++++++++++++++++++- .../functions/src/datetime/to_timestamp.rs | 15 ++++--- .../functions/src/datetime/to_unixtime.rs | 2 +- .../test_files/datetime/dates.slt | 8 ++++ .../source/user-guide/sql/scalar_functions.md | 19 +++++---- 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/datafusion/functions/src/datetime/to_date.rs b/datafusion/functions/src/datetime/to_date.rs index cd75ac6bed3ac..ed5b8b16320b7 100644 --- a/datafusion/functions/src/datetime/to_date.rs +++ b/datafusion/functions/src/datetime/to_date.rs @@ -61,7 +61,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo name = "format_n", description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression - an error will be returned." + an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -519,4 +519,44 @@ mod tests { panic!("Conversion of {date_str} succeeded, but should have failed. "); } } + + /// A NULL format must be skipped even when its slot still holds parseable + /// bytes, otherwise it can silently win over a later valid format. + #[test] + fn test_to_date_null_format_slot_retaining_bytes() { + use arrow::buffer::NullBuffer; + + // The first format physically holds "%d/%m/%Y", but is marked NULL. + let (offsets, values, _) = + GenericStringArray::::from(vec!["%d/%m/%Y"]).into_parts(); + let formats = + GenericStringArray::new(offsets, values, Some(NullBuffer::new_null(1))); + assert!(formats.is_null(0)); + assert_eq!(formats.value(0), "%d/%m/%Y"); + + // Without the validity check, the first format parses this as 2023-02-01 + // and incorrectly wins over the valid second format. + let values = GenericStringArray::::from(vec!["01/02/2023"]); + let fallback_formats = GenericStringArray::::from(vec!["%m/%d/%Y"]); + let res = invoke_to_date_with_args( + vec![ + ColumnarValue::Array(Arc::new(values)), + ColumnarValue::Array(Arc::new(formats)), + ColumnarValue::Array(Arc::new(fallback_formats)), + ], + 1, + ) + .unwrap(); + + let ColumnarValue::Array(res) = res else { + panic!("expected an array result"); + }; + let res = res.as_any().downcast_ref::().unwrap(); + + assert!(!res.is_null(0)); + assert_eq!( + res.value(0), + Date32Type::parse_formatted("01/02/2023", "%m/%d/%Y").unwrap() + ); + } } diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index f4507ab250559..1b45910f7261c 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -81,7 +81,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -131,7 +132,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -181,7 +183,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -231,7 +234,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -280,7 +284,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) diff --git a/datafusion/functions/src/datetime/to_unixtime.rs b/datafusion/functions/src/datetime/to_unixtime.rs index 9fcfd254ca74d..5b9734c05d7be 100644 --- a/datafusion/functions/src/datetime/to_unixtime.rs +++ b/datafusion/functions/src/datetime/to_unixtime.rs @@ -56,7 +56,7 @@ Integers, unsigned integers, and floats are interpreted as seconds since the uni ), argument( name = "format_n", - description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt index 68d87eceed99e..abf92e15659e5 100644 --- a/datafusion/sqllogictest/test_files/datetime/dates.slt +++ b/datafusion/sqllogictest/test_files/datetime/dates.slt @@ -319,6 +319,14 @@ ORDER BY id 1 2020-09-08 2 NULL +# Skipping NULL formats does not mask a later parse error. +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08' using format '%q': trailing input +SELECT to_date('2020-09-08', NULL::VARCHAR, '%q') + +# Invalid format types are rejected before NULL input propagation. +query error DataFusion error: Execution error: to_date function unsupported data type at index 1: Int64 +SELECT to_date(NULL::VARCHAR, 12345) + statement ok create table ts_utf8_data(ts varchar(100), format varchar(100)) as values ('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'), diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index a285b9e5f5cff..a865a3d182404 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2851,7 +2851,7 @@ to_date('2017-05-31', '%Y-%m-%d') - **expression**: String expression to operate on. Can be a constant, column, or function, and any combination of operators. - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression - an error will be returned. + an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. #### Example @@ -3006,7 +3006,8 @@ to_timestamp(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3050,7 +3051,8 @@ to_timestamp_micros(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3094,7 +3096,8 @@ to_timestamp_millis(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3137,7 +3140,8 @@ to_timestamp_nanos(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3181,7 +3185,8 @@ to_timestamp_seconds(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3218,7 +3223,7 @@ to_unixtime(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. #### Example From 2f25454ad0e58abd94f5f36cfc4e19854695b92a Mon Sep 17 00:00:00 2001 From: discord9 Date: Thu, 30 Jul 2026 02:53:48 +0800 Subject: [PATCH 069/109] fix: preserve aggregate filter pushdown order (#22926) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #22925. ## Rationale for this change `FilterPushdown` maps each child pushdown result back to its original parent filter by position. `AggregateExec::gather_filters_for_pushdown` previously split parent filters into safe and unsafe buckets and then concatenated the results, which changed their order. For example, in `cnt@2 = 1 AND b@1 = bar`, the grouping-column predicate on `b` can cross the aggregate, while the predicate on aggregate output `cnt` must remain above it. Reordering the returned results could make the optimizer associate the pushed-down `b` result with the `cnt` predicate and incorrectly remove the latter. Aggregate pushdown also needs to account for empty-input semantics. A global aggregate, or a grouping-sets aggregate containing `()`, can emit a row even when its input is empty. Moving any parent predicate below such an aggregate — including a column-free predicate such as `false` — can therefore change the result. ## What changes are included in this PR? - Preserve parent-filter order by constructing each child description once with `ChildFilterDescription::from_child_with_allowed_indices`. - Allow only grouping-output columns that are present in every grouping set; aggregate-result columns remain above the aggregate. - Mark all parent filters unsupported for global aggregates and grouping sets containing an empty grouping set, while retaining aggregate-generated dynamic filters. - Validate that every child returns one parent-filter result per input filter before positional remapping. - Add physical optimizer and SQL regression coverage for mixed filter order, global-aggregate name collisions, constant predicates, and grouping sets. ## Are these changes tested? Yes: - `cargo test -p datafusion --test core_integration physical_optimizer::filter_pushdown` - `cargo fmt --all -- --check` - `cargo check -p datafusion-physical-plan -p datafusion-physical-optimizer` - `git diff --check upstream/main` - Full GitHub CI, including Rust tests, clippy, and sqllogictests ## Are there any user-facing changes? There are no public API changes. Filter pushdown now preserves mixed aggregate predicates correctly and avoids moving predicates across aggregates when doing so could change empty-input results. --- .../physical_optimizer/filter_pushdown.rs | 199 +++++++++++++++--- .../physical-optimizer/src/filter_pushdown.rs | 8 + .../physical-plan/src/aggregates/mod.rs | 90 +++----- .../physical-plan/src/execution_plan.rs | 6 + .../physical-plan/src/filter_pushdown.rs | 3 + .../push_down_filter_regression.slt | 38 ++++ 6 files changed, 253 insertions(+), 91 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs index 909b80cadaae3..7593fe351548e 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs @@ -41,8 +41,13 @@ use datafusion_datasource::{ use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_expr::ScalarUDF; use datafusion_functions::math::random::RandomFunc; -use datafusion_functions_aggregate::{count::count_udaf, min_max::min_udaf}; -use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr, expressions::col}; +use datafusion_functions_aggregate::{ + count::count_udaf, + min_max::{max_udaf, min_udaf}, +}; +use datafusion_physical_expr::{ + LexOrdering, PhysicalSortExpr, expressions::col, utils::conjunction, +}; use datafusion_physical_expr::{ Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder, }; @@ -738,6 +743,65 @@ fn test_pushdown_through_aggregates_on_grouping_columns() { ); } +#[test] +fn test_pushdown_through_aggregates_preserves_parent_filter_order() { + // AggregateExec may push filters on grouping columns to its input, but must + // keep filters on aggregate outputs above itself. The parent-filter result + // order must match the incoming filter order, otherwise an unsupported + // aggregate-output filter can be reported as pushed down and removed. + let scan = TestScanBuilder::new(schema()).with_support(true).build(); + + let aggregate_expr = vec![ + AggregateExprBuilder::new(count_udaf(), vec![col("a", &schema()).unwrap()]) + .schema(schema()) + .alias("cnt") + .build() + .map(Arc::new) + .unwrap(), + ]; + let group_by = PhysicalGroupBy::new_single(vec![ + (col("a", &schema()).unwrap(), "a".to_string()), + (col("b", &schema()).unwrap(), "b".to_string()), + ]); + let aggregate = Arc::new( + AggregateExec::try_new( + AggregateMode::Final, + group_by, + aggregate_expr, + vec![None], + scan, + schema(), + ) + .unwrap(), + ); + + let aggregate_schema = aggregate.schema(); + let aggregate_output_filter = col_lit_predicate( + "cnt", + ScalarValue::Int64(Some(1)), + aggregate_schema.as_ref(), + ); + let grouping_key_filter = col_lit_predicate("b", "bar", aggregate_schema.as_ref()); + let predicate = conjunction(vec![aggregate_output_filter, grouping_key_filter]); + let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap()); + + insta::assert_snapshot!( + OptimizationTest::new(plan, FilterPushdown::new(), true), + @r" + OptimizationTest: + input: + - FilterExec: cnt@2 = 1 AND b@1 = bar + - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt] + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + output: + Ok: + - FilterExec: cnt@2 = 1 + - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([1]) + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=b@1 = bar + " + ); +} + /// Test various combinations of handling of child pushdown results /// in an ExecutionPlan in combination with support/not support in a DataSource. #[test] @@ -1753,6 +1817,16 @@ fn col_lit_predicate( )) } +fn assert_parent_filter_remains_above_aggregate(plan: Arc) { + let mut config = ConfigOptions::default(); + config.execution.parquet.pushdown_filters = true; + let optimized = FilterPushdown::new().optimize(plan, &config).unwrap(); + assert!( + optimized.downcast_ref::().is_some(), + "parent filter must remain above aggregate" + ); +} + // ==== Aggregate Dynamic Filter tests ==== // // The end-to-end min/max dynamic filter cases (simple/min/max/mixed/all-nulls) @@ -1997,13 +2071,65 @@ fn test_pushdown_grouping_sets_filter_on_common_column() { ); } +#[tokio::test] +async fn test_no_pushdown_through_global_aggregate_with_name_collision() { + let input_schema = + Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)])); + let scan = TestScanBuilder::new(Arc::clone(&input_schema)) + .with_support(true) + .with_batches(vec![record_batch!(("a", Int64, [1, 20])).unwrap()]) + .build(); + let aggregate_expr = vec![ + AggregateExprBuilder::new(max_udaf(), vec![col("a", &input_schema).unwrap()]) + .schema(Arc::clone(&input_schema)) + .alias("a") + .build() + .map(Arc::new) + .unwrap(), + ]; + let aggregate = Arc::new( + AggregateExec::try_new( + AggregateMode::Single, + PhysicalGroupBy::new_single(vec![]), + aggregate_expr, + vec![None], + scan, + input_schema, + ) + .unwrap(), + ); + + // This is a physical filter above the aggregate, not a SQL WHERE clause. + // Pushing it through would evaluate input `a` instead of MAX(a). + let predicate = Arc::new(BinaryExpr::new( + col("a", aggregate.schema().as_ref()).unwrap(), + Operator::Lt, + Arc::new(Literal::new(ScalarValue::Int64(Some(10)))), + )); + let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap()); + + let mut config = ConfigOptions::default(); + config.execution.parquet.pushdown_filters = true; + let optimized = FilterPushdown::new().optimize(plan, &config).unwrap(); + assert!(optimized.downcast_ref::().is_some()); + + let session_ctx = SessionContext::new(); + session_ctx.register_object_store( + ObjectStoreUrl::parse("test://").unwrap().as_ref(), + Arc::new(InMemory::new()), + ); + let batches = collect(optimized, session_ctx.state().task_ctx()) + .await + .unwrap(); + assert!( + batches.is_empty(), + "MAX(a) = 20 must be filtered out instead of applying a < 10 to input rows" + ); +} + #[test] -fn test_pushdown_with_empty_group_by() { - // Test that filters can be pushed down when GROUP BY is empty (no grouping columns) - // SELECT count(*) as cnt FROM table WHERE a = 'foo' - // There are no grouping columns, so the filter should still push down +fn test_no_pushdown_constant_false_through_global_aggregate() { let scan = TestScanBuilder::new(schema()).with_support(true).build(); - let aggregate_expr = vec![ AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()]) .schema(schema()) @@ -2012,41 +2138,58 @@ fn test_pushdown_with_empty_group_by() { .map(Arc::new) .unwrap(), ]; + let aggregate = Arc::new( + AggregateExec::try_new( + AggregateMode::Final, + PhysicalGroupBy::new_single(vec![]), + aggregate_expr, + vec![None], + scan, + schema(), + ) + .unwrap(), + ); + let predicate = Arc::new(Literal::new(ScalarValue::Boolean(Some(false)))); + let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap()); - // Empty GROUP BY - no grouping columns - let group_by = PhysicalGroupBy::new_single(vec![]); + assert_parent_filter_remains_above_aggregate(plan); +} +#[test] +fn test_no_pushdown_constant_false_through_empty_grouping_set() { + let scan = TestScanBuilder::new(schema()).with_support(true).build(); + let group_by = PhysicalGroupBy::new( + vec![(col("a", &schema()).unwrap(), "a".to_string())], + vec![( + Arc::new(Literal::new(ScalarValue::Utf8(None))), + "a".to_string(), + )], + vec![vec![true]], + true, + ); + let aggregate_expr = vec![ + AggregateExprBuilder::new(count_udaf(), vec![col("c", &schema()).unwrap()]) + .schema(schema()) + .alias("cnt") + .build() + .map(Arc::new) + .unwrap(), + ]; let aggregate = Arc::new( AggregateExec::try_new( AggregateMode::Final, group_by, - aggregate_expr.clone(), + aggregate_expr, vec![None], scan, schema(), ) .unwrap(), ); - - // Filter on 'a' - let predicate = col_lit_predicate("a", "foo", &schema()); + let predicate = Arc::new(Literal::new(ScalarValue::Boolean(Some(false)))); let plan = Arc::new(FilterExec::try_new(predicate, aggregate).unwrap()); - // The filter should be pushed down even with empty GROUP BY - insta::assert_snapshot!( - OptimizationTest::new(plan, FilterPushdown::new(), true), - @r" - OptimizationTest: - input: - - FilterExec: a@0 = foo - - AggregateExec: mode=Final, gby=[], aggr=[cnt] - - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true - output: - Ok: - - AggregateExec: mode=Final, gby=[], aggr=[cnt] - - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo - " - ); + assert_parent_filter_remains_above_aggregate(plan); } #[test] diff --git a/datafusion/physical-optimizer/src/filter_pushdown.rs b/datafusion/physical-optimizer/src/filter_pushdown.rs index 28f8155002a50..06aa632a9d3f3 100644 --- a/datafusion/physical-optimizer/src/filter_pushdown.rs +++ b/datafusion/physical-optimizer/src/filter_pushdown.rs @@ -486,6 +486,14 @@ fn push_down_filters( // currently. `self_filters` are the predicates which are provided by the current node, // and tried to be pushed down over the child similarly. + assert_eq_or_internal_err!( + parent_filters.len(), + parent_filtered.len(), + "Filter pushdown expected {} to return one parent filter result per input filter for child {}", + node.name(), + child_idx + ); + // Filter out self_filters that contain volatile expressions and track indices let self_filtered = FilteredVec::new(&self_filters, allow_pushdown_for_expr); diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 787cd4f03ff6c..370c75961608c 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -159,7 +159,7 @@ use crate::aggregates::{ use crate::execution_plan::{CardinalityEffect, EmissionType}; use crate::filter_pushdown::{ ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase, - FilterPushdownPropagation, PushedDownPredicate, + FilterPushdownPropagation, }; use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet}; use crate::statistics::{ChildStats, StatisticsArgs}; @@ -168,7 +168,6 @@ use crate::{ InputOrderMode, SendableRecordBatchStream, Statistics, check_if_same_properties, }; use datafusion_common::config::ConfigOptions; -use datafusion_physical_expr::utils::collect_columns; use parking_lot::Mutex; use std::collections::{HashMap, HashSet}; @@ -2041,70 +2040,35 @@ impl ExecutionPlan for AggregateExec { // This optimization is NOT safe for filters on aggregated columns (like filtering on // the result of SUM or COUNT), as those require computing all groups first. - // Build grouping columns using output indices because parent filters reference the - // AggregateExec's output schema where grouping columns in the output schema. The - // grouping expressions reference input columns which may not match the output schema. - // - // It is safe to assume that the output_schema contains group by columns in the same order - // as the group by expression. See [`create_schema`] and [`AggregateExec`]. - let output_schema = self.schema(); - let grouping_columns: HashSet<_> = (0..self.group_by.expr().len()) - .map(|i| Column::new(output_schema.field(i).name(), i)) - .collect(); - - // Analyze each filter separately to determine if it can be pushed down - let mut safe_filters = Vec::new(); - let mut unsafe_filters = Vec::new(); - - for filter in parent_filters { - let filter_columns: HashSet<_> = - collect_columns(&filter).into_iter().collect(); - - // Check if this filter references non-grouping columns - let references_non_grouping = !grouping_columns.is_empty() - && !filter_columns.is_subset(&grouping_columns); - - if references_non_grouping { - unsafe_filters.push(filter); - continue; - } - - // For GROUPING SETS, verify this filter's columns appear in all grouping sets - if self.group_by.groups().len() > 1 { - let filter_column_indices: Vec = filter_columns - .iter() - .filter_map(|filter_col| { - grouping_columns.get(filter_col).map(|col| col.index()) - }) - .collect(); - - // Check if any of this filter's columns are missing from any grouping set - let has_missing_column = self.group_by.groups().iter().any(|null_mask| { - filter_column_indices - .iter() - .any(|&idx| null_mask.get(idx) == Some(&true)) - }); - - if has_missing_column { - unsafe_filters.push(filter); - continue; - } - } - - // This filter is safe to push down - safe_filters.push(filter); + // Grouping columns are output before aggregate columns, in the same order + // as the grouping expressions. A grouping-set null mask marks grouping + // columns that are not available in that set. + let mut allowed_indices: HashSet = + (0..self.group_by.expr().len()).collect(); + for null_mask in self.group_by.groups() { + allowed_indices.retain(|idx| null_mask.get(*idx) != Some(&true)); } - // Build child filter description with both safe and unsafe filters let child = self.children()[0]; - let mut child_desc = ChildFilterDescription::from_child(&safe_filters, child)?; - - // Add unsafe filters as unsupported - child_desc.parent_filters.extend( - unsafe_filters - .into_iter() - .map(PushedDownPredicate::unsupported), - ); + // Global aggregates and grouping sets containing an empty grouping set + // emit a row even when their input is empty. Parent filters therefore + // cannot be pushed below them, including filters without column + // references. + let may_emit_on_empty_input = self.group_by.is_true_no_grouping() + || self + .group_by + .groups() + .iter() + .any(|null_mask| null_mask.iter().all(|is_null| *is_null)); + let mut child_desc = if may_emit_on_empty_input { + ChildFilterDescription::all_unsupported(&parent_filters) + } else { + ChildFilterDescription::from_child_with_allowed_indices( + &parent_filters, + allowed_indices, + child, + )? + }; // Include self dynamic filter when it's possible if phase == FilterPushdownPhase::Post diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index 3b9d5d258a838..11a8d69a37669 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -664,6 +664,12 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync { /// There are two different phases in filter pushdown, which some operators may handle the same and some differently. /// Depending on the phase the operator may or may not be allowed to modify the plan. /// See [`FilterPushdownPhase`] for more details. + /// + /// Implementations must preserve the order of `parent_filters` in the + /// returned child [`FilterDescription`]: each child parent-filter result is + /// matched back to the corresponding input parent filter by position. + /// Unsupported filters should therefore be marked unsupported in place, + /// rather than removed or appended after supported filters. fn gather_filters_for_pushdown( &self, _phase: FilterPushdownPhase, diff --git a/datafusion/physical-plan/src/filter_pushdown.rs b/datafusion/physical-plan/src/filter_pushdown.rs index 810f9ffcbcdb1..382967c7ee1ef 100644 --- a/datafusion/physical-plan/src/filter_pushdown.rs +++ b/datafusion/physical-plan/src/filter_pushdown.rs @@ -302,6 +302,9 @@ pub struct ChildFilterDescription { /// Description of which parent filters can be pushed down into this node. /// Since we need to transmit filter pushdown results back to this node's parent /// we need to track each parent filter for each child, even those that are unsupported / won't be pushed down. + /// The entries must stay in the same order as the input parent filters: the + /// filter pushdown optimizer maps child results back to parent filters by + /// position. pub(crate) parent_filters: Vec, /// Description of which filters this node is pushing down to its children. /// Since this is not transmitted back to the parents we can have variable sized inner arrays diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt index 7ab5e7c79d2ba..57509fd0395b9 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt @@ -515,6 +515,44 @@ physical_plan 05)--------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(agg_filter_pushdown.b)] 06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet +# Mixed filters on an aggregate output and a grouping column must preserve their +# parent filter result order. The grouping-column filter can push below the +# aggregate, but the aggregate-output filter must remain above it. +# Disable logical optimizer passes for this regression so the logical filter +# pushdown rule does not split the mixed predicate before the physical +# `AggregateExec::gather_filters_for_pushdown` path sees it. +statement ok +set datafusion.optimizer.max_passes = 0; + +query TT +EXPLAIN SELECT a, b, cnt FROM ( + SELECT a, b, count(b) AS cnt + FROM agg_filter_pushdown + GROUP BY a, b +) q WHERE cnt = 2 AND b = 'foo'; +---- +physical_plan +01)FilterExec: cnt@2 = 2 +02)--ProjectionExec: expr=[a@0 as a, b@1 as b, count(agg_filter_pushdown.b)@2 as cnt] +03)----AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b], aggr=[count(agg_filter_pushdown.b)] +04)------RepartitionExec: partitioning=Hash([a@0, b@1], 4), input_partitions=4 +05)--------AggregateExec: mode=Partial, gby=[a@0 as a, b@1 as b], aggr=[count(agg_filter_pushdown.b)] +06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_regression/agg_filter_pushdown.parquet]]}, projection=[a, b], file_type=parquet, predicate=b@1 = CAST(foo AS Utf8View), pruning_predicate=b_null_count@2 != row_count@3 AND b_min@0 <= foo AND foo <= b_max@1, required_guarantees=[] + +# If the aggregate-output filter is incorrectly removed, this query returns 1. +query I +SELECT count(*) FROM ( + SELECT a, b, count(b) AS cnt + FROM agg_filter_pushdown + GROUP BY a, b +) q WHERE cnt = 2 AND b = 'foo'; +---- +0 + +statement ok +reset datafusion.optimizer.max_passes; + statement ok drop table agg_filter_pushdown; From 2e3626e72eb18f724d1501a5a98bbca24ea1257c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C5=9Een?= Date: Thu, 30 Jul 2026 09:08:35 +0300 Subject: [PATCH 070/109] minor(test): cover partially ordered aggregate spilling (#23947) ## Which issue does this PR close? - Not closes but part of #13431 ## Rationale for this change `GroupedHashAggregateStream` supports spilling for partially sorted group input, but existing aggregate spill tests only cover unordered (`GroupOrdering::None`) input. The `GroupOrdering::Partial` + `OutOfMemoryMode::Spill` path, including spilling sorted intermediate state and merging it back, does not have direct coverage. ## What changes are included in this PR? added test coverage for the case above ## Are these changes tested? test only change and it can be tested via: ``` cargo test --test sqllogictests -- ordered_aggregate_spill ``` ## Are there any user-facing changes? no test only change --- .../test_files/ordered_aggregate_spill.slt | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt index a4d492ab82e12..2c53c94144eb3 100644 --- a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt +++ b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt @@ -176,6 +176,58 @@ Plan with Metrics 03)----AggregateExec: mode=Partial,aggr=[sum(t1.v1 * Int64(2)), min(t1.v1 % Int64(2))], ordering_mode=PartiallySorted([0]), metrics=[spill_count=0,] +# ================================================================================== +# Single mode: with one partition the whole aggregation runs in a `Single` mode +# AggregateExec. min() keeps one intermediate state and avg() keeps two (sum + +# count), so both single- and multi-state accumulators are spilled and merged. +# ================================================================================== + +statement ok +SET datafusion.execution.target_partitions = 1 + +# Reference round: enough memory to aggregate without spilling. +statement ok +SET datafusion.runtime.memory_limit = '10M' + +query TT +EXPLAIN ANALYZE +SELECT round(v1, -4), v1 % 5000, min(v1 * 2), avg(v1) +FROM generate_series(20000) AS t1(v1) +GROUP BY round(v1, -4), v1 % 5000 +---- +Plan with Metrics +01)AggregateExec: mode=Single,aggr=[min(t1.v1 * Int64(2)), avg(t1.v1)], ordering_mode=PartiallySorted([0]), metrics=[spill_count=0,] + + +query IIIR rowsort +SELECT round(v1, -4), v1 % 5000, min(v1 * 2), avg(v1) +FROM generate_series(20000) AS t1(v1) +GROUP BY round(v1, -4), v1 % 5000 +---- +60000 values hashing to 872df6cefd51f81820fc5c6e5d7480df + +# Spilling round: the same query under a 600 KB limit must spill. +statement ok +SET datafusion.runtime.memory_limit = '600K' + +query TT +EXPLAIN ANALYZE +SELECT round(v1, -4), v1 % 5000, min(v1 * 2), avg(v1) +FROM generate_series(20000) AS t1(v1) +GROUP BY round(v1, -4), v1 % 5000 +---- +Plan with Metrics +01)AggregateExec: mode=Single,aggr=[min(t1.v1 * Int64(2)), avg(t1.v1)], ordering_mode=PartiallySorted([0]), metrics=[spilled_bytes= KB,] + + +# Same result hash as the no-spill round above +query IIIR rowsort +SELECT round(v1, -4), v1 % 5000, min(v1 * 2), avg(v1) +FROM generate_series(20000) AS t1(v1) +GROUP BY round(v1, -4), v1 % 5000 +---- +60000 values hashing to 872df6cefd51f81820fc5c6e5d7480df + statement ok RESET datafusion.runtime.memory_limit From 95226ac7d002c0bc248a21b09e313f549a33b9d7 Mon Sep 17 00:00:00 2001 From: Saad Tajwar <59696464+saadtajwar@users.noreply.github.com> Date: Thu, 30 Jul 2026 04:06:54 -0400 Subject: [PATCH 071/109] feat: centralizing higher-order list lambda evaluation helpers (#23911) ## Which issue does this PR close? - Closes #23667 ## Rationale for this change `array_filter`, `array_any_match`, and `array_first` each implement their own version of the same higher-order list execution pattern: - normalize a list-like argument to List / LargeList, - extract the flattened child values with slice-aware semantics, - evaluate a lambda once over the flattened values, - spread captured outer columns to flattened row cardinality with list_values_row_number, - map the flattened lambda result back to one output value per input row using adjusted offsets and null-row handling. This PR introduces shared helpers that allow for one area where the shared logic lives ## What changes are included in this PR? Added a helper in `lambda_utils` that abstracts shared fields (original list/flat values/evaluated result) and functionality (creating the `BooleanArray`, adjusting offsets for `List`/`LargeList`, extracting `list`/`lambda` pairs, etc), and applying to `array_filter`/`array_first`/`array_any_match` ## Are these changes tested? Yes ## Are there any user-facing changes? No - shared helpers added are only visible within in the crate, rest is just rewrites --- .../functions-nested/src/array_any_match.rs | 134 ++++----- .../functions-nested/src/array_filter.rs | 115 +++++--- .../functions-nested/src/array_first.rs | 128 ++++----- .../functions-nested/src/lambda_utils.rs | 269 +++++++++++++++++- .../test_files/array/array_any_match.slt | 23 ++ .../test_files/array/array_filter.slt | 14 + 6 files changed, 497 insertions(+), 186 deletions(-) diff --git a/datafusion/functions-nested/src/array_any_match.rs b/datafusion/functions-nested/src/array_any_match.rs index b83c56e9e227f..0f620f18bd8f2 100644 --- a/datafusion/functions-nested/src/array_any_match.rs +++ b/datafusion/functions-nested/src/array_any_match.rs @@ -18,17 +18,11 @@ //! [`datafusion_expr::HigherOrderUDF`] definitions for array_any_match function. use arrow::{ - array::{Array, AsArray, BooleanArray, BooleanBuilder, new_null_array}, + array::{Array, BooleanArray, BooleanBuilder}, buffer::NullBuffer, - compute::take_arrays, - datatypes::{ArrowNativeType, DataType, Field, FieldRef}, -}; -use datafusion_common::{ - Result, exec_datafusion_err, exec_err, plan_err, - utils::{ - adjust_offsets_for_slice, list_values, list_values_row_number, take_function_args, - }, + datatypes::{DataType, Field, FieldRef}, }; +use datafusion_common::{Result, plan_err, utils::take_function_args}; use datafusion_expr::{ ColumnarValue, Documentation, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, @@ -37,7 +31,9 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::{fmt::Debug, sync::Arc}; -use crate::lambda_utils::coerce_single_list_arg; +use crate::lambda_utils::{ + SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate, +}; make_higher_order_function_expr_and_func!( ArrayAnyMatch, @@ -160,75 +156,25 @@ impl HigherOrderUDFImpl for ArrayAnyMatch { } fn invoke_with_args(&self, args: HigherOrderFunctionArgs) -> Result { - let [ValueOrLambda::Value(list), ValueOrLambda::Lambda(lambda)] = - take_function_args(self.name(), &args.args)? - else { - return exec_err!("{} expects a value followed by a lambda", self.name()); + let evaluated = match evaluate_single_list_predicate(self.name(), &args)? { + SingleListLambdaResult::EarlyReturn(v) => return Ok(v), + SingleListLambdaResult::Ready(v) => v, }; - let list_array = list.to_array(args.number_rows)?; + let predicate = evaluated.boolean_predicate(self.name())?; - // fast path: fully null input — also required for FixedSizeList which can't be - // handled by clear_null_values when fully null - if list_array.null_count() == list_array.len() { - return Ok(ColumnarValue::Array(new_null_array( - args.return_type(), - list_array.len(), - ))); - } - - let list_values = list_values(&list_array)?; - - let values_param = || Ok(Arc::clone(&list_values)); - - let predicate_results = lambda - .evaluate(&[&values_param], |arrays| { - let indices = list_values_row_number(&list_array)?; - Ok(take_arrays(arrays, &indices, None)?) - })? - .into_array(list_values.len())?; - - let predicate_bool = predicate_results - .as_any() - .downcast_ref::() - .ok_or_else(|| { - exec_datafusion_err!( - "{} predicate must return boolean array", - self.name() - ) - })?; - - let mut values = BooleanBuilder::with_capacity(list_array.len()); - - // Maps predicate results (flat over all elements) back to one Boolean per row. - // Uses adjusted offsets so sliced lists index correctly into the predicate array. - macro_rules! process_list { - ($list_typed:expr) => {{ - let offsets = adjust_offsets_for_slice($list_typed); - for i in 0..$list_typed.len() { - let start = offsets[i].as_usize(); - let end = offsets[i + 1].as_usize(); - // any_match_for_range returns None when nulls poison the result; - // null rows produce an empty range and return Some(false), but their - // null bit is preserved by attaching the original null bitmap below. - values.append_option(any_match_for_range(predicate_bool, start, end)); - } - }}; - } - - match list_array.data_type() { - DataType::List(_) => { - process_list!(list_array.as_list::()); - } - DataType::LargeList(_) => { - process_list!(list_array.as_list::()); - } - other => return exec_err!("expected list, got {other}"), + let mut values = BooleanBuilder::with_capacity(evaluated.len()); + for i in 0..evaluated.len() { + let (start, end) = evaluated.row_range(i); + // any_match_for_range returns None when nulls poison the result; + // null rows produce an empty range and return Some(false), but their + // null bit is preserved by attaching the original null bitmap below. + values.append_option(any_match_for_range(&predicate, start, end)); } let (boolean_buffer, predicate_nulls) = values.finish().into_parts(); // Merge: a row is null if the input list row was null or the predicate returned null. - let nulls = NullBuffer::union(list_array.nulls(), predicate_nulls.as_ref()); + let nulls = NullBuffer::union(evaluated.nulls(), predicate_nulls.as_ref()); Ok(ColumnarValue::Array(Arc::new(BooleanArray::new( boolean_buffer, nulls, @@ -260,6 +206,10 @@ mod tests { use datafusion_physical_expr::create_physical_expr; use crate::array_any_match::{ArrayAnyMatch, array_any_match_higher_order_function}; + use crate::lambda_utils::test_utils::{ + create_i32_large_list, create_i32_list, eval_hof_on_i32_list, + eval_hof_on_i32_list_with_outer, v, + }; fn run_any_match( list: impl arrow::array::Array + Clone + 'static, @@ -500,4 +450,44 @@ mod tests { ); Ok(()) } + + #[test] + fn test_any_match_large_list_parity() -> Result<()> { + let list = create_i32_large_list( + vec![1, 2, 3], + OffsetBuffer::::from_lengths(vec![3]), + None, + ); + let result = eval_hof_on_i32_list( + array_any_match_higher_order_function(), + list, + v().gt(lit(2i32)), + )?; + assert_eq!( + result.as_any().downcast_ref::().unwrap(), + &BooleanArray::from(vec![Some(true)]) + ); + Ok(()) + } + + #[test] + fn test_any_match_captured_outer_column() -> Result<()> { + let list = create_i32_list( + vec![1, 50, 4, 50, 7, 50], + OffsetBuffer::::from_lengths(vec![2, 2, 2]), + None, + ); + let number = Int32Array::from(vec![10, 40, 60]); + let result = eval_hof_on_i32_list_with_outer( + array_any_match_higher_order_function(), + list, + number, + v().gt(col("number")), + )?; + assert_eq!( + result.as_any().downcast_ref::().unwrap(), + &BooleanArray::from(vec![Some(true), Some(true), Some(false)]) + ); + Ok(()) + } } diff --git a/datafusion/functions-nested/src/array_filter.rs b/datafusion/functions-nested/src/array_filter.rs index 7dd7230ae9e06..3439699433272 100644 --- a/datafusion/functions-nested/src/array_filter.rs +++ b/datafusion/functions-nested/src/array_filter.rs @@ -23,13 +23,10 @@ use arrow::{ OffsetSizeTrait, new_empty_array, }, buffer::{OffsetBuffer, ScalarBuffer}, - compute::{filter as arrow_filter, take_arrays}, + compute::filter as arrow_filter, datatypes::{DataType, Field, FieldRef}, }; -use datafusion_common::{ - Result, ScalarValue, exec_err, - utils::{adjust_offsets_for_slice, list_values_row_number}, -}; +use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ ColumnarValue, Documentation, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, @@ -39,7 +36,7 @@ use datafusion_macros::user_doc; use std::sync::Arc; use crate::lambda_utils::{ - ListValuesResult, coerce_single_list_arg, extract_list_values, + SingleListLambdaResult, coerce_single_list_arg, evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair, }; @@ -130,12 +127,9 @@ impl HigherOrderUDFImpl for ArrayFilter { } fn invoke_with_args(&self, args: HigherOrderFunctionArgs) -> Result { - let (list, lambda) = value_lambda_pair(self.name(), &args.args)?; - let list_array = list.to_array(args.number_rows)?; - - let list_values = match extract_list_values(&list_array, args.return_type())? { - ListValuesResult::EarlyReturn(v) => return Ok(v), - ListValuesResult::Values(v) => v, + let evaluated = match evaluate_single_list_predicate(self.name(), &args)? { + SingleListLambdaResult::EarlyReturn(v) => return Ok(v), + SingleListLambdaResult::Ready(v) => v, }; let field = match args.return_field.data_type() { @@ -149,56 +143,47 @@ impl HigherOrderUDFImpl for ArrayFilter { } }; - let values_param = || Ok(Arc::clone(&list_values)); - let predicate_output = lambda.evaluate(&[&values_param], |arrays| { - let indices = list_values_row_number(&list_array)?; - Ok(take_arrays(arrays, &indices, None)?) - })?; - // Scalar predicate short-circuit: x -> true or x -> false/null - if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) = &predicate_output { + if let ColumnarValue::Scalar(ScalarValue::Boolean(b)) = + &evaluated.evaluated_result + { return match b { - Some(true) => Ok(ColumnarValue::Array(list_array)), + Some(true) => Ok(ColumnarValue::Array(evaluated.original_list)), _ => Ok(ColumnarValue::Array(empty_filtered_list( - &list_array, + &evaluated.original_list, field, )?)), }; } - let predicate = predicate_output.into_array(list_values.len())?; - let Some(predicate) = predicate.as_any().downcast_ref::() else { - return exec_err!( - "{} lambda must return boolean, got {}", - self.name(), - predicate.data_type() - ); - }; + let predicate = evaluated.boolean_predicate(self.name())?; // ListView and LargeListView are coerced to List/LargeList by coerce_value_types. - let filtered_list = match list_array.data_type() { + let filtered_list = match evaluated.original_list.data_type() { DataType::List(_) => { - let list = list_array.as_list::(); - let adjusted_offsets = adjust_offsets_for_slice(list); - let (filtered_values, new_offsets) = - filter_list_values(&list_values, predicate, &adjusted_offsets)?; + let (filtered_values, new_offsets) = filter_list_values( + &evaluated.flattened_values, + &predicate, + &evaluated.adjusted_offsets::(), + )?; Arc::new(ListArray::new( field, new_offsets, filtered_values, - list.nulls().cloned(), + evaluated.nulls().cloned(), )) as ArrayRef } DataType::LargeList(_) => { - let large_list = list_array.as_list::(); - let adjusted_offsets = adjust_offsets_for_slice(large_list); - let (filtered_values, new_offsets) = - filter_list_values(&list_values, predicate, &adjusted_offsets)?; + let (filtered_values, new_offsets) = filter_list_values( + &evaluated.flattened_values, + &predicate, + &evaluated.adjusted_offsets::(), + )?; Arc::new(LargeListArray::new( field, new_offsets, filtered_values, - large_list.nulls().cloned(), + evaluated.nulls().cloned(), )) } other => exec_err!("expected list, got {other}")?, @@ -284,9 +269,14 @@ mod tests { buffer::{NullBuffer, OffsetBuffer}, }; + use arrow::array::Int32Array; + use crate::array_filter::array_filter_higher_order_function; - use crate::lambda_utils::test_utils::{create_i32_list, eval_hof_on_i32_list, v}; - use datafusion_expr::lit; + use crate::lambda_utils::test_utils::{ + create_i32_large_list, create_i32_list, eval_hof_on_i32_list, + eval_hof_on_i32_list_with_outer, v, + }; + use datafusion_expr::{col, lit}; fn keep_greater_than_two( list: impl Array + Clone + 'static, @@ -456,4 +446,45 @@ mod tests { ); assert_eq!(actual, &expected); } + + #[test] + fn filter_large_list_parity() { + let list = create_i32_large_list( + vec![1, 2, 3, 4, 5], + OffsetBuffer::::from_lengths(vec![5]), + None, + ); + let res = keep_greater_than_two(list).unwrap(); + let actual = res.as_list::(); + let expected = create_i32_large_list( + vec![3, 4, 5], + OffsetBuffer::::from_lengths(vec![3]), + None, + ); + assert_eq!(actual, &expected); + } + + #[test] + fn filter_captured_outer_column() { + let list = create_i32_list( + vec![1, 50, 4, 50, 7, 50], + OffsetBuffer::::from_lengths(vec![2, 2, 2]), + None, + ); + let number = Int32Array::from(vec![10, 40, 60]); + let res = eval_hof_on_i32_list_with_outer( + array_filter_higher_order_function(), + list, + number, + v().gt(col("number")), + ) + .unwrap(); + let actual = res.as_list::(); + let expected = create_i32_list( + vec![50, 50], + OffsetBuffer::::from_lengths(vec![1, 1, 0]), + None, + ); + assert_eq!(actual, &expected); + } } diff --git a/datafusion/functions-nested/src/array_first.rs b/datafusion/functions-nested/src/array_first.rs index 07154a2db74b7..615dc47394379 100644 --- a/datafusion/functions-nested/src/array_first.rs +++ b/datafusion/functions-nested/src/array_first.rs @@ -18,17 +18,11 @@ //! [`datafusion_expr::HigherOrderUDF`] definitions for array_first function. use arrow::{ - array::{ - Array, AsArray, BooleanArray, GenericListArray, OffsetSizeTrait, UInt64Array, - UInt64Builder, new_null_array, - }, - compute::{take, take_arrays}, + array::{Array, BooleanArray, UInt64Array, UInt64Builder}, + compute::take, datatypes::{DataType, FieldRef}, }; -use datafusion_common::{ - Result, exec_datafusion_err, exec_err, plan_err, - utils::{adjust_offsets_for_slice, list_values, list_values_row_number}, -}; +use datafusion_common::{Result, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, Documentation, HigherOrderFunctionArgs, HigherOrderReturnFieldArgs, HigherOrderSignature, HigherOrderUDFImpl, LambdaParametersProgress, ValueOrLambda, @@ -38,7 +32,8 @@ use datafusion_macros::user_doc; use std::sync::Arc; use crate::lambda_utils::{ - coerce_single_list_arg, single_list_lambda_parameters, value_lambda_pair, + EvaluatedListLambda, SingleListLambdaResult, coerce_single_list_arg, + evaluate_single_list_predicate, single_list_lambda_parameters, value_lambda_pair, }; make_higher_order_function_expr_and_func!( @@ -147,58 +142,20 @@ impl HigherOrderUDFImpl for ArrayFirst { } fn invoke_with_args(&self, args: HigherOrderFunctionArgs) -> Result { - let (list, lambda) = value_lambda_pair(self.name(), &args.args)?; - - let list_array = list.to_array(args.number_rows)?; - - // Fast path: fully null input. Also required for FixedSizeList which - // can't be handled by clear_null_values when fully null. - if list_array.null_count() == list_array.len() { - return Ok(ColumnarValue::Array(new_null_array( - args.return_type(), - list_array.len(), - ))); - } + let evaluated = match evaluate_single_list_predicate(self.name(), &args)? { + SingleListLambdaResult::EarlyReturn(v) => return Ok(v), + SingleListLambdaResult::Ready(v) => v, + }; - let list_values = list_values(&list_array)?; - - // Evaluate the predicate over every flat element. Captured columns are - // spread to align with the flattened values via list_values_row_number. - let values_param = || Ok(Arc::clone(&list_values)); - - let predicate_results = lambda - .evaluate(&[&values_param], |arrays| { - let indices = list_values_row_number(&list_array)?; - Ok(take_arrays(arrays, &indices, None)?) - })? - .into_array(list_values.len())?; - - let predicate_bool = predicate_results - .as_any() - .downcast_ref::() - .ok_or_else(|| { - exec_datafusion_err!( - "{} predicate must return boolean array, got {}", - self.name(), - predicate_results.data_type() - ) - })?; - - // For each row, find the flat index of the first element whose predicate - // is true. Rows with no match, including empty rows and null rows that - // clear_null_values truncated to empty, map to a null index, producing - // a null result via `take`. - let indices = match list_array.data_type() { - DataType::List(_) => { - first_match_indices(list_array.as_list::(), predicate_bool) - } - DataType::LargeList(_) => { - first_match_indices(list_array.as_list::(), predicate_bool) + let predicate = evaluated.boolean_predicate(self.name())?; + let indices = match evaluated.original_list.data_type() { + DataType::List(_) | DataType::LargeList(_) => { + first_match_indices(&evaluated, &predicate) } other => return exec_err!("expected list, got {other}"), }; - let result = take(list_values.as_ref(), &indices, None)?; + let result = take(evaluated.flattened_values.as_ref(), &indices, None)?; Ok(ColumnarValue::Array(result)) } @@ -213,18 +170,14 @@ impl HigherOrderUDFImpl for ArrayFirst { /// /// A null predicate value is treated as not matching. The matched element itself /// may be null and is still returned. -fn first_match_indices( - list: &GenericListArray, +fn first_match_indices( + evaluated: &EvaluatedListLambda, predicate: &BooleanArray, ) -> UInt64Array { - // Offsets are adjusted so that sliced lists index correctly into the - // predicate / values arrays returned by list_values. - let offsets = adjust_offsets_for_slice(list); - let mut builder = UInt64Builder::with_capacity(list.len()); + let mut builder = UInt64Builder::with_capacity(evaluated.len()); - for i in 0..list.len() { - let start = offsets[i].as_usize(); - let end = offsets[i + 1].as_usize(); + for i in 0..evaluated.len() { + let (start, end) = evaluated.row_range(i); match (start..end).find(|&j| predicate.is_valid(j) && predicate.value(j)) { Some(j) => builder.append_value(j as u64), @@ -244,9 +197,12 @@ mod tests { }; use crate::array_first::array_first_higher_order_function; - use crate::lambda_utils::test_utils::{create_i32_list, eval_hof_on_i32_list, v}; + use crate::lambda_utils::test_utils::{ + create_i32_large_list, create_i32_list, eval_hof_on_i32_list, + eval_hof_on_i32_list_with_outer, v, + }; use datafusion_common::Result; - use datafusion_expr::lit; + use datafusion_expr::{col, lit}; fn first_greater_than_two( list: impl Array + Clone + 'static, @@ -411,6 +367,42 @@ mod tests { ); } + #[test] + fn test_first_large_list_parity() -> Result<()> { + let list = create_i32_large_list( + vec![1, 2, 3, 4, 5], + OffsetBuffer::::from_lengths(vec![5]), + None, + ); + let res = first_greater_than_two(list)?; + assert_eq!( + res.as_primitive::(), + &Int32Array::from(vec![Some(3)]) + ); + Ok(()) + } + + #[test] + fn test_first_captured_outer_column() -> Result<()> { + let list = create_i32_list( + vec![1, 50, 4, 50, 7, 50], + OffsetBuffer::::from_lengths(vec![2, 2, 2]), + None, + ); + let number = Int32Array::from(vec![10, 40, 60]); + let res = eval_hof_on_i32_list_with_outer( + array_first_higher_order_function(), + list, + number, + v().gt(col("number")), + )?; + assert_eq!( + res.as_primitive::(), + &Int32Array::from(vec![Some(50), Some(50), None]) + ); + Ok(()) + } + #[test] fn test_first_string_elements() -> Result<()> { use arrow::array::ListArray; diff --git a/datafusion/functions-nested/src/lambda_utils.rs b/datafusion/functions-nested/src/lambda_utils.rs index 927ca5a51461c..4b01ae314e4c7 100644 --- a/datafusion/functions-nested/src/lambda_utils.rs +++ b/datafusion/functions-nested/src/lambda_utils.rs @@ -17,13 +17,19 @@ //! Shared utilities for `(array, lambda)` style higher-order functions. -use arrow::array::ArrayRef; -use arrow::datatypes::{DataType, FieldRef}; +use arrow::array::{ArrayRef, AsArray, BooleanArray, OffsetSizeTrait, new_null_array}; +use arrow::buffer::{NullBuffer, OffsetBuffer}; +use arrow::compute::take_arrays; +use arrow::datatypes::{ArrowNativeType, DataType, FieldRef}; +use datafusion_common::utils::{adjust_offsets_for_slice, list_values_row_number}; use datafusion_common::{ Result, ScalarValue, plan_err, utils::{list_values, take_function_args}, }; -use datafusion_expr::{ColumnarValue, LambdaParametersProgress, ValueOrLambda}; +use datafusion_common::{exec_datafusion_err, exec_err}; +use datafusion_expr::{ + ColumnarValue, HigherOrderFunctionArgs, LambdaParametersProgress, ValueOrLambda, +}; use std::sync::Arc; /// Extracts a `(value, lambda)` pair from a [`ValueOrLambda`] slice. @@ -126,12 +132,225 @@ pub(crate) fn extract_list_values( Ok(ListValuesResult::Values(values)) } +pub(crate) enum SingleListLambdaResult { + EarlyReturn(ColumnarValue), + Ready(EvaluatedListLambda), +} + +pub(crate) struct EvaluatedListLambda { + pub original_list: ArrayRef, + pub flattened_values: ArrayRef, + pub evaluated_result: ColumnarValue, + row_offsets: Vec, +} + +impl EvaluatedListLambda { + pub(crate) fn len(&self) -> usize { + self.original_list.len() + } + + pub(crate) fn nulls(&self) -> Option<&NullBuffer> { + self.original_list.nulls() + } + + pub(crate) fn row_range(&self, i: usize) -> (usize, usize) { + (self.row_offsets[i], self.row_offsets[i + 1]) + } + + pub(crate) fn adjusted_offsets(&self) -> OffsetBuffer { + OffsetBuffer::from_lengths(self.row_offsets.windows(2).map(|w| w[1] - w[0])) + } + + pub(crate) fn boolean_predicate(&self, name: &str) -> Result { + let arr = self + .evaluated_result + .clone() + .into_array(self.flattened_values.len())?; + + let predicate = arr.as_any().downcast_ref::().ok_or_else(|| { + exec_datafusion_err!("{} predicate must return boolean array", name) + })?; + + Ok(predicate.clone()) + } +} + +fn adjusted_row_offsets(list: &ArrayRef) -> Result> { + Ok(match list.data_type() { + DataType::List(_) => adjust_offsets_for_slice(list.as_list::()) + .iter() + .map(|o| o.as_usize()) + .collect(), + DataType::LargeList(_) => adjust_offsets_for_slice(list.as_list::()) + .iter() + .map(|o| o.as_usize()) + .collect(), + other => return exec_err!("expected list, got {other}"), + }) +} + +fn evaluate_single_list_lambda( + name: &str, + args: &HigherOrderFunctionArgs, +) -> Result { + let (original_list, lambda) = value_lambda_pair(name, &args.args)?; + let original_list = original_list.to_array(args.number_rows)?; + + if original_list.null_count() == original_list.len() { + return Ok(SingleListLambdaResult::EarlyReturn(ColumnarValue::Array( + new_null_array(args.return_type(), original_list.len()), + ))); + } + + let flattened_values = list_values(&original_list)?; + let values_param = || Ok(Arc::clone(&flattened_values)); + + let evaluated_result = lambda.evaluate(&[&values_param], |arrays| { + let indices = list_values_row_number(&original_list)?; + Ok(take_arrays(arrays, &indices, None)?) + })?; + + let row_offsets = adjusted_row_offsets(&original_list)?; + + Ok(SingleListLambdaResult::Ready(EvaluatedListLambda { + original_list, + flattened_values, + evaluated_result, + row_offsets, + })) +} + +pub(crate) fn evaluate_single_list_predicate( + name: &str, + args: &HigherOrderFunctionArgs, +) -> Result { + let result = evaluate_single_list_lambda(name, args)?; + let SingleListLambdaResult::Ready(evaluated_list_lambda) = &result else { + return Ok(result); + }; + + match &evaluated_list_lambda.evaluated_result { + ColumnarValue::Scalar(ScalarValue::Boolean(_)) => Ok(result), + ColumnarValue::Scalar(scalar) => exec_err!( + "{name} lambda must return boolean, got {}", + scalar.data_type() + ), + ColumnarValue::Array(array) if array.as_any().is::() => Ok(result), + ColumnarValue::Array(array) => exec_err!( + "{name} lambda must return boolean, got {}", + array.data_type() + ), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow::{ + array::ArrayRef, + buffer::{NullBuffer, OffsetBuffer}, + datatypes::{DataType, Field}, + }; + use datafusion_common::Result; + + use super::{adjusted_row_offsets, coerce_single_list_arg}; + use crate::lambda_utils::test_utils::{create_i32_large_list, create_i32_list}; + + #[test] + fn adjusted_row_offsets_matches_list_lengths() -> Result<()> { + let list = create_i32_list( + vec![1, 2, 3, 4, 5], + OffsetBuffer::::from_lengths(vec![2, 0, 3]), + None, + ); + let list = Arc::new(list) as ArrayRef; + assert_eq!(adjusted_row_offsets(&list)?, vec![0, 2, 2, 5]); + Ok(()) + } + + #[test] + fn adjusted_row_offsets_on_sliced_list() -> Result<()> { + let list = create_i32_list( + vec![10, 1, 2, 3, 4], + OffsetBuffer::::from_lengths(vec![1, 2, 2]), + None, + ) + .slice(1, 2); + let list = Arc::new(list) as ArrayRef; + assert_eq!(adjusted_row_offsets(&list)?, vec![0, 2, 4]); + Ok(()) + } + + #[test] + fn adjusted_row_offsets_null_rows_keep_backing_lengths() -> Result<()> { + let list = create_i32_list( + vec![1, 99, 100, 2], + OffsetBuffer::::from_lengths(vec![1, 2, 1]), + Some(NullBuffer::from(vec![true, false, true])), + ); + let list = Arc::new(list) as ArrayRef; + assert_eq!(adjusted_row_offsets(&list)?, vec![0, 1, 3, 4]); + Ok(()) + } + + #[test] + fn adjusted_row_offsets_large_list_parity() -> Result<()> { + let list = create_i32_large_list( + vec![1, 2, 3, 4], + OffsetBuffer::::from_lengths(vec![1, 3]), + None, + ); + let list = Arc::new(list) as ArrayRef; + assert_eq!(adjusted_row_offsets(&list)?, vec![0, 1, 4]); + Ok(()) + } + + #[test] + fn coerce_single_list_arg_supports_advertised_list_likes() -> Result<()> { + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); + assert_eq!( + coerce_single_list_arg("test", &[DataType::List(Arc::clone(&field))])?, + vec![DataType::List(Arc::clone(&field))] + ); + assert_eq!( + coerce_single_list_arg("test", &[DataType::LargeList(Arc::clone(&field))])?, + vec![DataType::LargeList(Arc::clone(&field))] + ); + assert_eq!( + coerce_single_list_arg( + "test", + &[DataType::FixedSizeList(Arc::clone(&field), 3)] + )?, + vec![DataType::List(Arc::clone(&field))] + ); + assert_eq!( + coerce_single_list_arg("test", &[DataType::ListView(Arc::clone(&field))])?, + vec![DataType::List(Arc::clone(&field))] + ); + assert_eq!( + coerce_single_list_arg( + "test", + &[DataType::LargeListView(Arc::clone(&field))] + )?, + vec![DataType::LargeList(field)] + ); + Ok(()) + } + + #[test] + fn coerce_single_list_arg_rejects_non_list() { + let err = coerce_single_list_arg("test", &[DataType::Int32]).unwrap_err(); + assert!(err.to_string().contains("expected a list")); + } +} + #[cfg(test)] pub(crate) mod test_utils { use std::{collections::HashMap, sync::Arc}; use arrow::{ - array::{Array, ArrayRef, Int32Array, ListArray, RecordBatch}, + array::{Array, ArrayRef, Int32Array, LargeListArray, ListArray, RecordBatch}, buffer::{NullBuffer, OffsetBuffer}, datatypes::{DataType, Field}, }; @@ -154,6 +373,15 @@ pub(crate) mod test_utils { ListArray::new(list_field, offsets, Arc::new(values.into()), nulls) } + pub(crate) fn create_i32_large_list( + values: impl Into, + offsets: OffsetBuffer, + nulls: Option, + ) -> LargeListArray { + let list_field = Arc::new(Field::new_list_field(DataType::Int32, true)); + LargeListArray::new(list_field, offsets, Arc::new(values.into()), nulls) + } + pub(crate) fn eval_hof_on_i32_list( func: Arc, list: impl Array + Clone + 'static, @@ -185,6 +413,39 @@ pub(crate) mod test_utils { .into_array(list.len()) } + /// Evaluates a HOF whose lambda body may capture an outer `number` column. + pub(crate) fn eval_hof_on_i32_list_with_outer( + func: Arc, + list: impl Array + Clone + 'static, + number: Int32Array, + lambda_body: Expr, + ) -> Result { + assert_eq!(list.len(), number.len()); + let schema = DFSchema::from_unqualified_fields( + vec![ + Field::new("list", list.data_type().clone(), list.is_nullable()), + Field::new("number", DataType::Int32, true), + ] + .into(), + HashMap::new(), + )?; + + create_physical_expr( + &Expr::HigherOrderFunction(HigherOrderFunction::new( + func, + vec![col("list"), lambda(["v"], lambda_body)], + )), + &schema, + &ExecutionProps::new(), + &PhysicalPlanningContext::default(), + )? + .evaluate(&RecordBatch::try_new( + Arc::clone(schema.inner()), + vec![Arc::new(list.clone()), Arc::new(number)], + )?)? + .into_array(list.len()) + } + pub(crate) fn v() -> Expr { Expr::LambdaVariable(LambdaVariable::new( "v".to_string(), diff --git a/datafusion/sqllogictest/test_files/array/array_any_match.slt b/datafusion/sqllogictest/test_files/array/array_any_match.slt index 37aa47c55adcf..82133054e118a 100644 --- a/datafusion/sqllogictest/test_files/array/array_any_match.slt +++ b/datafusion/sqllogictest/test_files/array/array_any_match.slt @@ -109,6 +109,29 @@ SELECT array_any_match(NULL, x -> x > 2); ---- NULL +# predicate can reference an outer column +query B +SELECT array_any_match(list, x -> x > number) FROM t; +---- +true +true +false + +# large list works +query B +SELECT array_any_match(arrow_cast([1, 2, 3], 'LargeList(Int32)'), x -> x > 2); +---- +true + +# other list representations are coerced during planning +query BBB +SELECT + array_any_match(arrow_cast([1, 2, 3], 'FixedSizeList(3, Int32)'), x -> x > 2), + array_any_match(arrow_cast([1, 2, 3], 'ListView(Int32)'), x -> x > 2), + array_any_match(arrow_cast([1, 2, 3], 'LargeListView(Int32)'), x -> x > 2); +---- +true true true + statement ok drop table t; diff --git a/datafusion/sqllogictest/test_files/array/array_filter.slt b/datafusion/sqllogictest/test_files/array/array_filter.slt index 9b564c5061205..b6d73fbe7d09d 100644 --- a/datafusion/sqllogictest/test_files/array/array_filter.slt +++ b/datafusion/sqllogictest/test_files/array/array_filter.slt @@ -120,6 +120,20 @@ SELECT array_filter(arrow_cast(list, 'ListView(Int32)'), v -> v > 2) from t; [4, 50] [7, 50] +# large list works +query ? +SELECT array_filter(arrow_cast([1, 2, 3, 4, 5], 'LargeList(Int32)'), v -> v > 2); +---- +[3, 4, 5] + +# FixedSizeList / LargeListView coercions during planning +query ?? +SELECT + array_filter(arrow_cast([1, 2, 3, 4], 'FixedSizeList(4, Int32)'), v -> v > 2), + array_filter(arrow_cast([1, 2, 3, 4], 'LargeListView(Int32)'), v -> v > 2); +---- +[3, 4] [3, 4] + # null array argument returns null query ? SELECT array_filter(arrow_cast(NULL, 'List(Int32)'), v -> v > 0); From 0f7bdddd37b91c49a430997f0937de12e47e7328 Mon Sep 17 00:00:00 2001 From: Alessandro Solimando Date: Thu, 30 Jul 2026 10:13:35 +0200 Subject: [PATCH 072/109] fix(common): preserve an exact zero through filter selectivity estimation (#23936) ## Which issue does this PR close? - Part of #8227. ## Rationale for this change `Precision::with_estimated_selectivity` always demoted to inexact, so `Exact(0)` became `Inexact(0)`. Scaling zero by any selectivity is still exactly zero: filtering a provably empty input cannot produce rows. Because operators widen exact to inexact, emptiness could not propagate up a plan through statistics at all, it was lost at the first `FilterExec`. That operator already preserved the exact zero for a contradictory predicate and for column byte sizes, so the same fact came out exact or inexact depending on the path taken. This is the same reasoning as #23670, which stops `FileScanConfig::statistics()` demoting an exact zero when a filter is present, applied one level up. The two are independent, but together they let a proof of emptiness survive from the scan through the filter. ## What changes are included in this PR? - `with_estimated_selectivity` returns an exact zero unchanged. `Inexact(0)` is untouched: an estimate of zero is not a proof of zero. - `scale_byte_size` in `filter.rs` is removed, subsumed by the above. ## Are these changes tested? Yes. A unit test for the method, and a `FilterExec` test that a satisfiable predicate over an exactly empty input keeps `num_rows`, `total_byte_size` (proving that `scale_byte_size` can be safely removed) and column `byte_size` exact. `test_filter_statistics_empty_input_equality_ndv_zero` asserted `Inexact(0)` and now asserts `Exact(0)`, which is expected. ## Are there any user-facing changes? No breaking changes. `Precision::with_estimated_selectivity` is public and keeps its signature, but its documented contract changes: it previously stated it would always return inexact statistics, and it now preserves an exact zero. Callers relying on the old wording will see `Exact(0)` where they saw `Inexact(0)`. This is an improvement and could be arguably considered a bug-fix, so I consider this contract change justified. `FilterExec` over a provably empty input therefore reports exact statistics, visible in `EXPLAIN` output that shows statistics. ---- Disclaimer: I used AI to assist in the code generation, I have manually reviewed the output and it matches my intention and understanding. --- datafusion/common/src/stats.rs | 44 ++++++++++++++++++- datafusion/physical-plan/src/filter.rs | 58 +++++++++++++++++++------- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rs index a2ede7ec3de52..b7db556ee8e3a 100644 --- a/datafusion/common/src/stats.rs +++ b/datafusion/common/src/stats.rs @@ -195,8 +195,12 @@ impl Precision { /// Return the estimate of applying a filter with estimated selectivity /// `selectivity` to this Precision. A selectivity of `1.0` means that all /// rows are selected. A selectivity of `0.5` means half the rows are - /// selected. Will always return inexact statistics. + /// selected. An exact zero is preserved, since filtering an empty input + /// cannot produce rows; any other known value is demoted to inexact. pub fn with_estimated_selectivity(self, selectivity: f64) -> Self { + if self == Precision::Exact(0) { + return self; + } self.map(|v| ((v as f64 * selectivity).ceil()) as usize) .to_inexact() } @@ -1202,6 +1206,44 @@ mod tests { assert_eq!(absent_precision.get_value(), None); } + #[test] + fn test_with_estimated_selectivity() { + // Filtering an empty input cannot produce rows, so the zero stays exact. + assert_eq!( + Precision::Exact(0).with_estimated_selectivity(0.5), + Precision::Exact(0) + ); + assert_eq!( + Precision::Exact(0).with_estimated_selectivity(1.0), + Precision::Exact(0) + ); + + // Any other known value is scaled and demoted, since the selectivity is + // itself an estimate. + assert_eq!( + Precision::Exact(100).with_estimated_selectivity(0.5), + Precision::Inexact(50) + ); + assert_eq!( + Precision::Exact(100).with_estimated_selectivity(1.0), + Precision::Inexact(100) + ); + assert_eq!( + Precision::Exact(3).with_estimated_selectivity(0.5), + Precision::Inexact(2) + ); + + // An inexact zero is an estimate, not a proof, and stays inexact. + assert_eq!( + Precision::Inexact(0).with_estimated_selectivity(0.5), + Precision::Inexact(0) + ); + assert_eq!( + Precision::::Absent.with_estimated_selectivity(0.5), + Precision::Absent + ); + } + #[test] fn test_map() { let exact_precision = Precision::Exact(42); diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index d367be16eb6ed..511be0bbdd9e2 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -384,7 +384,8 @@ impl FilterExec { input_num_rows.with_estimated_selectivity(selectivity); let mut cs = input_stats.to_inexact().column_statistics; for (idx, col_stat) in cs.iter_mut().enumerate() { - col_stat.byte_size = scale_byte_size(col_stat.byte_size, selectivity); + col_stat.byte_size = + col_stat.byte_size.with_estimated_selectivity(selectivity); col_stat.null_count = if null_rejecting_columns.contains(&idx) { Precision::Exact(0) } else { @@ -1030,16 +1031,6 @@ fn interval_bound_to_precision( } } -/// Scales a column's `byte_size` by the estimated filter `selectivity`. An -/// exact zero is preserved: an empty column stays exactly empty after -/// filtering. -fn scale_byte_size(byte_size: Precision, selectivity: f64) -> Precision { - match byte_size { - Precision::Exact(0) => Precision::Exact(0), - byte_size => byte_size.with_estimated_selectivity(selectivity), - } -} - /// Caps a row-bounded column statistic (a null count or distinct count) at the /// filtered row estimate, since a column cannot have more nulls or distinct /// values than it has rows. Known counts are demoted to inexact because the @@ -1133,8 +1124,9 @@ fn collect_new_statistics( } else { cap_at_rows(input_column_stats[idx].null_count, filtered_num_rows) }; - let byte_size = - scale_byte_size(input_column_stats[idx].byte_size, selectivity); + let byte_size = input_column_stats[idx] + .byte_size + .with_estimated_selectivity(selectivity); ColumnStatistics { null_count: capped_null_count, max_value, @@ -2909,6 +2901,42 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_filter_statistics_preserves_exactly_empty_input() -> Result<()> { + // A satisfiable predicate over an exactly empty input: the filter cannot + // produce rows, so the whole estimate stays exact. + let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + let input_stats = Statistics { + num_rows: Precision::Exact(0), + total_byte_size: Precision::Exact(0), + column_statistics: vec![ColumnStatistics { + null_count: Precision::Exact(0), + byte_size: Precision::Exact(0), + ..Default::default() + }], + }; + let predicate = Arc::new(BinaryExpr::new( + Arc::new(Column::new("a", 0)), + Operator::Gt, + Arc::new(Literal::new(ScalarValue::Int32(Some(5)))), + )); + + let input = Arc::new(StatisticsExec::new(input_stats, schema)); + let filter: Arc = + Arc::new(FilterExec::try_new(predicate, input)?); + let statistics = + StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?; + + assert_eq!(statistics.num_rows, Precision::Exact(0)); + assert_eq!(statistics.total_byte_size, Precision::Exact(0)); + assert_eq!( + statistics.column_statistics[0].byte_size, + Precision::Exact(0) + ); + + Ok(()) + } + #[tokio::test] async fn test_filter_statistics_empty_input_equality_ndv_zero() -> Result<()> { let cases: Vec<(&str, Schema, Statistics, Arc)> = vec![ @@ -2963,12 +2991,12 @@ mod tests { assert_eq!( statistics.num_rows, - Precision::Inexact(0), + Precision::Exact(0), "case '{desc}': row count mismatch" ); assert_eq!( statistics.column_statistics[0].distinct_count, - Precision::Inexact(0), + Precision::Exact(0), "case '{desc}': NDV should be capped at zero rows" ); } From 86da5a85a2180a763269317e29cb01b69b2d07ce Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:33:08 +0300 Subject: [PATCH 073/109] chore(ordered-partial-aggregate): move `OrderedPartialAggregateStream` to generators for readability (#23951) ## Which issue does this PR close? N/A ## Rationale for this change Simplify the code by making it linear and with less state to hold in mind ## What changes are included in this PR? change `OrderedPartialAggregateStream` into async generators ## Are these changes tested? existing tests ## Are there any user-facing changes? no --------- Co-authored-by: Yongting You <2010youy01@gmail.com> --- .../physical-plan/src/aggregates/mod.rs | 7 +- .../src/aggregates/ordered_partial_stream.rs | 390 ++++++------------ 2 files changed, 136 insertions(+), 261 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 370c75961608c..db1eb951d6fbc 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -697,7 +697,7 @@ impl From for SendableRecordBatchStream { StreamType::PartialReduceHash(stream) => Box::pin(stream), StreamType::FinalHash(stream) => Box::pin(stream), StreamType::SingleHash(stream) => Box::pin(stream), - StreamType::OrderedPartialAggregate(stream) => Box::pin(stream), + StreamType::OrderedPartialAggregate(stream) => stream.into_stream(), StreamType::OrderedFinalAggregate(stream) => Box::pin(stream), StreamType::GroupedHash(stream) => Box::pin(stream), StreamType::GroupedPriorityQueue(stream) => Box::pin(stream), @@ -4429,9 +4429,8 @@ mod tests { .with_session_config(session_config), ); - let mut stream: SendableRecordBatchStream = Box::pin( - OrderedPartialAggregateStream::new(&aggregate, &task_ctx, 0)?, - ); + let mut stream: SendableRecordBatchStream = + OrderedPartialAggregateStream::new(&aggregate, &task_ctx, 0)?.into_stream(); while let Some(result) = stream.next().await { if let Err(e) = result { diff --git a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs index 73d15a8278692..975acc198007f 100644 --- a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs +++ b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs @@ -17,24 +17,22 @@ //! Partial aggregate stream for ordered group input. -use std::ops::ControlFlow; use std::sync::Arc; -use std::task::{Context, Poll}; use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use datafusion_common::{DataFusionError, Result}; -use datafusion_execution::TaskContext; use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation}; +use datafusion_execution::{TaskContext, TryEmitter, async_try_stream}; use futures::stream::{Stream, StreamExt}; use super::AggregateExec; use super::aggregate_hash_table::{OrderedAggregateTable, PartialMarker}; use crate::aggregates::AggregateMode; use crate::aggregates::order::GroupOrdering; -use crate::metrics::{BaselineMetrics, MetricBuilder, RecordOutput, SpillMetrics}; -use crate::stream::EmptyRecordBatchStream; -use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream, metrics}; +use crate::metrics::{BaselineMetrics, MetricBuilder, SpillMetrics}; +use crate::stream::{EmptyRecordBatchStream, ObservedStream, RecordBatchStreamAdapter}; +use crate::{InputOrderMode, SendableRecordBatchStream, metrics}; /// Partial aggregate stream for `InputOrderMode::Sorted` and /// `InputOrderMode::PartiallySorted`. @@ -118,26 +116,9 @@ pub(crate) struct OrderedPartialAggregateStream { reservation: MemoryReservation, baseline_metrics: BaselineMetrics, reduction_factor: metrics::RatioMetrics, - state: Option, + table: Option>, } -/// See comments at `poll_next()` for details. -enum OrderedPartialAggregateState { - ReadingInput { - table: OrderedAggregateTable, - }, - DrainingFinal { - table: OrderedAggregateTable, - }, - Done, -} - -type OrderedPartialAggregatePoll = Poll>>; -type OrderedPartialAggregateStateTransition = ControlFlow< - (OrderedPartialAggregatePoll, OrderedPartialAggregateState), - OrderedPartialAggregateState, ->; - impl OrderedPartialAggregateStream { pub fn new( agg: &AggregateExec, @@ -178,7 +159,82 @@ impl OrderedPartialAggregateStream { reservation, baseline_metrics, reduction_factor, - state: Some(OrderedPartialAggregateState::ReadingInput { table }), + table: Some(table), + }) + } + + pub(crate) fn into_stream(self) -> SendableRecordBatchStream { + let schema_clone = Arc::clone(&self.schema); + + let cloned_metrics = self.baseline_metrics.clone(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema_clone, + self.create_stream(), + )); + + Box::pin(ObservedStream::new(stream, cloned_metrics, None)) + } + + /// Entry point for the ordered partial aggregate state machine. + /// + /// See comments in [`OrderedPartialAggregateStream`] for high-level ideas. + /// + /// State transitions are implemented using the generator pattern; see the comments in [`async_try_stream`]. + /// + /// Conceptual state-transition graph: + /// + /// ```text + /// (start) + /// -> ReadingInput + /// The stream starts by polling ordered input and aggregating batches + /// into the ordered partial aggregate table. + /// + /// ReadingInput + /// -> ReadingInput + /// Aggregate one input batch. If the ordering proves some groups are + /// complete, yield one partial-state batch immediately, then continue + /// reading input. Otherwise continue directly with the next input batch. + /// -> DrainingFinal + /// Input was exhausted. Mark the table input as done so every remaining + /// group is safe to emit. + /// + /// DrainingFinal + /// -> DrainingFinal + /// One remaining partial-state batch was yielded; repeat to continue + /// draining the table. + /// -> Done + /// All remaining groups were emitted. + /// + /// Done + /// -> (end) + /// ``` + fn create_stream(mut self) -> impl Stream> { + async_try_stream(|mut emitter| async move { + let mut table = self + .table + .take() + .expect("OrderedPartialAggregateStream state should not be None"); + + self.handle_reading_input(&mut table, &mut emitter).await?; + + // Input has exhausted, move to the final draining stage. + self.close_input(); + table.input_done(); + + let last_batch = self.handle_draining_final(&mut table, &mut emitter).await?; + + // Clear memory before emitting last batch so we don't have to wait for next poll to clear + { + // Clear memory + drop(table); + let _ = self.reservation.try_resize(0); + } + + if let Some(last_batch) = last_batch { + emitter.emit(last_batch).await; + } + + Ok(()) }) } @@ -190,113 +246,43 @@ impl OrderedPartialAggregateStream { /// Consumes one ordered input batch, then immediately emits completed groups /// if the ordering proves any group is ready. /// - /// See comments at `poll_next()` for details. - /// - /// Returns the next operator state with control flow decision. - fn handle_reading_input( + /// See comments at [`Self::create_stream`] for details. + async fn handle_reading_input( &mut self, - cx: &mut Context<'_>, - original_state: OrderedPartialAggregateState, - ) -> OrderedPartialAggregateStateTransition { - let OrderedPartialAggregateState::ReadingInput { mut table } = original_state - else { - unreachable!("expected reading input state") - }; + table: &mut OrderedAggregateTable, + emitter: &mut TryEmitter, + ) -> Result<()> { + let elapsed_compute = self.baseline_metrics.elapsed_compute().clone(); - match self.input.poll_next_unpin(cx) { - Poll::Pending => ControlFlow::Break(( - Poll::Pending, - OrderedPartialAggregateState::ReadingInput { table }, - )), - Poll::Ready(Some(Ok(batch))) => { - let input_rows = batch.num_rows(); - self.reduction_factor.add_total(input_rows); - - let elapsed_compute = self.baseline_metrics.elapsed_compute().clone(); - let timer = elapsed_compute.timer(); - let result = table.aggregate_batch(&batch); - timer.done(); - - if let Err(e) = result { - return ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::ReadingInput { table }, - )); - } - - // Check memory reservation. See function comments for details. - match self.resize_or_take_state_batch(&mut table) { - Ok(Some(batch)) => { - self.reduction_factor.add_part(batch.num_rows()); - return ControlFlow::Break(( - Poll::Ready(Some(Ok( - batch.record_output(&self.baseline_metrics) - ))), - OrderedPartialAggregateState::ReadingInput { table }, - )); - } - Ok(None) => {} - Err(e) => { - self.close_input(); - self.reservation.free(); - return ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::Done, - )); - } - } - - let timer = elapsed_compute.timer(); - let result = table.next_output_batch(); - timer.done(); - - match result { - // There is some previous group results can be emitted: emit - // them, and next continuing aggreagting input (loop in the - // current state) - Ok(Some(batch)) => { - self.reduction_factor.add_part(batch.num_rows()); - if let Err(e) = self.reservation.try_resize(table.memory_size()) { - self.close_input(); - self.reservation.free(); - return ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::Done, - )); - } - let next_state = - OrderedPartialAggregateState::ReadingInput { table }; - - ControlFlow::Break(( - Poll::Ready(Some(Ok( - batch.record_output(&self.baseline_metrics) - ))), - next_state, - )) - } - // Can't do early emit, continue aggregating. - Ok(None) => ControlFlow::Continue( - OrderedPartialAggregateState::ReadingInput { table }, - ), - Err(e) => ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::ReadingInput { table }, - )), - } - } - Poll::Ready(Some(Err(e))) => ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::ReadingInput { table }, - )), - // Input has exhausted, move to the final draining stage. - Poll::Ready(None) => { - self.close_input(); - table.input_done(); - ControlFlow::Continue(OrderedPartialAggregateState::DrainingFinal { - table, - }) + while let Some(batch) = self.input.next().await.transpose()? { + let input_rows = batch.num_rows(); + self.reduction_factor.add_total(input_rows); + + let timer = elapsed_compute.timer(); + + table.aggregate_batch(&batch)?; + + // Check memory reservation. See function comments for details. + if let Some(batch) = self.resize_or_take_state_batch(table)? { + self.reduction_factor.add_part(batch.num_rows()); + drop(timer); + emitter.emit(batch).await; + continue; } + + let Some(batch) = table.next_output_batch()? else { + // Can't do early emit, continue aggregating. + continue; + }; + + self.reduction_factor.add_part(batch.num_rows()); + self.reservation.try_resize(table.memory_size())?; + + drop(timer); + emitter.emit(batch).await; } + + Ok(()) } /// Update the memory reservation, and: @@ -340,143 +326,33 @@ impl OrderedPartialAggregateStream { /// `table.input_done()` has already made every remaining group safe to emit, /// so this state keeps draining until the table is empty. /// - /// See comments at `poll_next()` for details. + /// Returns the last batch to emit so we can free all the state and memory before emitting, + /// and we won't need to hold while waiting for the next poll. /// - /// Returns the next operator state with control flow decision. - fn handle_draining_final( + /// See comments at [`Self::create_stream`] for details. + /// + async fn handle_draining_final( &mut self, - original_state: OrderedPartialAggregateState, - ) -> OrderedPartialAggregateStateTransition { - let OrderedPartialAggregateState::DrainingFinal { table } = original_state else { - unreachable!("expected draining final state") - }; - - let mut table = table; + table: &mut OrderedAggregateTable, + emitter: &mut TryEmitter, + ) -> Result> { let elapsed_compute = self.baseline_metrics.elapsed_compute().clone(); - let timer = elapsed_compute.timer(); - let result = table.next_output_batch(); - timer.done(); + let mut timer = elapsed_compute.timer(); + while let Some(batch) = table.next_output_batch()? { + self.reduction_factor.add_part(batch.num_rows()); - match result { - Ok(Some(batch)) => { - self.reduction_factor.add_part(batch.num_rows()); - let next_state = if table.is_empty() { - OrderedPartialAggregateState::Done - } else { - OrderedPartialAggregateState::DrainingFinal { table } - }; - if let Err(e) = self.resize_reservation_for_state(&next_state) { - return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)); - } - - ControlFlow::Break(( - Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))), - next_state, - )) - } - Err(e) => ControlFlow::Break(( - Poll::Ready(Some(Err(e))), - OrderedPartialAggregateState::DrainingFinal { table }, - )), - Ok(None) => { - let next_state = OrderedPartialAggregateState::Done; - if let Err(e) = self.resize_reservation_for_state(&next_state) { - return ControlFlow::Break((Poll::Ready(Some(Err(e))), next_state)); - } - ControlFlow::Continue(next_state) + if table.is_empty() { + return Ok(Some(batch)); } - } - } - fn resize_reservation_for_state( - &mut self, - state: &OrderedPartialAggregateState, - ) -> Result<()> { - let new_size = match state { - OrderedPartialAggregateState::ReadingInput { table } - | OrderedPartialAggregateState::DrainingFinal { table } => { - table.memory_size() - } - OrderedPartialAggregateState::Done => 0, - }; - self.reservation.try_resize(new_size) - } -} + self.reservation.try_resize(table.memory_size())?; -impl Stream for OrderedPartialAggregateStream { - type Item = Result; - - /// Entry point for the ordered partial aggregate state machine. - /// - /// See comments in [`OrderedPartialAggregateStream`] for high-level ideas. - /// - /// State transition graph: - /// - /// ```text - /// (start) - /// -> ReadingInput - /// The stream starts by polling ordered input and aggregating batches - /// into the ordered partial aggregate table. - /// - /// ReadingInput - /// -> ReadingInput - /// Aggregate one input batch. If the ordering proves some groups are - /// complete, yield one partial-state batch immediately, then continue - /// reading input. Otherwise continue directly with the next input batch. - /// -> DrainingFinal - /// Input was exhausted. Mark the table input as done so every remaining - /// group is safe to emit. - /// - /// DrainingFinal - /// -> DrainingFinal - /// One remaining partial-state batch was yielded; repeat to continue - /// draining the table. - /// -> Done - /// All remaining groups were emitted. - /// - /// Done - /// -> (end) - /// ``` - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - loop { - let cur_state = self - .state - .take() - .expect("OrderedPartialAggregateStream state should not be None"); - - let next_state = match cur_state { - state @ OrderedPartialAggregateState::ReadingInput { .. } => { - self.handle_reading_input(cx, state) - } - state @ OrderedPartialAggregateState::DrainingFinal { .. } => { - self.handle_draining_final(state) - } - state @ OrderedPartialAggregateState::Done => { - let _ = self.reservation.try_resize(0); - self.state = Some(state); - return Poll::Ready(None); - } - }; - - match next_state { - ControlFlow::Continue(next_state) => { - self.state = Some(next_state); - continue; - } - ControlFlow::Break((poll, next_state)) => { - self.state = Some(next_state); - return poll; - } - } + timer.done(); + emitter.emit(batch).await; + timer = elapsed_compute.timer(); } - } -} -impl RecordBatchStream for OrderedPartialAggregateStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) + // was empty + Ok(None) } } From 46510030a0a08a6b92fca7efc21bffd689906829 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 30 Jul 2026 19:03:41 +0800 Subject: [PATCH 074/109] test: improve `round` sqllogictest coverage (#23973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change Codecov reports low test coverage for this function, so this PR adds more SLT tests for it. The tests were AI-generated, and I manually verified that the results are correct. See `Codecov` bot comment -> `☔ View full report in Codecov by Harness` -> `Indirect Changes` for the updated test coverage. ## What changes are included in this PR? ## Are these changes tested? ## Are there any user-facing changes? --- datafusion/sqllogictest/test_files/math.slt | 96 +++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index 668809632e476..55320c0261884 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -88,6 +88,102 @@ SELECT round(125.2345, -3), round(125.2345, -2), round(125.2345, -1), round(125. ---- 0 100 130 125 125 125.2 125.23 125.235 +# Round signed and unsigned integer scalar widths +query IIIIIIII +SELECT + round(arrow_cast('115', 'Int8'), -1), + round(arrow_cast('-115', 'Int16'), -1), + round(arrow_cast('115', 'Int32'), -1), + round(arrow_cast('-115', 'Int64'), -1), + round(arrow_cast('115', 'UInt8'), -1), + round(arrow_cast('115', 'UInt16'), -1), + round(arrow_cast('115', 'UInt32'), -1), + round(arrow_cast('115', 'UInt64'), -1); +---- +120 -120 120 -120 120 120 120 120 + +# Round signed and unsigned integer arrays, including null and oversized scales +query IIIIIIII +SELECT + round(arrow_cast(column1, 'Int8'), column2), + round(arrow_cast(column1, 'Int16'), column2), + round(arrow_cast(column1, 'Int32'), column2), + round(arrow_cast(column1, 'Int64'), column2), + round(arrow_cast(column1, 'UInt8'), column2), + round(arrow_cast(column1, 'UInt16'), column2), + round(arrow_cast(column1, 'UInt32'), column2), + round(arrow_cast(column1, 'UInt64'), column2) +FROM (VALUES ('115', -1), ('0', -1), (NULL, -20)) AS t(column1, column2); +---- +120 120 120 120 120 120 120 120 +0 0 0 0 0 0 0 0 +NULL NULL NULL NULL NULL NULL NULL NULL + +# Test columns without null +query I +SELECT + round(column1, column2) +FROM (VALUES (115, -20), (0, -1), (21, -1)) AS t(column1, column2); +---- +0 +0 +20 + +# Round all decimal widths as scalars +query RRRR +SELECT + round(arrow_cast('125.55', 'Decimal32(7,2)'), 1), + round(arrow_cast('-125.55', 'Decimal64(16,2)'), 1), + round(arrow_cast('125.55', 'Decimal128(30,2)'), 1), + round(arrow_cast('-125.55', 'Decimal256(40,2)'), 1); +---- +125.6 -125.6 125.6 -125.6 + +# Round all decimal widths as arrays with per-row decimal places +query RRRR +SELECT + round(arrow_cast(column1, 'Decimal32(7,2)'), column2), + round(arrow_cast(column1, 'Decimal64(16,2)'), column2), + round(arrow_cast(column1, 'Decimal128(30,2)'), column2), + round(arrow_cast(column1, 'Decimal256(40,2)'), column2) +FROM (VALUES ('125.55', 1), ('-125.55', 0), ('125.55', -1), (NULL, 1)) AS t(column1, column2); +---- +125.6 125.6 125.6 125.6 +-126 -126 -126 -126 +130 130 130 130 +NULL NULL NULL NULL + +# Float arrays with scalar and per-row decimal places +query RRRR +SELECT + round(arrow_cast(column1, 'Float32'), 1), + round(arrow_cast(column1, 'Float64'), 1), + round(arrow_cast(column1, 'Float32'), column2), + round(arrow_cast(column1, 'Float64'), column2) +FROM (VALUES ('125.55', 1), ('-125.55', 0), (NULL, -1)) AS t(column1, column2); +---- +125.6 125.6 125.6 125.6 +-125.6 -125.6 -126 -126 +NULL NULL NULL NULL + +# Null decimal places, invalid argument count/type, and out-of-range scale +query R +SELECT round(1.25, NULL); +---- +NULL + +query error DataFusion error: Error during planning: 'round' does not support zero arguments +SELECT round(); + +query error Error during planning: Internal error: Function 'round' failed to match any signature +SELECT round(1, 2, 3); + +query error Error during planning: Internal error: Function 'round' failed to match any signature +SELECT round('x'); + +query error round decimal_places 2147483648 is out of supported i32 range +SELECT round(1.25, 2147483648); + # atan2 query RRRRRRR SELECT atan2(2.0, 1.0), atan2(-2.0, 1.0), atan2(2.0, -1.0), atan2(-2.0, -1.0), atan2(NULL, 1.0), atan2(2.0, NULL), atan2(NULL, NULL); From 802d8d2dc11c4c8c39e875316524327f13313e24 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 30 Jul 2026 19:04:04 +0800 Subject: [PATCH 075/109] test: improve `gcd` sqllogictest coverage (#23972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change Codecov reports low test coverage for this function, so this PR adds more SLT tests for it. The tests were AI-generated, and I manually verified that the results are correct. See `Codecov` bot comment -> `☔ View full report in Codecov by Harness` -> `Indirect Changes` for the updated test coverage. ## What changes are included in this PR? ## Are these changes tested? ## Are there any user-facing changes? --- datafusion/sqllogictest/test_files/math.slt | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/datafusion/sqllogictest/test_files/math.slt b/datafusion/sqllogictest/test_files/math.slt index 55320c0261884..999709dfe77ea 100644 --- a/datafusion/sqllogictest/test_files/math.slt +++ b/datafusion/sqllogictest/test_files/math.slt @@ -1333,6 +1333,61 @@ SELECT gcd(arrow_cast(column1, 'Decimal128(38,0)'), arrow_cast(column2, 'Decimal 5 15 +# gcd with the remaining decimal array widths +query R +SELECT gcd(arrow_cast(column1, 'Decimal32(7,0)'), arrow_cast(column2, 'Decimal32(7,0)')) FROM (VALUES (15, 10), (15, 25), (15, 0)); +---- +5 +5 +15 + +query R +SELECT gcd(arrow_cast(column1, 'Decimal64(16,0)'), arrow_cast(column2, 'Decimal64(16,0)')) FROM (VALUES (15, 10), (15, 25), (15, 0)); +---- +5 +5 +15 + +query R +SELECT gcd(arrow_cast(column1, 'Decimal256(40,0)'), arrow_cast(column2, 'Decimal256(40,0)')) FROM (VALUES (15, 10), (15, 25), (15, 0)); +---- +5 +5 +15 + +# gcd array with zero, minimum, and null scalars +query I +SELECT gcd(column1, 0) FROM (VALUES (1), (2), (0), (NULL)); +---- +1 +2 +0 +NULL + +query I +SELECT gcd(column1, -9223372036854775808) FROM (VALUES (1), (2), (NULL)); +---- +1 +2 +NULL + +query I +SELECT gcd(column1, NULL) FROM (VALUES (1), (2), (NULL)); +---- +NULL +NULL +NULL + +# invalid argument count and type +query error gcd function requires 2 arguments, got 0 +SELECT gcd(); + +query error gcd function requires 2 arguments, got 3 +SELECT gcd(1, 2, 3); + +query error Unsupported argument types Utf8 and Utf8 for function gcd +SELECT gcd('x', 'y'); + # gcd array and scalar with nulls in the array query I From cc1326a6098dc050b334a28fc5b6c9754102a8a2 Mon Sep 17 00:00:00 2001 From: discord9 Date: Thu, 30 Jul 2026 19:05:29 +0800 Subject: [PATCH 076/109] fix: preserve dictionary-value nulls in scalar regex operators (#23966) ## Which issue does this PR close? - Closes #23963. ## Rationale for this change The scalar regex fast path for Dictionary arrays evaluates each dictionary value once and gathers the resulting booleans with `BooleanArray::take_iter`. That gather preserves null keys but drops null validity from the evaluated dictionary values. A key referencing a null dictionary value therefore becomes valid `false`, and negated regex operators can incorrectly turn it into `true`. This is distinct from #23722, which updates the array-pattern `regex_match_dyn` path. This PR fixes null propagation in `regex_match_dyn_scalar`. ## What changes are included in this PR? - Replace the validity-dropping dictionary gather with Arrow `take`, preserving both key validity and evaluated child validity. - Add a physical-expression regression test for `RegexMatch`, `RegexIMatch`, `RegexNotMatch`, and `RegexNotIMatch`. The test covers a null key and a key referencing a null dictionary value, and compares the Dictionary result with the equivalent cast-to-`Utf8` evaluation. ## Are these changes tested? Yes. - Focused regression: 1 passed. - `datafusion-physical-expr`: 1,649 passed, including the new regression. - Remaining workspace: 7,928 passed, 0 failed, 7 ignored; sqllogictest completed 499/499 files. - `datafusion` library: 461 passed. - `datafusion` core integration: 1,057 passed; four local memory-validation runners exceeded their host RSS thresholds and were isolated as environment-sensitive, unrelated to this patch. - `datafusion-cli`: 106 passed. - Formatting, workspace clippy with warnings denied, TOML formatting, license headers, changed-file typo checks, and `git diff --check` passed. ## Are there any user-facing changes? Dictionary-encoded string inputs now preserve SQL null semantics for scalar regex operators. There are no public API changes. --------- Signed-off-by: discord9 --- .../physical-expr/src/expressions/binary.rs | 72 +++++++++++++++++++ .../src/expressions/binary/kernels.rs | 22 +++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 8b71b3ec409c7..170c1a4d02700 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -3353,6 +3353,78 @@ mod tests { Ok(()) } + #[test] + fn regex_scalar_with_dictionary_nulls() -> Result<()> { + let dictionary_values = Arc::new(StringArray::from(vec![ + Some("abc"), + None, + Some("ABC"), + Some("def"), + ])); + let keys = UInt32Array::from(vec![Some(0), None, Some(1), Some(2), Some(3)]); + let dictionary = + Arc::new(DictionaryArray::try_new(keys, dictionary_values)?) as ArrayRef; + let utf8 = cast(&dictionary, &DataType::Utf8)?; + let pattern = ScalarValue::Utf8(Some("^abc$".to_string())); + let dictionary_schema = Arc::new(Schema::new(vec![Field::new( + "a", + dictionary.data_type().clone(), + true, + )])); + let utf8_schema = + Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)])); + + let evaluate = + |schema: &SchemaRef, array: &ArrayRef, op: Operator| -> Result { + let expr = binary(col("a", schema)?, op, lit(pattern.clone()), schema)?; + let batch = + RecordBatch::try_new(Arc::clone(schema), vec![Arc::clone(array)])?; + Ok(expr + .evaluate(&batch)? + .into_array(batch.num_rows()) + .expect("Failed to convert to array")) + }; + + for (op, expected) in [ + ( + Operator::RegexMatch, + BooleanArray::from(vec![ + Some(true), + None, + None, + Some(false), + Some(false), + ]), + ), + ( + Operator::RegexIMatch, + BooleanArray::from(vec![Some(true), None, None, Some(true), Some(false)]), + ), + ( + Operator::RegexNotMatch, + BooleanArray::from(vec![Some(false), None, None, Some(true), Some(true)]), + ), + ( + Operator::RegexNotIMatch, + BooleanArray::from(vec![ + Some(false), + None, + None, + Some(false), + Some(true), + ]), + ), + ] { + let dictionary_result = evaluate(&dictionary_schema, &dictionary, op)?; + let utf8_result = evaluate(&utf8_schema, &utf8, op)?; + + assert_eq!(dictionary_result.as_ref(), &expected); + assert_eq!(&dictionary_result, &utf8_result); + } + + Ok(()) + } + #[test] fn regex_mismatched_array_types_error() -> Result<()> { // The analyzer coerces both operands of a regex operator to a common diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index fca824c14bee0..a123fba1f9da2 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -270,7 +270,8 @@ pub(crate) fn regex_match_dyn_scalar( regexp_is_match_flag_scalar!(left, right, LargeStringArray, not_match, flag) } DataType::Dictionary(_, _) => { - let values = left.as_any_dictionary().values(); + let dictionary = left.as_any_dictionary(); + let values = dictionary.values(); match values.data_type() { DataType::Utf8 => regexp_is_match_flag_scalar!(values, right, StringArray, not_match, flag), @@ -280,16 +281,15 @@ pub(crate) fn regex_match_dyn_scalar( "Data type {} not supported as a dictionary value type for operation 'regex_match_dyn_scalar' on string array", other ), - }.map( - // downcast_dictionary_array duplicates code per possible key type, so we aim to do all prep work before - |evaluated_values| downcast_dictionary_array! { - left => { - let unpacked_dict = evaluated_values.take_iter(left.keys().iter().map(|opt| opt.map(|v| v as _))).collect::(); - Arc::new(unpacked_dict) as ArrayRef - }, - _ => unreachable!(), - } - ) + } + .and_then(|evaluated_values| { + // Expand back to rows while preserving nulls from both keys and values. + Ok(arrow::compute::take( + evaluated_values.as_ref(), + dictionary.keys(), + None, + )?) + }) } other => internal_err!( "Data type {} not supported for operation 'regex_match_dyn_scalar' on string array", From 0bcbe40024f9405abfc60b945206997a93d1e3e4 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 30 Jul 2026 19:06:18 +0800 Subject: [PATCH 077/109] test: improve `rpad` sqllogictest coverage (#23968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change Codecov reports low test coverage for this function, so this PR adds more SLT tests for it. The tests were AI-generated, and I manually verified that the results are correct. See `Codecov` bot comment -> `☔ View full report in Codecov by Harness` -> `Indirect Changes` for the updated test coverage. ## What changes are included in this PR? ## Are these changes tested? ## Are there any user-facing changes? --- .../test_files/string/string_literal.slt | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt index c175f52a35f99..6962a6d337827 100644 --- a/datafusion/sqllogictest/test_files/string/string_literal.slt +++ b/datafusion/sqllogictest/test_files/string/string_literal.slt @@ -664,6 +664,64 @@ SELECT rpad('x', 5, 'e' || chr(769)) = 'x' || 'e' || chr(769) || 'e' || chr(769) ---- true 5 +# rpad with string, length, and fill arrays in every string width +query BBB +SELECT + rpad(arrow_cast(column1, 'Utf8'), column2, arrow_cast(column3, 'Utf8')) IS NOT DISTINCT FROM column4, + rpad(arrow_cast(column1, 'LargeUtf8'), column2, arrow_cast(column3, 'LargeUtf8')) IS NOT DISTINCT FROM column4, + rpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4 +FROM (VALUES + ('hi', 5, 'xy', 'hixyx'), + ('abcdef', 3, 'z', 'abc'), + ('é', 4, '好', 'é好好好'), + ('hi', 5, '', 'hi'), + (NULL, 5, 'x', NULL), + ('hi', NULL, 'x', NULL), + ('hi', 5, NULL, NULL) +) AS t(column1, column2, column3, column4); +---- +true true true +true true true +true true true +true true true +true true true +true true true +true true true + +# rpad array path with the default fill +query BBB +SELECT + rpad(arrow_cast(column1, 'Utf8'), column2) IS NOT DISTINCT FROM column3, + rpad(arrow_cast(column1, 'LargeUtf8'), column2) IS NOT DISTINCT FROM column3, + rpad(arrow_cast(column1, 'Utf8View'), column2) IS NOT DISTINCT FROM column3 +FROM (VALUES ('hi', 5, 'hi '), ('abcdef', 3, 'abc'), (NULL, 5, NULL)) AS t(column1, column2, column3); +---- +true true true +true true true +true true true + +# a large scalar target length skips the scalar fast path +query I +SELECT character_length(rpad('x', 16385, 'a')); +---- +16385 + +# invalid argument count/type and excessive target length +query error 'rpad' does not support zero arguments +SELECT rpad(); + +query error Failed to coerce arguments to satisfy a call to 'rpad' function: coercion from Utf8 to the signature +SELECT rpad('x'); + +query error Failed to coerce arguments to satisfy a call to 'rpad' function: coercion from Utf8, Int64, Utf8, Utf8 to the signature +SELECT rpad('x', 2, 'y', 'z'); + +query error Failed to coerce arguments to satisfy a call to 'rpad' function: coercion from Utf8, Utf8 to the signature +SELECT rpad('x', 'bad'); + +query error rpad requested length 2147483648 too large +SELECT rpad('x', 2147483648, 'y'); + query I SELECT char_length('') ---- From 64bc392b52d7ab5ca1407040f06d159325af51cc Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 30 Jul 2026 19:16:38 +0800 Subject: [PATCH 078/109] test: improve `lpad` sqllogictest coverage (#23969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change Codecov reports low test coverage for this function, so this PR adds more SLT tests for it. The tests were AI-generated, and I manually verified that the results are correct. See `Codecov` bot comment -> `☔ View full report in Codecov by Harness` -> `Indirect Changes` for the updated test coverage. ## What changes are included in this PR? ## Are these changes tested? ## Are there any user-facing changes? --- .../test_files/string/string_literal.slt | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt index 6962a6d337827..06d8bf2a4c99d 100644 --- a/datafusion/sqllogictest/test_files/string/string_literal.slt +++ b/datafusion/sqllogictest/test_files/string/string_literal.slt @@ -341,6 +341,64 @@ SELECT lpad('x', 5, 'e' || chr(769)) = 'e' || chr(769) || 'e' || chr(769) || 'x' ---- true 5 +# lpad with string, length, and fill arrays in every string width +query BBB +SELECT + lpad(arrow_cast(column1, 'Utf8'), column2, arrow_cast(column3, 'Utf8')) IS NOT DISTINCT FROM column4, + lpad(arrow_cast(column1, 'LargeUtf8'), column2, arrow_cast(column3, 'LargeUtf8')) IS NOT DISTINCT FROM column4, + lpad(arrow_cast(column1, 'Utf8View'), column2, arrow_cast(column3, 'Utf8View')) IS NOT DISTINCT FROM column4 +FROM (VALUES + ('hi', 5, 'xy', 'xyxhi'), + ('abcdef', 3, 'z', 'abc'), + ('é', 4, '好', '好好好é'), + ('hi', 5, '', 'hi'), + (NULL, 5, 'x', NULL), + ('hi', NULL, 'x', NULL), + ('hi', 5, NULL, NULL) +) AS t(column1, column2, column3, column4); +---- +true true true +true true true +true true true +true true true +true true true +true true true +true true true + +# lpad array path with the default fill +query BBB +SELECT + lpad(arrow_cast(column1, 'Utf8'), column2) IS NOT DISTINCT FROM column3, + lpad(arrow_cast(column1, 'LargeUtf8'), column2) IS NOT DISTINCT FROM column3, + lpad(arrow_cast(column1, 'Utf8View'), column2) IS NOT DISTINCT FROM column3 +FROM (VALUES ('hi', 5, ' hi'), ('abcdef', 3, 'abc'), (NULL, 5, NULL)) AS t(column1, column2, column3); +---- +true true true +true true true +true true true + +# a large scalar target length skips the scalar fast path +query I +SELECT character_length(lpad('x', 16385, 'a')); +---- +16385 + +# invalid argument count/type and excessive target length +query error 'lpad' does not support zero arguments +SELECT lpad(); + +query error Failed to coerce arguments to satisfy a call to 'lpad' function: coercion from Utf8 to the signature +SELECT lpad('x'); + +query error Failed to coerce arguments to satisfy a call to 'lpad' function: coercion from Utf8, Int64, Utf8, Utf8 to the signature +SELECT lpad('x', 2, 'y', 'z'); + +query error Failed to coerce arguments to satisfy a call to 'lpad' function: coercion from Utf8, Utf8 to the signature +SELECT lpad('x', 'bad'); + +query error lpad requested length 2147483648 too large +SELECT lpad('x', 2147483648, 'y'); + query T SELECT regexp_replace('foobar', 'bar', 'xx', 'gi') ---- From eacdf7554a88e1c842921d531a82d445eff6591f Mon Sep 17 00:00:00 2001 From: Shehab Ali <89369967+shehab-ali@users.noreply.github.com> Date: Thu, 30 Jul 2026 07:21:27 -0400 Subject: [PATCH 079/109] Add benchmarks for hashjoin candidate equality filtering (#23980) ## Which issue does this PR close? - Part of #23237. ## Rationale for this change Issue #23237 identifies hash join candidate equality filtering as a potential performance bottleneck. Before changing that implementation, we need stable benchmarks that reproduce the relevant workloads and let us measure the effect of a proposed optimization independently. The existing hash join benchmark suite covers high-fanout joins, but it does not isolate these two cases: 1. A single hot hash bucket in which all build-side rows use the same long string key. 2. A skewed composite-key join in which candidate pairs must be checked across multiple key columns. This PR intentionally adds only the benchmarks. The implementation change for `equal_rows_arr` will be submitted separately so that the benchmark workloads can be reviewed and established independently of the optimization. ## What changes are included in this PR? This PR adds two hash join benchmarks: - **Q24: single-hot-bucket long string-key join** - Creates 3,000 build-side rows with the same long string key. - Selects matching probe-side rows that use that same key. - Causes every selected probe row to fan out across the entire build-side bucket. - Uses `count(*)` so the benchmark emphasizes hash match and candidate equality processing without materializing and retaining the full joined output. - **Q25: skewed composite-key join** - Uses an integer key and a long string key. - Distributes the build-side integer key over 256 values while all selected probe rows use one value. - Produces a high-fanout set of candidate matches for one integer-key value. - Requires the second string-key column to be checked as part of composite join-key equality. - Uses `count(*)` to focus the measurement on the join match path. Both benchmarks: - Assert that the input data is available. - Assert that the physical plan contains `HashJoinExec`. - Use the no-statistics configuration and force Partitioned hash join mode. - Are registered in the `hj` benchmark runner with descriptive build/probe metadata. - Can be used to compare the existing implementation against the follow-up optimization for #23237. ## Are these changes tested? No hash join implementation is changed by this PR. ## Are there any user-facing changes? No. --- .../hj/benchmarks/q24.benchmark | 31 +++++++++++++ .../hj/benchmarks/q25.benchmark | 33 +++++++++++++ benchmarks/src/hj.rs | 46 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 benchmarks/sql_benchmarks/hj/benchmarks/q24.benchmark create mode 100644 benchmarks/sql_benchmarks/hj/benchmarks/q25.benchmark diff --git a/benchmarks/sql_benchmarks/hj/benchmarks/q24.benchmark b/benchmarks/sql_benchmarks/hj/benchmarks/q24.benchmark new file mode 100644 index 0000000000000..2ea60f0f87009 --- /dev/null +++ b/benchmarks/sql_benchmarks/hj/benchmarks/q24.benchmark @@ -0,0 +1,31 @@ +name Q24 +group hj + +init sql_benchmarks/hj/init/set_config_no_stats.sql + +load sql_benchmarks/hj/init/load.sql + +assert I +SELECT count(*) > 0 FROM lineitem +---- +true + +expect_plan HashJoinExec + +run +-- Q24: single-hot-bucket long string-key inner join. +-- Build rows all share one long string key, so each matching probe row fans +-- out to the whole build side. count(*) focuses the benchmark on hash match +-- and equality filtering without buffering joined rows. +-- Thresholds zeroed to force Partitioned mode (simulates absent row-count stats). +SELECT count(*) +FROM ( + SELECT 'single_hot_bucket_string_join_key' as k + FROM supplier + WHERE s_suppkey <= 3000 +) s +JOIN ( + SELECT 'single_hot_bucket_string_join_key' as k + FROM lineitem + WHERE l_orderkey % 3000 = 0 +) l ON s.k = l.k; diff --git a/benchmarks/sql_benchmarks/hj/benchmarks/q25.benchmark b/benchmarks/sql_benchmarks/hj/benchmarks/q25.benchmark new file mode 100644 index 0000000000000..b29d6b959a853 --- /dev/null +++ b/benchmarks/sql_benchmarks/hj/benchmarks/q25.benchmark @@ -0,0 +1,33 @@ +name Q25 +group hj + +init sql_benchmarks/hj/init/set_config_no_stats.sql + +load sql_benchmarks/hj/init/load.sql + +assert I +SELECT count(*) > 0 FROM lineitem +---- +true + +expect_plan HashJoinExec + +run +-- Q25: skewed high-fanout multi-column string-key inner join. +-- This tracks candidate-pair filtering for composite keys: the first key is +-- skewed and the second long string key must also be checked before emitting +-- each match. count(*) isolates the match path. +-- Thresholds zeroed to force Partitioned mode (simulates absent row-count stats). +SELECT count(*) +FROM ( + SELECT CAST((s_suppkey % 256) + 1 AS INT) as k1, + 'multi_column_high_fanout_key' as k2 + FROM supplier + WHERE s_suppkey <= 20000 +) s +JOIN ( + SELECT CAST(1 AS INT) as k1, + 'multi_column_high_fanout_key' as k2 + FROM lineitem + WHERE l_orderkey % 250 = 0 +) l ON s.k1 = l.k1 AND s.k2 = l.k2; diff --git a/benchmarks/src/hj.rs b/benchmarks/src/hj.rs index 7d33bc3aa9e50..9a91e2714ac86 100644 --- a/benchmarks/src/hj.rs +++ b/benchmarks/src/hj.rs @@ -472,6 +472,52 @@ const HASH_QUERIES: &[HashJoinQuery] = &[ probe_size: "2.3M_long_keys_count", isolate_partitioned_join: true, }, + // Q24: single-hot-bucket long string-key inner join. + // Build rows all share one long string key, so each matching probe row fans + // out to the whole build side. The output is counted to focus on the hash + // match/equality path without buffering the joined rows. + HashJoinQuery { + sql: r###"SELECT count(*) + FROM ( + SELECT 'single_hot_bucket_string_join_key' as k + FROM supplier + WHERE s_suppkey <= 3000 + ) s + JOIN ( + SELECT 'single_hot_bucket_string_join_key' as k + FROM lineitem + WHERE l_orderkey % 3000 = 0 + ) l ON s.k = l.k"###, + density: 1.0, + prob_hit: 1.0, + build_size: "3K_(single_hot_bucket)", + probe_size: "20K_long_keys_count", + isolate_partitioned_join: true, + }, + // Q25: skewed high-fanout multi-column string-key inner join. + // This tracks the same candidate-pair filtering path for composite join + // keys, where the first key is skewed and the second long string key must + // also be checked before emitting each match. + HashJoinQuery { + sql: r###"SELECT count(*) + FROM ( + SELECT CAST((s_suppkey % 256) + 1 AS INT) as k1, + 'multi_column_high_fanout_key' as k2 + FROM supplier + WHERE s_suppkey <= 20000 + ) s + JOIN ( + SELECT CAST(1 AS INT) as k1, + 'multi_column_high_fanout_key' as k2 + FROM lineitem + WHERE l_orderkey % 250 = 0 + ) l ON s.k1 = l.k1 AND s.k2 = l.k2"###, + density: 1.0, + prob_hit: 1.0, + build_size: "20K_(fanout~78_multi_key)", + probe_size: "240K_multi_key_count", + isolate_partitioned_join: true, + }, ]; impl RunOpt { From 455a3add52d051a20df9960a726ee9acb98528a3 Mon Sep 17 00:00:00 2001 From: Vikrant Mehta <95346283+vikrantmehta123@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:07:04 +0530 Subject: [PATCH 080/109] docs: Fixes incorrect type name in `UserDefinedLogicalNode` comment (#23992) ## Which issue does this PR close? - N/A. This fixes a trivial doc comment. No issue filed. ## Rationale for this change The doc comment in `datafusion/expr/src/logical_plan/extension.rs` incorrectly said it "derives UserDefinedLogicalNode to `UserDefinedLogicalNode`" ( It named the same type twice ). This PR fixes this trivial documentation. ## What changes are included in this PR? A single line containing the documentation fix, which corrects the source trait name from `UserDefinedLogicalNode` to `UserDefinedLogicalNodeCore`. ## Are these changes tested? No tests needed as this is a comment-only change. ## Are there any user-facing changes? No. --- datafusion/expr/src/logical_plan/extension.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/expr/src/logical_plan/extension.rs b/datafusion/expr/src/logical_plan/extension.rs index fe324d40fd952..e1ee273968676 100644 --- a/datafusion/expr/src/logical_plan/extension.rs +++ b/datafusion/expr/src/logical_plan/extension.rs @@ -314,7 +314,7 @@ pub trait UserDefinedLogicalNodeCore: } } -/// Automatically derive UserDefinedLogicalNode to `UserDefinedLogicalNode` +/// Automatically derive `UserDefinedLogicalNode` from `UserDefinedLogicalNodeCore` /// to avoid boiler plate for implementing `as_any`, `Hash`, `PartialEq` and `PartialOrd`. impl UserDefinedLogicalNode for T { fn as_any(&self) -> &dyn Any { From 44aae9486590971280175ad98e2325d0b4ca523b Mon Sep 17 00:00:00 2001 From: linfeng <33561138+lyne7-sc@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:19:32 +0800 Subject: [PATCH 081/109] perf: precompile formats in to_time (#23964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #. ## Rationale for this change `to_time` previously parsed the Chrono format strings for every non-null input value. For array inputs, this repeated the same format parsing for each row in a batch. This PR precompiles the format strings once per udf invocation and reuses the compiled format items while parsing each value. ## What changes are included in this PR? - Precompile and reuse the to_time format strings. - Add tests for multiple formats and NULL values. ## Are these changes tested? Yes. Existing tests pass, and new slt coverage was added for multiple formats and NULL values. ## Are there any user-facing changes? No. This is an internal performance optimization. ## Benchmarks ``` group main optimize-to-time ----- ---------- ---------------- to_time_10pct_nulls_100k 1.83 7.3±0.34ms ? ?/sec 1.00 4.0±0.47ms ? ?/sec to_time_no_nulls_100k 1.82 7.9±0.33ms ? ?/sec 1.00 4.4±0.42ms ? ?/sec ``` --- datafusion/functions/src/datetime/to_time.rs | 31 ++++++++++++++++--- .../test_files/datetime/timestamps.slt | 22 +++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/datafusion/functions/src/datetime/to_time.rs b/datafusion/functions/src/datetime/to_time.rs index 94aa49fbbad2f..45664e9416f04 100644 --- a/datafusion/functions/src/datetime/to_time.rs +++ b/datafusion/functions/src/datetime/to_time.rs @@ -22,7 +22,7 @@ use arrow::array::types::Time64NanosecondType; use arrow::array::{Array, PrimitiveArray, StringArrayType}; use arrow::datatypes::DataType; use arrow::datatypes::DataType::*; -use chrono::NaiveTime; +use chrono::format::{Item, Parsed, StrftimeItems, parse}; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -141,6 +141,7 @@ impl ScalarUDFImpl for ToTimeFunc { /// Convert string arguments to time (standalone function, not a method on ToTimeFunc) fn string_to_time(args: &[ColumnarValue]) -> Result { let formats = collect_formats(args)?; + let formats = compile_formats(&formats); match &args[0] { ColumnarValue::Scalar(ScalarValue::Utf8(s)) @@ -207,10 +208,25 @@ fn timestamp_to_time(arg: &ColumnarValue) -> Result { arg.cast_to(&Time64(arrow::datatypes::TimeUnit::Nanosecond), None) } +struct CompiledTimeFormat<'a> { + source: &'a str, + items: Vec>, +} + +fn compile_formats<'a>(formats: &[&'a str]) -> Vec> { + formats + .iter() + .map(|source| CompiledTimeFormat { + source, + items: StrftimeItems::new(source).collect(), + }) + .collect() +} + /// Parse time array using the provided formats fn parse_time_array<'a, A: StringArrayType<'a>>( array: &A, - formats: &[&str], + formats: &[CompiledTimeFormat<'_>], ) -> Result> { let mut values = Vec::with_capacity(array.len()); for i in 0..array.len() { @@ -224,10 +240,12 @@ fn parse_time_array<'a, A: StringArrayType<'a>>( } /// Parse time string using provided formats -fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result { +fn parse_time_with_formats(s: &str, formats: &[CompiledTimeFormat<'_>]) -> Result { for format in formats { - if let Ok(time) = NaiveTime::parse_from_str(s, format) { - // Use Arrow's time_to_time64ns function instead of custom implementation + let mut parsed = Parsed::new(); + if parse(&mut parsed, s, format.items.iter()).is_ok() + && let Ok(time) = parsed.to_naive_time() + { return Ok(time_to_time64ns(time)); } } @@ -235,5 +253,8 @@ fn parse_time_with_formats(s: &str, formats: &[&str]) -> Result { "Error parsing '{}' as time. Tried formats: {:?}", s, formats + .iter() + .map(|format| format.source) + .collect::>() ) } diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 002111d3f252a..d73bc6eb06de8 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -3529,6 +3529,28 @@ select to_time(time_str) from time_strings; statement ok drop table time_strings; +# Table input with multiple formats +# `%Q` is intentionally invalid; subsequent formats should still be tried. +query D rowsort +select to_time( + time_str, + '%Q', + '%H:%M:%S', + '%H-%M-%S', + '%H/%M/%S' +) from ( + values + ('12:30:45'), + ('14-25-30'), + ('09/05/01'), + (NULL) +) as formatted_time_strings(time_str); +---- +09:05:01 +12:30:45 +14:25:30 +NULL + # Error cases query error Error parsing 'not_a_time' as time From 6a0e7715498e9e92fa0a879491302088e7f6ba3a Mon Sep 17 00:00:00 2001 From: Floze <88098863+floze-the-genius@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:34:13 +0400 Subject: [PATCH 082/109] Optimize Spark hex null handling (#23688) ## Which issue does this PR close? - Closes #23674. ## Rationale for this change Spark hex encoding preserves input validity exactly, but the byte-array path rebuilt the null bitmap row by row and always iterated through `Option` values. The input arrays already expose both the original `NullBuffer` and an all-valid fast path. ## What changes are included in this PR? - pass the byte array accessor into `hex_encode_bytes` so the output can clone and reuse the input `NullBuffer` - split nullable and no-null loops, avoiding per-row validity checks when the input has no null buffer - preserve exact offset, overflow, casing, dictionary, and null semantics - add a sliced-array regression that verifies semantic equality and bitmap pointer reuse - add no-null output coverage and dedicated UTF-8/binary Criterion cases ## Are these changes tested? Yes. - `cargo test -p datafusion-spark function::math::hex --lib`: 10 passed - `cargo fmt --all -- --check`: passed - `cargo clippy -p datafusion-spark --all-targets --all-features --no-deps -- -D warnings`: passed - `git diff --check`: passed - benchmark target compiled with `--features core` The workspace dependency clippy invocation without `--no-deps` reaches an unrelated existing `dead_code` warning in `datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs:464`; the changed crate is clean. ### Benchmark Command (same warmed build and profile for upstream `main` and this branch): ```bash CARGO_BUILD_JOBS=2 \ CARGO_PROFILE_BENCH_LTO=false \ CARGO_PROFILE_BENCH_CODEGEN_UNITS=16 \ cargo bench -p datafusion-spark --features core --bench hex -- \ 'hex_(utf8|binary)_no_nulls' ``` Mean times from the immediate upstream-then-branch sequence: | case | upstream main | this PR | change | | --- | ---: | ---: | ---: | | UTF-8, 1,024 rows | 79.895 us | 59.181 us | -25.9% | | UTF-8, 4,096 rows | 317.64 us | 244.88 us | -22.9% | | UTF-8, 8,192 rows | 509.64 us | 411.27 us | -19.3% | | Binary, 1,024 rows | 86.135 us | 40.600 us | -52.9% | | Binary, 4,096 rows | 346.16 us | 253.13 us | -26.9% | | Binary, 8,192 rows | 635.98 us | 424.78 us | -33.2% | ## Are there any user-facing changes? No API or behavior changes. This is an internal performance optimization for Spark-compatible hex expressions. --- datafusion/spark/benches/hex.rs | 10 ++ datafusion/spark/src/function/math/hex.rs | 194 ++++++++++++++-------- 2 files changed, 138 insertions(+), 66 deletions(-) diff --git a/datafusion/spark/benches/hex.rs b/datafusion/spark/benches/hex.rs index 9785371cc5827..38a59cb944e50 100644 --- a/datafusion/spark/benches/hex.rs +++ b/datafusion/spark/benches/hex.rs @@ -135,11 +135,21 @@ fn criterion_benchmark(c: &mut Criterion) { run_benchmark(c, "hex_utf8", size, Arc::new(data)); } + for &size in &sizes { + let data = generate_utf8_data(size, 0.0); + run_benchmark(c, "hex_utf8_no_nulls", size, Arc::new(data)); + } + for &size in &sizes { let data = generate_binary_data(size, null_density); run_benchmark(c, "hex_binary", size, Arc::new(data)); } + for &size in &sizes { + let data = generate_binary_data(size, 0.0); + run_benchmark(c, "hex_binary_no_nulls", size, Arc::new(data)); + } + for &size in &sizes { let data = generate_int64_dict_data(size, null_density); run_benchmark(c, "hex_int64_dict", size, Arc::new(data)); diff --git a/datafusion/spark/src/function/math/hex.rs b/datafusion/spark/src/function/math/hex.rs index 1f505fda21b6f..d098169cf188d 100644 --- a/datafusion/spark/src/function/math/hex.rs +++ b/datafusion/spark/src/function/math/hex.rs @@ -18,7 +18,7 @@ use std::str::from_utf8_unchecked; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, NullBufferBuilder, StringArray, StringBuilder}; +use arrow::array::{Array, ArrayAccessor, ArrayRef, StringArray, StringBuilder}; use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::datatypes::DataType; use arrow::{ @@ -111,21 +111,40 @@ impl ScalarUDFImpl for SparkHex { } } +#[inline] +fn append_hex_bytes( + values: &mut Vec, + bytes: &[u8], + case: HexCase, +) -> Result { + let additional = bytes + .len() + .checked_mul(2) + .ok_or_else(|| exec_datafusion_err!("hex output size overflow"))?; + values.try_reserve(additional).map_err(|e| { + exec_datafusion_err!("failed to reserve {additional} bytes for hex output: {e}") + })?; + encode_bytes_into(bytes, case, values); + i32::try_from(values.len()) + .map_err(|_| exec_datafusion_err!("hex output exceeds i32 offset range")) +} + /// Generic hex encoding for byte array types -fn hex_encode_bytes<'a, I, T>( - iter: I, +fn hex_encode_bytes<'a, A, T>( + array: &A, lowercase: bool, - len: usize, ) -> Result where - I: Iterator>, - T: AsRef<[u8]> + 'a, + A: ArrayAccessor, + T: AsRef<[u8]> + ?Sized + 'a, { let case = if lowercase { HexCase::Lower } else { HexCase::Upper }; + let len = array.len(); + let nulls = array.nulls().cloned(); // Write hex digits directly into one growing value buffer, tracking offsets // ourselves. Each input byte becomes exactly two output bytes, so there is @@ -134,30 +153,25 @@ where let mut values: Vec = Vec::with_capacity(len * 64); let mut offsets: Vec = Vec::with_capacity(len + 1); offsets.push(0); - let mut nulls = NullBufferBuilder::new(len); - for v in iter { - if let Some(b) = v { - let bytes = b.as_ref(); - let additional = bytes - .len() - .checked_mul(2) - .ok_or_else(|| exec_datafusion_err!("hex output size overflow"))?; - values.try_reserve(additional).map_err(|e| { - exec_datafusion_err!( - "failed to reserve {additional} bytes for hex output: {e}" - ) - })?; - encode_bytes_into(bytes, case, &mut values); - nulls.append_non_null(); - } else { - nulls.append_null(); + if let Some(ref nulls) = nulls { + for i in 0..len { + if nulls.is_valid(i) { + // SAFETY: `i` is in bounds and the validity buffer marks it valid. + let bytes = unsafe { array.value_unchecked(i) }.as_ref(); + offsets.push(append_hex_bytes(&mut values, bytes, case)?); + } else { + offsets.push(i32::try_from(values.len()).map_err(|_| { + exec_datafusion_err!("hex output exceeds i32 offset range") + })?); + } + } + } else { + for i in 0..len { + // SAFETY: `i` is in bounds and no null buffer means every value is valid. + let bytes = unsafe { array.value_unchecked(i) }.as_ref(); + offsets.push(append_hex_bytes(&mut values, bytes, case)?); } - offsets.push( - i32::try_from(values.len()).map_err(|_| { - exec_datafusion_err!("hex output exceeds i32 offset range") - })?, - ); } // SAFETY: the value buffer contains only ASCII hex digits (valid UTF-8) and @@ -168,7 +182,7 @@ where StringArray::new_unchecked( OffsetBuffer::new(offsets.into()), Buffer::from_vec(values), - nulls.finish(), + nulls, ) }; Ok(Arc::new(array)) @@ -227,51 +241,27 @@ pub fn compute_hex( } DataType::Utf8 => { let array = as_string_array(array); - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::Utf8View => { let array = as_string_view_array(array)?; - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::LargeUtf8 => { let array = as_largestring_array(array); - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::Binary => { let array = as_binary_array(array)?; - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::LargeBinary => { let array = as_large_binary_array(array)?; - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::FixedSizeBinary(_) => { let array = as_fixed_size_binary_array(array)?; - Ok(ColumnarValue::Array(hex_encode_bytes( - array.iter(), - lowercase, - array.len(), - )?)) + Ok(ColumnarValue::Array(hex_encode_bytes(&array, lowercase)?)) } DataType::Dictionary(key_type, _) => { if **key_type != DataType::Int32 { @@ -291,27 +281,27 @@ pub fn compute_hex( } DataType::Utf8 => { let arr = as_string_array(dict_values); - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } DataType::LargeUtf8 => { let arr = as_largestring_array(dict_values); - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } DataType::Utf8View => { let arr = as_string_view_array(dict_values)?; - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } DataType::Binary => { let arr = as_binary_array(dict_values)?; - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } DataType::LargeBinary => { let arr = as_large_binary_array(dict_values)?; - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } DataType::FixedSizeBinary(_) => { let arr = as_fixed_size_binary_array(dict_values)?; - hex_encode_bytes(arr.iter(), lowercase, arr.len())? + hex_encode_bytes(&arr, lowercase)? } _ => { return exec_err!( @@ -465,7 +455,8 @@ mod test { // is reachable only via `spark_sha2_hex`, which has no in-workspace // caller, so it otherwise has no coverage. Drive it directly here. let input = StringArray::from(vec![Some("hi"), Some("bye"), None, Some("rust")]); - let result = super::hex_encode_bytes(input.iter(), true, input.len()).unwrap(); + let input_ref = &input; + let result = super::hex_encode_bytes(&input_ref, true).unwrap(); let result = as_string_array(&result); let expected = @@ -495,6 +486,56 @@ mod test { assert_eq!(strings.value(0), expected); } + #[test] + fn test_spark_hex_binary_no_nulls() { + let input = BinaryArray::from(vec![ + b"".as_slice(), + b"\x00\x7f\x80\xff".as_slice(), + b"DataFusion".as_slice(), + ]); + + let result = super::spark_hex(&[ColumnarValue::Array(Arc::new(input))]).unwrap(); + let array = match result { + ColumnarValue::Array(array) => array, + _ => panic!("Expected array"), + }; + let strings = as_string_array(&array); + + assert_eq!(strings.nulls(), None); + assert_eq!( + strings, + &StringArray::from(vec!["", "007F80FF", "44617461467573696F6E"]) + ); + } + + #[test] + fn test_spark_hex_binary_reuses_input_nulls() { + let input = BinaryArray::from(vec![ + Some(b"skip".as_slice()), + None, + Some(b"\x00\xff".as_slice()), + Some(b"hex".as_slice()), + None, + ]) + .slice(1, 4); + let input_nulls = input.nulls().unwrap().clone(); + + let result = super::spark_hex(&[ColumnarValue::Array(Arc::new(input))]).unwrap(); + let array = match result { + ColumnarValue::Array(array) => array, + _ => panic!("Expected array"), + }; + let strings = as_string_array(&array); + let output_nulls = strings.nulls().unwrap(); + + assert_eq!(output_nulls, &input_nulls); + assert!(output_nulls.inner().ptr_eq(input_nulls.inner())); + assert_eq!( + strings, + &StringArray::from(vec![None, Some("00FF"), Some("686578"), None]) + ); + } + #[test] fn test_spark_hex_int64() { let int_array = Int64Array::from(vec![Some(1), Some(2), None, Some(3)]); @@ -540,4 +581,25 @@ mod test { assert_eq!(&expected, result); } + + #[test] + fn test_dict_binary_values_null() { + let keys = Int32Array::from(vec![Some(0), None, Some(1)]); + let vals = BinaryArray::from(vec![Some(b"hi".as_slice()), None]); + // [b"hi", null, null] + let dict = DictionaryArray::new(keys, Arc::new(vals)); + + let result = super::spark_hex(&[ColumnarValue::Array(Arc::new(dict))]).unwrap(); + let result = match result { + ColumnarValue::Array(array) => array, + _ => panic!("Expected array"), + }; + let result = as_dictionary_array(&result).unwrap(); + + let keys = Int32Array::from(vec![Some(0), None, Some(1)]); + let vals = StringArray::from(vec![Some("6869"), None]); + let expected = DictionaryArray::new(keys, Arc::new(vals)); + + assert_eq!(&expected, result); + } } From 62cfc0ceae30f92e12a34ee79d2a1be8c03b0d80 Mon Sep 17 00:00:00 2001 From: Tobias Schwarzinger Date: Thu, 30 Jul 2026 15:41:04 +0200 Subject: [PATCH 083/109] fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670) ## Which issue does this PR close? Minor fix I stumbled across. There is no issue (I can create one if this warrants discussion). ## Rationale for this change If the file scan is guaranteed to have no results (`num_rows == Precision::Exact(0)`), filtering them does not change the result. It will be zero anyway. ## What changes are included in this PR? Only change `Exact` -> `Inexact` if there are non-zero rows ## Are these changes tested? Yes. ## Are there any user-facing changes? It can happen that the optimizer can now better optimize the query plans (which we are looking forward to). --- .../datasource/src/file_scan_config/mod.rs | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs index 962df06302386..d1dd3c11fca7d 100644 --- a/datafusion/datasource/src/file_scan_config/mod.rs +++ b/datafusion/datasource/src/file_scan_config/mod.rs @@ -39,6 +39,7 @@ use datafusion_execution::{ use datafusion_expr::Operator; use crate::source::OpenArgs; +use datafusion_common::stats::Precision; use datafusion_physical_expr::expressions::{BinaryExpr, Column}; use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping}; use datafusion_physical_expr::utils::reassign_expr_columns; @@ -1233,7 +1234,9 @@ impl FileScanConfig { /// we can't guarantee the statistics are exact because we don't know how many /// rows will be filtered out. pub fn statistics(&self) -> Statistics { - if self.file_source.filter().is_some() { + let filter_may_change_row_count = self.file_source.filter().is_some() + && self.statistics.num_rows != Precision::Exact(0); + if filter_may_change_row_count { self.statistics.clone().to_inexact() } else { self.statistics.clone() @@ -2415,7 +2418,6 @@ mod tests { use crate::source::DataSourceExec; use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext}; - // Create a schema with 4 columns let schema = Arc::new(Schema::new(vec![ Field::new("col0", DataType::Int32, false), Field::new("col1", DataType::Int32, false), @@ -2499,6 +2501,45 @@ mod tests { assert_eq!(partition_stats.total_byte_size, Precision::Exact(800)); } + #[test] + fn test_statistics_with_filter() { + assert_num_rows_with_filter(Precision::Absent, Precision::Absent); + assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100)); + assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100)); + assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0)); + + /// Creates a [`FileScanConfig`] with a filter and calls [`FileScanConfig::statistics`]. + /// Then the function checks the output num_rows stats, given the input num_rows stats. + fn assert_num_rows_with_filter( + input_num_rows: Precision, + expected_num_rows: Precision, + ) { + let schema = Arc::new(Schema::new(vec![Field::new( + "col0", + DataType::Int32, + false, + )])); + + let stats = + Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows); + let file_group = + FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]); + + let table_schema = TableSchema::from(&schema); + let config = FileScanConfigBuilder::new( + ObjectStoreUrl::parse("test:///").unwrap(), + Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new( + Literal::new(ScalarValue::Boolean(Some(true))), + ))), + ) + .with_file_groups(vec![file_group]) + .with_statistics(stats) + .build(); + + assert_eq!(config.statistics().num_rows, expected_num_rows,); + } + } + /// Regression test for reusing a `DataSourceExec` after its execution-local /// shared work queue has been drained. /// From 3aef16c74084d56b19b7295e89108f46da5ad826 Mon Sep 17 00:00:00 2001 From: Phoenix Date: Thu, 30 Jul 2026 23:05:55 +0800 Subject: [PATCH 084/109] refactor(proto): migrate scalar subquery serde (#23915) ## Which issue does this PR close? - Closes #23515. ## Rationale for this change ScalarSubqueryExpr nodes must share the results container populated by their enclosing ScalarSubqueryExec. A normal child decode loses that scope when serialization moves into the plan implementation. Add a scoped child decode operation so the new plan hooks preserve the shared container while removing the central serialization path. ## What changes are included in this PR? ## Are these changes tested? ## Are there any user-facing changes? Signed-off-by: Jiawei Zhao --- datafusion/physical-plan/src/proto.rs | 20 ++++ .../physical-plan/src/scalar_subquery.rs | 62 ++++++++++++ datafusion/proto/src/physical_plan/mod.rs | 98 +++++++------------ 3 files changed, 120 insertions(+), 60 deletions(-) diff --git a/datafusion/physical-plan/src/proto.rs b/datafusion/physical-plan/src/proto.rs index 1731203f6c767..2c6636a392462 100644 --- a/datafusion/physical-plan/src/proto.rs +++ b/datafusion/physical-plan/src/proto.rs @@ -62,6 +62,7 @@ use std::sync::Arc; use arrow::datatypes::Schema; use datafusion_common::{Result, internal_datafusion_err}; use datafusion_execution::TaskContext; +use datafusion_expr::physical_planning_context::ScalarSubqueryResults; use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF}; use datafusion_physical_expr::PhysicalExpr; use datafusion_proto_models::protobuf::{PhysicalExprNode, PhysicalPlanNode}; @@ -102,6 +103,14 @@ pub trait ExecutionPlanDecode { /// deserializer, so the child's own `try_from_proto` is honored). fn decode_plan(&self, node: &PhysicalPlanNode) -> Result>; + /// Deserialize a child plan with `results` active for scalar subquery + /// expressions in that plan's subtree. + fn decode_plan_with_scalar_subquery_results( + &self, + node: &PhysicalPlanNode, + results: ScalarSubqueryResults, + ) -> Result>; + /// Deserialize a physical expression against `input_schema`. fn decode_expr( &self, @@ -215,6 +224,17 @@ impl<'a> ExecutionPlanDecodeCtx<'a> { self.decoder.decode_plan(node) } + /// Deserialize a child plan with `results` active for scalar subquery + /// expressions in that plan's subtree. + pub fn decode_child_with_scalar_subquery_results( + &self, + node: &PhysicalPlanNode, + results: ScalarSubqueryResults, + ) -> Result> { + self.decoder + .decode_plan_with_scalar_subquery_results(node, results) + } + /// Deserialize a required child plan, producing a uniform "missing required /// field" error when the optional wire field is absent. pub fn decode_required_child( diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rs index 2e04b5456bfdd..73acb2ab13480 100644 --- a/datafusion/physical-plan/src/scalar_subquery.rs +++ b/datafusion/physical-plan/src/scalar_subquery.rs @@ -254,6 +254,68 @@ impl ExecutionPlan for ScalarSubqueryExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + + #[cfg(feature = "proto")] + fn try_to_proto( + &self, + ctx: &crate::proto::ExecutionPlanEncodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let input = ctx.encode_child(self.input())?; + // Subquery indices are positional and recovered during decoding. + let subqueries = + ctx.encode_children(self.subqueries().iter().map(|subquery| &subquery.plan))?; + Ok(Some(protobuf::PhysicalPlanNode { + physical_plan_type: Some( + protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery(Box::new( + protobuf::ScalarSubqueryExecNode { + input: Some(Box::new(input)), + subqueries, + }, + )), + ), + })) + } +} + +#[cfg(feature = "proto")] +impl ScalarSubqueryExec { + /// Reconstruct a [`ScalarSubqueryExec`] from its protobuf representation. + pub fn try_from_proto( + node: &datafusion_proto_models::protobuf::PhysicalPlanNode, + ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>, + ) -> Result> { + use datafusion_proto_models::protobuf; + + let scalar_subquery = crate::expect_plan_variant!( + node, + protobuf::physical_plan_node::PhysicalPlanType::ScalarSubquery, + "ScalarSubqueryExec", + ); + let results = ScalarSubqueryResults::new(scalar_subquery.subqueries.len()); + let input_node = scalar_subquery.input.as_deref().ok_or_else(|| { + datafusion_common::internal_datafusion_err!( + "ScalarSubqueryExec is missing required field 'input'" + ) + })?; + // The input's ScalarSubqueryExpr nodes must share this results container. + let input = + ctx.decode_child_with_scalar_subquery_results(input_node, results.clone())?; + let subqueries = scalar_subquery + .subqueries + .iter() + .enumerate() + .map(|(index, plan)| { + Ok(ScalarSubqueryLink { + plan: ctx.decode_child(plan)?, + index: SubqueryIndex::new(index), + }) + }) + .collect::>>()?; + + Ok(Arc::new(Self::new(input, subqueries, results))) + } } /// Wait for the subquery execution future to complete. diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 7ee173cb36868..79c6394933eae 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -49,7 +49,7 @@ use datafusion_datasource_parquet::source::ParquetSource; #[cfg(feature = "parquet")] use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_execution::{FunctionRegistry, TaskContext}; -use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex}; +use datafusion_expr::physical_planning_context::ScalarSubqueryResults; use datafusion_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF}; use datafusion_functions_table::generate_series::{ Empty, GenSeriesArgs, GenerateSeriesTable, GenericSeriesState, TimestampValue, @@ -85,7 +85,7 @@ use datafusion_physical_plan::proto::{ ExecutionPlanEncodeCtx, }; use datafusion_physical_plan::repartition::RepartitionExec; -use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink}; +use datafusion_physical_plan::scalar_subquery::ScalarSubqueryExec; use datafusion_physical_plan::sorts::sort::SortExec; use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion_physical_plan::union::{InterleaveExec, UnionExec}; @@ -811,8 +811,8 @@ pub trait PhysicalPlanNodeExt: Sized { PhysicalPlanType::Buffer(_) => { BufferExec::try_from_proto(self.node(), &decode_ctx) } - PhysicalPlanType::ScalarSubquery(sq) => { - self.try_into_scalar_subquery_physical_plan(sq, ctx, proto_converter) + PhysicalPlanType::ScalarSubquery(_) => { + ScalarSubqueryExec::try_from_proto(self.node(), &decode_ctx) } } } @@ -872,14 +872,6 @@ pub trait PhysicalPlanNodeExt: Sized { return Ok(node); } - if let Some(exec) = plan.downcast_ref::() { - return protobuf::PhysicalPlanNode::try_from_scalar_subquery_exec( - exec, - codec, - proto_converter, - ); - } - let mut buf: Vec = vec![]; match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) { Ok(_) => { @@ -1950,38 +1942,27 @@ pub trait PhysicalPlanNodeExt: Sized { BufferExec::try_from_proto(&node, &decode_ctx) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `ScalarSubqueryExec` deserializes itself via `ScalarSubqueryExec::try_from_proto`" + )] fn try_into_scalar_subquery_physical_plan( &self, sq: &protobuf::ScalarSubqueryExecNode, ctx: &PhysicalPlanDecodeContext<'_>, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - // First, deserialize the main input plan. We set up the subquery results - // container first, so that ScalarSubqueryExpr nodes can reference it. - let subquery_results = ScalarSubqueryResults::new(sq.subqueries.len()); - let input_ctx = ctx.with_scalar_subquery_results(subquery_results.clone()); - let input = into_physical_plan(&sq.input, &input_ctx, proto_converter)?; - - // Now deserialize the subquery children. - let subqueries: Vec = sq - .subqueries - .iter() - .enumerate() - .map(|(index, sq_plan)| { - let plan = - sq_plan.try_into_physical_plan_with_context(ctx, proto_converter)?; - Ok(ScalarSubqueryLink { - plan, - index: SubqueryIndex::new(index), - }) - }) - .collect::>>()?; - - Ok(Arc::new(ScalarSubqueryExec::new( - input, - subqueries, - subquery_results, - ))) + let node = protobuf::PhysicalPlanNode { + physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new( + sq.clone(), + ))), + }; + let decoder = ConverterPlanDecoder { + ctx, + proto_converter, + }; + let decode_ctx = ExecutionPlanDecodeCtx::new(&decoder); + ScalarSubqueryExec::try_from_proto(&node, &decode_ctx) } #[deprecated( @@ -2866,35 +2847,22 @@ pub trait PhysicalPlanNodeExt: Sized { .ok_or_else(|| internal_datafusion_err!("BufferExec is not serializable")) } + #[deprecated( + since = "55.0.0", + note = "unused by DataFusion; `ScalarSubqueryExec` serializes itself via `ExecutionPlan::try_to_proto`" + )] fn try_from_scalar_subquery_exec( exec: &ScalarSubqueryExec, codec: &dyn PhysicalExtensionCodec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result { - let input = protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - Arc::clone(exec.input()), + let encoder = ConverterPlanEncoder { codec, proto_converter, - )?; - let subqueries = exec - .subqueries() - .iter() - .map(|sq| { - protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter( - Arc::clone(&sq.plan), - codec, - proto_converter, - ) - }) - .collect::>>()?; - - Ok(protobuf::PhysicalPlanNode { - physical_plan_type: Some(PhysicalPlanType::ScalarSubquery(Box::new( - protobuf::ScalarSubqueryExecNode { - input: Some(Box::new(input)), - subqueries, - }, - ))), + }; + let encode_ctx = ExecutionPlanEncodeCtx::new(&encoder); + exec.try_to_proto(&encode_ctx)?.ok_or_else(|| { + internal_datafusion_err!("ScalarSubqueryExec is not serializable") }) } } @@ -3485,6 +3453,16 @@ impl ExecutionPlanDecode for ConverterPlanDecoder<'_, '_> { self.proto_converter.proto_to_execution_plan(node, self.ctx) } + fn decode_plan_with_scalar_subquery_results( + &self, + node: &protobuf::PhysicalPlanNode, + results: ScalarSubqueryResults, + ) -> Result> { + let scoped_ctx = self.ctx.with_scalar_subquery_results(results); + self.proto_converter + .proto_to_execution_plan(node, &scoped_ctx) + } + fn decode_expr( &self, node: &protobuf::PhysicalExprNode, From 541caab85a265c104888f1988fe54b08fa022f53 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:34:26 -0500 Subject: [PATCH 085/109] refactor: mark the ExecutionPlan proto dispatch traits as non-public API (#24001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #23494. Precursor to #23915 (and to the remaining `DataSource` / `DataSink` items, #23497 / #23498). ## Rationale for this change `ExecutionPlanEncode` / `ExecutionPlanDecode` (added in #23495) are dispatch details of the `try_to_proto` / `try_from_proto` hooks, not extension points. They are defined in `datafusion-physical-plan`, implemented only by `ConverterPlanEncoder` / `ConverterPlanDecoder` in `datafusion-proto`, and plans reach them exclusively through `ExecutionPlanEncodeCtx` / `ExecutionPlanDecodeCtx` — their own docs already say so: > Internal dispatch trait backing [`ExecutionPlanEncodeCtx`]. Implemented by `datafusion-proto`. > Plan authors never name this trait. They are `pub` only because those adapters live in another crate. Nothing said that to the tooling, so every capability the epic still has to add to the ctx reads as a major breaking change. #23915 hits this first: it needs `decode_plan_with_scalar_subquery_results` so `ScalarSubqueryExec` can decode its input with the subquery-results container in scope, and `cargo-semver-checks` flags the required method as `trait_method_added`. The `DataSource` / `DataSink` families will want their own primitives next. `#[doc(hidden)]` states what was already true, and makes those additions changes to something that was never public API — rather than asking each follow-up PR to explain away a breakage report. ## What changes are included in this PR? Mark `ExecutionPlanEncode` and `ExecutionPlanDecode` `#[doc(hidden)]`, and say why in their docs and in the module overview. One file, no behavior, wire-format, or signature changes. A sealed-trait supertrait was the first thing I tried. It is worse here: sealing across a crate boundary needs a `pub` marker anyway, so it does not actually prevent a downstream impl — it just adds a public item in order to say "this is not public API", plus an `impl` line for every implementor including test doubles. `#[doc(hidden)]` says the same thing by removing API surface instead of adding it, and `cargo-semver-checks` honors both identically (measured below). ## Are these changes tested? The property this PR buys is a `cargo-semver-checks` classification, so it is verified with that tool directly — v0.49.0, the version CI installs, invoked the way CI invokes it: | baseline | change under test | result | |---|---|---| | `main` | add a required method to `ExecutionPlanDecode` | `trait_method_added` — **major** | | this PR | the same required method | 223 checks pass, **no semver update required** | Worth recording, since it drove the shape of this PR: the lint reads `public_api_sealed` from the **baseline**, so this has to land before the PRs that add methods, not alongside them. Also, `cargo-semver-checks` does not re-qualify the traits as public API even though `ExecutionPlanEncodeCtx::new` still names them in a public signature. Also ran, on the pinned 1.97.0 toolchain: `cargo fmt --all -- --check`, `cargo clippy --all-targets --all-features` for both crates, `cargo doc` with `-D warnings` (no broken intra-doc links from the module docs to the now-hidden traits), and the full `datafusion-proto` integration suite (211 passed). ## Are there any user-facing changes? `cargo-semver-checks` will report `trait_now_doc_hidden` on this PR, and that is the intended content of the change. Real-world impact is nil: both traits were introduced by #23495, which merged after `54.1.0` was tagged, so no released version of `datafusion-physical-plan` contains them. The load-bearing public API — `ExecutionPlan::try_to_proto`, the two ctx types, `AsExecutionPlan`, `PhysicalExtensionCodec`, `PhysicalProtoConverterExtension` — is untouched, and `datafusion-proto` needs no change at all. Note for contributors on #23494: the expression-side equivalents (`PhysicalExprEncode` / `PhysicalExprDecode` in `physical-expr-common`) have the same shape and the same argument, but several test doubles across crates implement them. Left for a follow-up. Co-authored-by: Claude Opus 5 --- datafusion/physical-plan/src/proto.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/proto.rs b/datafusion/physical-plan/src/proto.rs index 2c6636a392462..f84cd67d46e3b 100644 --- a/datafusion/physical-plan/src/proto.rs +++ b/datafusion/physical-plan/src/proto.rs @@ -33,7 +33,8 @@ //! * [`ExecutionPlanEncode`] / [`ExecutionPlanDecode`] — internal dispatch //! traits, *defined* here but *implemented* in `datafusion-proto`, that the //! context types delegate to. This is the dependency inversion that keeps the -//! proto types flowing in one direction only. +//! proto types flowing in one direction only. They are `#[doc(hidden)]`: not +//! public API, `pub` only because their implementors live in another crate. //! //! `datafusion-physical-plan` depends on the pure prost types in //! `datafusion-proto-models` (feature `proto`), never on `datafusion-proto`. @@ -73,6 +74,11 @@ use crate::ExecutionPlan; /// /// Implemented by `datafusion-proto`. Plan authors never name this trait; they /// call methods on [`ExecutionPlanEncodeCtx`] instead. +/// +/// **Not public API.** `pub` only because the implementors live in another +/// crate; `#[doc(hidden)]` records that, so encoding primitives can be added +/// here as the serialization hooks grow without breaking downstream code. +#[doc(hidden)] pub trait ExecutionPlanEncode { /// Serialize a child execution plan (recursing through the central /// serializer, so the child's own `try_to_proto` hook is honored). @@ -98,6 +104,11 @@ pub trait ExecutionPlanEncode { /// /// Implemented by `datafusion-proto`. Plan authors never name this trait; they /// call methods on [`ExecutionPlanDecodeCtx`] instead. +/// +/// **Not public API.** `pub` only because the implementors live in another +/// crate; `#[doc(hidden)]` records that, so decoding primitives can be added +/// here as the serialization hooks grow without breaking downstream code. +#[doc(hidden)] pub trait ExecutionPlanDecode { /// Deserialize a child execution plan (recursing through the central /// deserializer, so the child's own `try_from_proto` is honored). From 833e501459dcc418042b3280a0ea9cb2980005d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C5=9Een?= Date: Thu, 30 Jul 2026 19:38:35 +0300 Subject: [PATCH 086/109] fix(proto): prevent duplicate partition statistics on roundtrip (#23999) ## Which issue does this PR close? - Closes #23998 ## Rationale for this change Decoding a `PartitionedFile` re-appended partition-column statistics already present in protobuf. ## What changes are included in this PR? Assign decoded statistics directly. ## Are these changes tested? Yes new roundtrip test for this ## Are there any user-facing changes? no --------- Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- .../proto/src/physical_plan/from_proto.rs | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs index d5a1e0efac6b6..7aa6376313c96 100644 --- a/datafusion/proto/src/physical_plan/from_proto.rs +++ b/datafusion/proto/src/physical_plan/from_proto.rs @@ -660,7 +660,10 @@ impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile { pf = pf.with_range(file_range.start, file_range.end); } if let Some(proto_stats) = val.statistics.as_ref() { - pf = pf.with_statistics(Arc::new(proto_stats.try_into()?)); + // The wire format carries statistics for the full table schema (file + partition + // columns), so assign directly — `with_statistics` would append the partition + // column stats a second time. + pf.statistics = Some(Arc::new(proto_stats.try_into()?)); } Ok(pf) } @@ -807,8 +810,8 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD #[cfg(test)] mod tests { - use super::*; + use arrow::datatypes::{DataType, Field, Schema}; #[test] fn partitioned_file_path_roundtrip_percent_encoded() { @@ -833,7 +836,6 @@ mod tests { #[test] fn partitioned_file_arrow_schema_roundtrip() { - use arrow::datatypes::{DataType, Field, Schema}; use std::collections::HashMap; let arrow_schema = Arc::new(Schema::new_with_metadata( @@ -858,6 +860,28 @@ mod tests { ); } + #[test] + fn partitioned_file_statistics_roundtrip_with_partition_values() { + use datafusion_common::Statistics; + let file_schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]); + let pf = PartitionedFile::new("foo/bar.parquet", 1234) + .with_partition_values(vec![ScalarValue::from("2024-01-01")]) + .with_statistics(Arc::new(Statistics::new_unknown(&file_schema))); + + // `statistics` covers the full table schema: file columns followed by one + // entry per partition column. + let expected_len = file_schema.fields().len() + pf.partition_values.len(); + assert_eq!( + pf.statistics.as_ref().unwrap().column_statistics.len(), + expected_len + ); + + let proto = protobuf::PartitionedFile::try_from_proto(&pf).unwrap(); + let decoded = PartitionedFile::try_from_proto(&proto).unwrap(); + + assert_eq!(decoded.statistics, pf.statistics); + } + #[test] fn partitioned_file_from_proto_invalid_path() { let proto = protobuf::PartitionedFile { From f398301748b4a6f5279fc4ff553103e1c778469a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:30:15 -0500 Subject: [PATCH 087/109] Report peak MemoryPool reservation per query in benchmarks (#23985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Part of #22758. Doesn't close an issue. ## Rationale for this change DataFusion's `MemoryPool` deliberately accounts for only the "large" allocations that scale with input size; intermediate batches flowing between operators are assumed to be small and left untracked. The `MemoryPool` docs therefore advise reserving "some overhead (e.g. 10%)" on top of the configured limit. Nothing measures what that overhead actually is. `MemoryPool::reserved()` is a live value that has usually fallen back to zero by the time a query finishes, so the peak is never observed, and benchmarks report peak RSS with nothing to compare it against. This adds the missing number so the two can be compared. It is measurement only — nothing enforces a relationship between the pool's accounting and actual allocation, and no threshold or CI check is added. This is complementary to the accounting work discussed in #22898. That issue proposes changing *what* gets accounted (Arrow `claim()` / builder `with_pool()`); because `ArrowMemoryPool` grows a DataFusion `MemoryReservation` against the pool it wraps, `reserved()` is where both models converge. This PR just makes the peak of that value observable per query, so the effect of any such change — or of a regression — is visible as a number rather than inferred. Concretely, that means the peak follows the accounting instead of being pinned to the current set of manually tracked consumers: nothing claims buffers today, but if #22898 lands in either form, those bytes show up here without further changes. There is a test covering that path. ## What changes are included in this PR? - `PeakRecordingPool`, a delegating `MemoryPool` wrapper recording the high-water mark of `reserved()`. Every method delegates; wrapping does not change how memory is granted, limited, or reported. - `CommonOpt::runtime_env_builder` installs it around the pool it already builds, so every benchmark run with a memory limit reports the peak without further changes. - `BenchQuery` gains `pool_peak_bytes`, reset per case by `start_new_case`, so each query gets its own reading rather than inheriting the high-water mark of the queries before it. - `print_memory_stats` prints the run-wide peak after the existing mimalloc line, leaving `mem_profile`'s output parser untouched. Everything is confined to the `benchmarks` crate. No trait changes. `arrow-buffer/pool` and `datafusion-execution/arrow_buffer_pool` are enabled as **dev-dependencies only**, so the benchmark binaries build with exactly the features they had before — confirmed absent from `cargo tree --no-dev-dependencies`. ## Are these changes tested? Yes — 5 unit tests plus a doctest, and verified end-to-end with `dfbench nlj --query 1 -i 2 --memory-limit 512M`, which emits `"pool_peak_bytes": 262528` per query. One of those tests builds an `ArrowMemoryPool` over the recording pool and asserts an Arrow-side reservation both raises the peak and releases on drop, pinning the interaction described above. The pre-existing `test_runtime_env_builder_reads_env_var` still asserts `MemoryLimit::Finite(2G)` *through* the wrapper, which is direct evidence the delegation is transparent. The wrapper is installed only when a memory limit is configured, since without one there is no pool to wrap. `pool_peak_bytes` is omitted from the results JSON in that case, leaving output byte-identical to before for runs without a limit — confirmed `compare.py` parses both old and new files. ## Are there any user-facing changes? No. Confined to the benchmarks crate; adds one optional field to the benchmark results JSON. --------- Co-authored-by: Claude --- Cargo.lock | 2 + benchmarks/Cargo.toml | 5 + benchmarks/README.md | 8 + benchmarks/src/bin/benchmark_runner.rs | 5 +- benchmarks/src/bin/external_aggr.rs | 16 +- benchmarks/src/clickbench.rs | 3 +- benchmarks/src/dict.rs | 1 + benchmarks/src/h2o.rs | 3 +- benchmarks/src/hj.rs | 1 + benchmarks/src/imdb/run.rs | 13 +- benchmarks/src/nlj.rs | 1 + benchmarks/src/smj.rs | 1 + benchmarks/src/sort_pushdown.rs | 13 +- benchmarks/src/sort_tpch.rs | 14 +- benchmarks/src/sql_benchmark_runner.rs | 2 +- benchmarks/src/tpcds/run.rs | 3 +- benchmarks/src/tpch/run.rs | 3 +- benchmarks/src/util/memory.rs | 30 +- benchmarks/src/util/memory_pool.rs | 381 +++++++++++++++++++++++++ benchmarks/src/util/mod.rs | 2 + benchmarks/src/util/options.rs | 5 +- benchmarks/src/util/run.rs | 140 ++++++++- 22 files changed, 631 insertions(+), 21 deletions(-) create mode 100644 benchmarks/src/util/memory_pool.rs diff --git a/Cargo.lock b/Cargo.lock index a41734e064d4c..451ff70b1cd48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1780,6 +1780,7 @@ name = "datafusion-benchmarks" version = "54.1.0" dependencies = [ "arrow", + "arrow-buffer", "async-trait", "bytes", "clap", @@ -1787,6 +1788,7 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-common-runtime", + "datafusion-execution", "datafusion-proto", "env_logger", "futures", diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 5dae70761f9a7..282b27e48101d 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -64,6 +64,11 @@ tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] } tokio-util = { version = "0.7.17" } [dev-dependencies] +# `pool`/`arrow_buffer_pool` are enabled only for tests, so the benchmark +# binaries are built exactly as before. They let `memory_pool`'s tests cover +# Arrow-side reservations reaching the pool via `ArrowMemoryPool`. +arrow-buffer = { workspace = true, features = ["pool"] } +datafusion-execution = { workspace = true, features = ["arrow_buffer_pool"] } datafusion-proto = { workspace = true, features = ["parquet"] } tempfile = { workspace = true } diff --git a/benchmarks/README.md b/benchmarks/README.md index 34c67e5151ba1..b6a7705cf94e3 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -483,6 +483,14 @@ Your benchmark should create and use an instance of `BenchmarkRun` defined in `b - Call its `start_new_case` method with a string that will appear in the "Query" column of the compare output. - Use `write_iter` to record elapsed times for the behavior you're benchmarking. +- Call `set_memory_pool` with the `RuntimeEnv`'s memory pool (`ctx.runtime_env().memory_pool`), + and again for each new runtime if your benchmark builds one per query. Each case then reports a + `pool_peak_bytes` field: the peak `MemoryPool` reservation reached while running it, which is the + largest value across that case's iterations. The field is omitted when the benchmark runs without + `--memory-limit`, since no pool is installed to record. Comparing it against the peak RSS printed + by `print_memory_stats` shows how much of the run's memory the pool actually accounted for; the + pool only tracks the "large" allocations that scale with input size, so the two are expected to + differ. - When all cases are done, call the `BenchmarkRun`'s `maybe_write_json` method, giving it the value of the `--output` structopt field on `RunOpt`. diff --git a/benchmarks/src/bin/benchmark_runner.rs b/benchmarks/src/bin/benchmark_runner.rs index 71b3b1b9e0a87..c7a16086c9677 100644 --- a/benchmarks/src/bin/benchmark_runner.rs +++ b/benchmarks/src/bin/benchmark_runner.rs @@ -299,6 +299,9 @@ async fn run_simple_benchmark( let case_name = benchmark_case_name(benchmark); + // Each case gets its own `SessionContext`, so hand over its pool before the + // case starts. + run.set_memory_pool(&ctx.runtime_env().memory_pool); run.start_new_case(&case_name); for iteration in 0..config.common.iterations { @@ -312,7 +315,7 @@ async fn run_simple_benchmark( run.write_iter(elapsed, row_count); } - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(()) } diff --git a/benchmarks/src/bin/external_aggr.rs b/benchmarks/src/bin/external_aggr.rs index 42f25c2cb010c..226a619192ac9 100644 --- a/benchmarks/src/bin/external_aggr.rs +++ b/benchmarks/src/bin/external_aggr.rs @@ -39,7 +39,9 @@ use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{collect, displayable}; use datafusion::prelude::*; -use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, QueryResult}; +use datafusion_benchmarks::util::{ + BenchmarkRun, CommonOpt, PeakRecordingPool, QueryResult, +}; use datafusion_common::instant::Instant; use datafusion_common::utils::get_available_parallelism; use datafusion_common::{DEFAULT_PARQUET_EXTENSION, exec_err}; @@ -169,7 +171,7 @@ impl ExternalAggrConfig { )); let query_results = self - .benchmark_query(query_id, mem_limit, mem_pool_type) + .benchmark_query(query_id, mem_limit, mem_pool_type, &mut benchmark_run) .await?; for iter in query_results { benchmark_run.write_iter(iter.elapsed, iter.row_count); @@ -182,11 +184,15 @@ impl ExternalAggrConfig { } /// Benchmark query `query_id` in `AGGR_QUERIES` + /// + /// `benchmark_run` is handed this query's runtime, which is built here + /// because each query runs under its own memory limit. async fn benchmark_query( &self, query_id: usize, mem_limit: u64, mem_pool_type: &str, + benchmark_run: &mut BenchmarkRun, ) -> Result> { let query_name = format!("Q{query_id}({})", human_readable_size(mem_limit as usize)); @@ -198,6 +204,12 @@ impl ExternalAggrConfig { return exec_err!("Invalid memory pool type: {}", mem_pool_type); } }; + // This benchmark builds its pool directly rather than going through + // `CommonOpt::runtime_env_builder`, so it has to install the recorder + // itself to report a peak. + let memory_pool: Arc = + Arc::new(PeakRecordingPool::new(memory_pool)); + benchmark_run.set_memory_pool(&memory_pool); let runtime_env = RuntimeEnvBuilder::new() .with_memory_pool(memory_pool) .build_arc()?; diff --git a/benchmarks/src/clickbench.rs b/benchmarks/src/clickbench.rs index 70aaeb7d2d192..a2e65aa5618a9 100644 --- a/benchmarks/src/clickbench.rs +++ b/benchmarks/src/clickbench.rs @@ -213,6 +213,7 @@ impl RunOpt { self.register_hits(&ctx).await?; let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { let query_path = get_query_path(&self.queries_path, query_id); let Some(sql) = get_query_sql(&query_path)? else { @@ -278,7 +279,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/dict.rs b/benchmarks/src/dict.rs index f8451715ea81e..e04b5f816adcc 100644 --- a/benchmarks/src/dict.rs +++ b/benchmarks/src/dict.rs @@ -333,6 +333,7 @@ impl RunOpt { let rt = self.common.build_runtime()?; let ctx = SessionContext::new_with_config_rt(config, rt); let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { let query = &DICTIONARY_QUERIES[query_id - 1]; diff --git a/benchmarks/src/h2o.rs b/benchmarks/src/h2o.rs index 8b6e04932cb39..feb4bf2fa11ce 100644 --- a/benchmarks/src/h2o.rs +++ b/benchmarks/src/h2o.rs @@ -109,6 +109,7 @@ impl RunOpt { let iterations = self.common.iterations; let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { benchmark_run.start_new_case(&format!("Query {query_id}")); let sql = queries.get_query(query_id)?; @@ -131,7 +132,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); if self.common.debug { ctx.sql(sql) diff --git a/benchmarks/src/hj.rs b/benchmarks/src/hj.rs index 9a91e2714ac86..4f97b24d0f02c 100644 --- a/benchmarks/src/hj.rs +++ b/benchmarks/src/hj.rs @@ -564,6 +564,7 @@ impl RunOpt { } let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { let query_index = query_id - 1; diff --git a/benchmarks/src/imdb/run.rs b/benchmarks/src/imdb/run.rs index a8e202888794f..5822bbcb0d89e 100644 --- a/benchmarks/src/imdb/run.rs +++ b/benchmarks/src/imdb/run.rs @@ -295,7 +295,7 @@ impl RunOpt { let mut benchmark_run = BenchmarkRun::new(); for query_id in query_range { benchmark_run.start_new_case(&format!("Query {query_id}")); - let query_run = self.benchmark_query(query_id).await?; + let query_run = self.benchmark_query(query_id, &mut benchmark_run).await?; for iter in query_run { benchmark_run.write_iter(iter.elapsed, iter.row_count); } @@ -304,7 +304,13 @@ impl RunOpt { Ok(()) } - async fn benchmark_query(&self, query_id: usize) -> Result> { + /// `benchmark_run` is handed this query's runtime, which is built here so + /// each query gets a pool of its own. + async fn benchmark_query( + &self, + query_id: usize, + benchmark_run: &mut BenchmarkRun, + ) -> Result> { let mut config = self .common .config()? @@ -314,6 +320,7 @@ impl RunOpt { self.hash_join_buffering_capacity; let rt = self.common.build_runtime()?; let ctx = SessionContext::new_with_config_rt(config, rt); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); // register tables self.register_tables(&ctx).await?; @@ -348,7 +355,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/nlj.rs b/benchmarks/src/nlj.rs index 361cc35ec200c..485ee069d1bba 100644 --- a/benchmarks/src/nlj.rs +++ b/benchmarks/src/nlj.rs @@ -211,6 +211,7 @@ impl RunOpt { let ctx = SessionContext::new_with_config_rt(config, rt); let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { let query_index = query_id - 1; // Convert 1-based to 0-based index diff --git a/benchmarks/src/smj.rs b/benchmarks/src/smj.rs index 3d173b7116e2b..9282f72c2fab6 100644 --- a/benchmarks/src/smj.rs +++ b/benchmarks/src/smj.rs @@ -550,6 +550,7 @@ impl RunOpt { let ctx = SessionContext::new_with_config_rt(config, rt); let mut benchmark_run = BenchmarkRun::new(); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); for query_id in query_range { let query_index = query_id - 1; // Convert 1-based to 0-based index diff --git a/benchmarks/src/sort_pushdown.rs b/benchmarks/src/sort_pushdown.rs index 86f1c0f5c1119..77f889e702e3d 100644 --- a/benchmarks/src/sort_pushdown.rs +++ b/benchmarks/src/sort_pushdown.rs @@ -137,7 +137,7 @@ impl RunOpt { for query_id in query_ids { benchmark_run.start_new_case(&format!("{query_id}")); - let query_results = self.benchmark_query(query_id).await; + let query_results = self.benchmark_query(query_id, &mut benchmark_run).await; match query_results { Ok(query_results) => { for iter in query_results { @@ -156,7 +156,13 @@ impl RunOpt { Ok(()) } - async fn benchmark_query(&self, query_id: usize) -> Result> { + /// `benchmark_run` is handed this query's runtime, which is built here so + /// each query gets a pool of its own. + async fn benchmark_query( + &self, + query_id: usize, + benchmark_run: &mut BenchmarkRun, + ) -> Result> { let sql = self.load_query(query_id)?; let config = self.common.config()?; @@ -168,6 +174,7 @@ impl RunOpt { .with_default_features() .build(); let ctx = SessionContext::from(state); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); self.register_tables(&ctx).await?; @@ -191,7 +198,7 @@ impl RunOpt { let avg = millis.iter().sum::() / millis.len() as f64; println!("Query {query_id} avg time: {avg:.2} ms"); - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/sort_tpch.rs b/benchmarks/src/sort_tpch.rs index 2182d1a383633..d5f81c04a3ba4 100644 --- a/benchmarks/src/sort_tpch.rs +++ b/benchmarks/src/sort_tpch.rs @@ -187,7 +187,7 @@ impl RunOpt { for query_id in query_range { benchmark_run.start_new_case(&format!("{query_id}")); - let query_results = self.benchmark_query(query_id).await; + let query_results = self.benchmark_query(query_id, &mut benchmark_run).await; match query_results { Ok(query_results) => { for iter in query_results { @@ -207,7 +207,14 @@ impl RunOpt { } /// Benchmark query `query_id` in `SORT_QUERIES` - async fn benchmark_query(&self, query_id: usize) -> Result> { + /// + /// `benchmark_run` is handed this query's runtime, which is built here so + /// each query gets a pool of its own. + async fn benchmark_query( + &self, + query_id: usize, + benchmark_run: &mut BenchmarkRun, + ) -> Result> { let config = self.common.config()?; let rt = self.common.build_runtime()?; let state = SessionStateBuilder::new() @@ -216,6 +223,7 @@ impl RunOpt { .with_default_features() .build(); let ctx = SessionContext::from(state); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); // register tables self.register_tables(&ctx).await?; @@ -250,7 +258,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory usage stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/sql_benchmark_runner.rs b/benchmarks/src/sql_benchmark_runner.rs index f1cb3ad2f71b4..420e780c645aa 100644 --- a/benchmarks/src/sql_benchmark_runner.rs +++ b/benchmarks/src/sql_benchmark_runner.rs @@ -110,7 +110,7 @@ fn run_criterion_benchmark( match result { Ok(()) => { - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(()) } Err(payload) => Err(panic_payload_to_error(payload.as_ref())), diff --git a/benchmarks/src/tpcds/run.rs b/benchmarks/src/tpcds/run.rs index 2e0274c935de3..3eaaf172c0f16 100644 --- a/benchmarks/src/tpcds/run.rs +++ b/benchmarks/src/tpcds/run.rs @@ -226,6 +226,7 @@ impl RunOpt { self.hash_join_buffering_capacity; let rt = self.common.build_runtime()?; let ctx = SessionContext::new_with_config_rt(config, rt); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); // register tables self.register_tables(&ctx).await?; @@ -290,7 +291,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs index 422bcec9ea066..47edfbac4b5a7 100644 --- a/benchmarks/src/tpch/run.rs +++ b/benchmarks/src/tpch/run.rs @@ -137,6 +137,7 @@ impl RunOpt { self.hash_join_buffering_capacity; let rt = self.common.build_runtime()?; let ctx = SessionContext::new_with_config_rt(config, rt); + benchmark_run.set_memory_pool(&ctx.runtime_env().memory_pool); // register tables self.register_tables(&ctx).await?; let scale_factor = self.scale_factor()?; @@ -208,7 +209,7 @@ impl RunOpt { println!("Query {query_id} avg time: {avg:.2} ms"); // Print memory stats using mimalloc (only when compiled with --features mimalloc_extended) - print_memory_stats(); + print_memory_stats(&*ctx.runtime_env().memory_pool); Ok(query_results) } diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rs index 11b96ef227756..2b186c79c3516 100644 --- a/benchmarks/src/util/memory.rs +++ b/benchmarks/src/util/memory.rs @@ -15,8 +15,34 @@ // specific language governing permissions and limitations // under the License. -/// Print Peak RSS, Peak Commit, Page Faults based on mimalloc api -pub fn print_memory_stats() { +use datafusion::execution::memory_pool::MemoryPool; + +use super::PeakRecordingPool; + +/// Print Peak RSS, Peak Commit, Page Faults based on mimalloc api, followed by +/// the peak reservation of `memory_pool` when a memory limit was configured. +pub fn print_memory_stats(memory_pool: &dyn MemoryPool) { + print_allocator_stats(); + print_pool_stats(memory_pool); +} + +/// Print the peak reservation `memory_pool` has seen. +/// +/// Prints nothing when the benchmark ran without a memory limit, since no +/// [`PeakRecordingPool`] was installed to record. Comparing this against the +/// peak RSS above shows how much of a run's memory the pool actually accounted +/// for — DataFusion only tracks the "large" allocations that scale with input +/// size, so the two are expected to differ. +fn print_pool_stats(memory_pool: &dyn MemoryPool) { + if let Some(recorder) = PeakRecordingPool::from_pool(memory_pool) { + println!( + "Peak pool reserved: {}", + datafusion_common::human_readable_size(recorder.max_reserved()) + ); + } +} + +fn print_allocator_stats() { #[cfg(all(feature = "mimalloc", feature = "mimalloc_extended"))] { use datafusion_common::human_readable_size; diff --git a/benchmarks/src/util/memory_pool.rs b/benchmarks/src/util/memory_pool.rs new file mode 100644 index 0000000000000..a3606ca0a7b7a --- /dev/null +++ b/benchmarks/src/util/memory_pool.rs @@ -0,0 +1,381 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Records the peak [`MemoryPool`] reservation reached during a benchmark. +//! +//! DataFusion's [`MemoryPool`] deliberately accounts for only the "large" +//! allocations that scale with input size; intermediate batches flowing between +//! operators are assumed to be small and are left untracked. The [`MemoryPool`] +//! documentation therefore advises reserving "some overhead (e.g. 10%)" on top +//! of the configured limit. +//! +//! Nothing reports what that overhead actually is, because the peak reservation +//! itself is never recorded — [`MemoryPool::reserved`] is a live value that has +//! usually fallen back to zero by the time a query finishes. This module records +//! the high-water mark so benchmarks can emit it alongside the peak RSS that +//! [`print_memory_stats`] already prints, making the gap between the two +//! measurable. +//! +//! This is measurement only: nothing here enforces a relationship between the +//! two numbers. +//! +//! What lands in the peak is whatever the pool accounts for, so this follows +//! the accounting rather than fixing it in place. Arrow-side reservations made +//! through `ArrowMemoryPool` are included, because that adapter grows a +//! DataFusion reservation against the pool it wraps; nothing claims buffers +//! today, but the peak picks it up when something does. +//! +//! [`print_memory_stats`]: super::print_memory_stats + +use std::{ + fmt::{Debug, Display, Formatter}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use datafusion::execution::memory_pool::{ + MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation, +}; +use datafusion_common::Result; + +/// Wraps a [`MemoryPool`], recording the high-water mark of +/// [`MemoryPool::reserved`] as reservations come and go. +/// +/// Every method delegates to the wrapped pool, so wrapping does not change how +/// memory is granted, limited, or reported. The one thing it does change is +/// downcasting: `rt.memory_pool.downcast_ref::()` now finds this +/// wrapper instead of the pool it wraps. Nothing in the benchmarks relies on +/// that, and [`Self::from_pool`] uses the same mechanism to find the recorder. +/// +/// Both high-water marks are held per instance, so a benchmark that builds a +/// fresh runtime per query gets a reading scoped to that query without any +/// coordination. +/// +/// # Example +/// +/// ``` +/// # use std::sync::Arc; +/// # use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool}; +/// # use datafusion_benchmarks::util::PeakRecordingPool; +/// let recording = Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new(1024)))); +/// let pool: Arc = Arc::clone(&recording) as _; +/// +/// let reservation = MemoryConsumer::new("example").register(&pool); +/// reservation.try_grow(512)?; +/// reservation.shrink(512); +/// +/// // The pool is back to empty, but the high-water mark is retained. +/// assert_eq!(pool.reserved(), 0); +/// assert_eq!(recording.peak_reserved(), 512); +/// +/// // The recorder can also be recovered from the pool it was installed as. +/// assert_eq!(PeakRecordingPool::from_pool(&*pool).unwrap().peak_reserved(), 512); +/// # Ok::<(), datafusion_common::DataFusionError>(()) +/// ``` +pub struct PeakRecordingPool { + inner: Arc, + /// Running total of everything granted through this wrapper, kept so the + /// peak can be maintained without asking `inner` for its total. + reserved: AtomicUsize, + /// High-water mark since the last [`PeakRecordingPool::reset_peak`]. + peak: AtomicUsize, + /// High-water mark since this pool was created. Never reset. + max: AtomicUsize, +} + +impl PeakRecordingPool { + /// Wrap `inner`, recording its peak reservation from here on. + /// + /// `inner` is expected to be empty: the running total starts at zero, so + /// anything reserved before wrapping is not counted. + pub fn new(inner: Arc) -> Self { + Self { + inner, + reserved: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + max: AtomicUsize::new(0), + } + } + + /// The recorder installed as `pool`, if there is one. + /// + /// Returns `None` whenever a benchmark runs without a memory limit, since + /// [`CommonOpt::runtime_env_builder`] only installs the wrapper alongside a + /// pool it has a limit for. + /// + /// [`CommonOpt::runtime_env_builder`]: super::CommonOpt::runtime_env_builder + pub fn from_pool(pool: &dyn MemoryPool) -> Option<&Self> { + pool.downcast_ref::() + } + + /// Peak reservation, in bytes, since the last [`Self::reset_peak`]. + pub fn peak_reserved(&self) -> usize { + self.peak.load(Ordering::Relaxed) + } + + /// Peak reservation, in bytes, since this pool was created. + /// + /// Unlike [`Self::peak_reserved`] this is never reset, so it reports the + /// peak across every query that shared this pool. + pub fn max_reserved(&self) -> usize { + self.max.load(Ordering::Relaxed) + } + + /// Reset the value returned by [`Self::peak_reserved`] to what is reserved + /// right now, so the next reading covers only what follows. + /// + /// [`BenchmarkRun::start_new_case`] calls this, giving each benchmark query + /// its own reading. Anything still held when a query starts — data the + /// benchmark loaded up front, say — stays in the reading, since the query + /// runs with those bytes reserved. + /// + /// [`BenchmarkRun::start_new_case`]: super::BenchmarkRun::start_new_case + pub fn reset_peak(&self) { + self.peak + .store(self.reserved.load(Ordering::Relaxed), Ordering::Relaxed); + } + + /// Add `additional` granted bytes to the running total and publish it to + /// both high-water marks. + /// + /// Accumulating deltas rather than reading [`MemoryPool::reserved`] keeps + /// the wrapped pool's own bookkeeping off this path: `FairSpillPool` takes + /// its state lock to answer `reserved()`, which would double the lock + /// traffic of every accounted allocation in the benchmark being measured. + /// The total stays exact because the trait grants exactly what is asked + /// for — `grow` is infallible and `try_grow` either grants `additional` or + /// returns an error, leaving the reservation untouched. + fn record(&self, additional: usize) { + let reserved = + self.reserved.fetch_add(additional, Ordering::Relaxed) + additional; + self.peak.fetch_max(reserved, Ordering::Relaxed); + self.max.fetch_max(reserved, Ordering::Relaxed); + } +} + +impl Debug for PeakRecordingPool { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PeakRecordingPool") + .field("inner", &self.inner) + .field("peak", &self.peak_reserved()) + .field("max", &self.max_reserved()) + .finish() + } +} + +impl Display for PeakRecordingPool { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Deferring to the wrapped pool keeps `SHOW ALL`-style output and error + // messages identical to running without the wrapper. + Display::fmt(&self.inner, f) + } +} + +impl MemoryPool for PeakRecordingPool { + fn name(&self) -> &str { + self.inner.name() + } + + fn register(&self, consumer: &MemoryConsumer) { + self.inner.register(consumer); + } + + fn unregister(&self, consumer: &MemoryConsumer) { + self.inner.unregister(consumer); + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.inner.grow(reservation, additional); + self.record(additional); + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + self.inner.shrink(reservation, shrink); + self.reserved.fetch_sub(shrink, Ordering::Relaxed); + } + + fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> { + self.inner.try_grow(reservation, additional)?; + self.record(additional); + Ok(()) + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } + + fn memory_limit(&self) -> MemoryLimit { + self.inner.memory_limit() + } +} + +#[cfg(test)] +mod tests { + use datafusion::execution::memory_pool::GreedyMemoryPool; + + use super::*; + + /// A recording pool over a `GreedyMemoryPool`, returned both as the + /// recorder (to read the marks) and as the pool reservations register with. + fn pool(limit: usize) -> (Arc, Arc) { + let recording = Arc::new(PeakRecordingPool::new(Arc::new( + GreedyMemoryPool::new(limit), + ))); + let pool = Arc::clone(&recording) as Arc; + (recording, pool) + } + + #[test] + fn records_high_water_mark_across_reservations() { + let (recording, pool) = pool(1024); + + let a = MemoryConsumer::new("a").register(&pool); + let b = MemoryConsumer::new("b").register(&pool); + + a.try_grow(300).unwrap(); + b.try_grow(400).unwrap(); + // Peak of the sum, not the largest single reservation. + assert_eq!(recording.peak_reserved(), 700); + + a.shrink(300); + b.try_grow(100).unwrap(); + + // Falling back below the peak leaves it untouched, and the later growth + // does not reach it. + assert_eq!(pool.reserved(), 500); + assert_eq!(recording.peak_reserved(), 700); + } + + #[test] + fn failed_growth_does_not_move_the_peak() { + let (recording, pool) = pool(1024); + + let reservation = MemoryConsumer::new("a").register(&pool); + reservation.try_grow(600).unwrap(); + reservation + .try_grow(600) + .expect_err("should exceed the 1024 byte pool"); + + assert_eq!(recording.peak_reserved(), 600); + } + + #[test] + fn reset_clears_the_window_but_not_the_run_maximum() { + let (recording, pool) = pool(1024); + + let reservation = MemoryConsumer::new("a").register(&pool); + reservation.try_grow(800).unwrap(); + reservation.shrink(800); + + recording.reset_peak(); + assert_eq!(recording.peak_reserved(), 0); + assert_eq!(recording.max_reserved(), 800); + + reservation.try_grow(100).unwrap(); + assert_eq!(recording.peak_reserved(), 100); + assert_eq!(recording.max_reserved(), 800); + } + + #[test] + fn reset_keeps_what_is_still_reserved() { + let (recording, pool) = pool(1024); + + // Something a benchmark loaded up front and holds across queries. + let held = MemoryConsumer::new("held").register(&pool); + held.try_grow(300).unwrap(); + + recording.reset_peak(); + assert_eq!(recording.peak_reserved(), 300); + + let query = MemoryConsumer::new("query").register(&pool); + query.try_grow(200).unwrap(); + assert_eq!(recording.peak_reserved(), 500); + } + + #[test] + fn marks_are_per_instance() { + let (one, one_pool) = pool(1024); + let (two, _two_pool) = pool(1024); + + MemoryConsumer::new("a") + .register(&one_pool) + .try_grow(512) + .unwrap(); + + assert_eq!(one.peak_reserved(), 512); + assert_eq!(two.peak_reserved(), 0); + } + + #[test] + fn is_recoverable_from_the_pool_it_is_installed_as() { + let (recording, pool) = pool(1024); + + MemoryConsumer::new("a") + .register(&pool) + .try_grow(512) + .unwrap(); + + let found = PeakRecordingPool::from_pool(&*pool).expect("recorder installed"); + assert_eq!(found.peak_reserved(), recording.peak_reserved()); + + // A pool with no recorder in front of it reports nothing. + let plain: Arc = Arc::new(GreedyMemoryPool::new(1024)); + assert!(PeakRecordingPool::from_pool(&*plain).is_none()); + } + + #[test] + fn delegates_limit_and_name_to_the_wrapped_pool() { + let inner: Arc = Arc::new(GreedyMemoryPool::new(4096)); + let wrapped = PeakRecordingPool::new(Arc::clone(&inner)); + + assert_eq!(wrapped.name(), inner.name()); + assert_eq!(wrapped.to_string(), inner.to_string()); + assert!(matches!(wrapped.memory_limit(), MemoryLimit::Finite(4096))); + } + + /// Arrow-side reservations reach the recorder too. + /// + /// [`ArrowMemoryPool`] implements Arrow's `MemoryPool` by growing a + /// DataFusion [`MemoryReservation`] against the pool it wraps, so a buffer + /// claimed through it lands in `grow` here. Nothing in DataFusion claims + /// buffers yet (see apache/datafusion#22898), but when something does, the + /// bytes show up in this peak without further changes — as long as the + /// adapter is built from the `RuntimeEnv`'s pool, which is the wrapped one. + /// This test pins that. + #[test] + fn records_reservations_arriving_through_the_arrow_adapter() { + use arrow_buffer::MemoryPool as ArrowMemoryPoolTrait; + use datafusion_execution::memory_pool::arrow::ArrowMemoryPool; + + let (recording, pool) = pool(4096); + + let arrow_pool = + ArrowMemoryPool::new(Arc::clone(&pool), MemoryConsumer::new("arrow")); + let reservation = arrow_pool.reserve(1024); + + // The Arrow-side reservation is visible as DataFusion pool usage... + assert_eq!(pool.reserved(), 1024); + assert_eq!(recording.peak_reserved(), 1024); + + // ...and dropping it releases the bytes while the peak is retained. + drop(reservation); + assert_eq!(pool.reserved(), 0); + assert_eq!(recording.peak_reserved(), 1024); + } +} diff --git a/benchmarks/src/util/mod.rs b/benchmarks/src/util/mod.rs index 6dc11c0f425bd..43855ea468ef5 100644 --- a/benchmarks/src/util/mod.rs +++ b/benchmarks/src/util/mod.rs @@ -18,9 +18,11 @@ //! Shared benchmark utilities pub mod latency_object_store; mod memory; +mod memory_pool; mod options; mod run; pub use memory::print_memory_stats; +pub use memory_pool::PeakRecordingPool; pub use options::CommonOpt; pub use run::{BenchQuery, BenchmarkRun, QueryResult}; diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rs index a3e6d2a4c5538..c744d0bf31c7f 100644 --- a/benchmarks/src/util/options.rs +++ b/benchmarks/src/util/options.rs @@ -30,7 +30,7 @@ use datafusion::{ use datafusion_common::{DataFusionError, Result}; use object_store::local::LocalFileSystem; -use super::latency_object_store::LatencyObjectStore; +use super::{latency_object_store::LatencyObjectStore, memory_pool::PeakRecordingPool}; // Common benchmark options (don't use doc comments otherwise this doc // shows up in help files) @@ -125,6 +125,9 @@ impl CommonOpt { ))); } }; + // Record the peak reservation so benchmarks can report it next to + // peak RSS. Purely observational: every call is delegated. + let pool: Arc = Arc::new(PeakRecordingPool::new(pool)); rt_builder = rt_builder .with_memory_pool(pool) .with_disk_manager_builder(DiskManagerBuilder::default()); diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rs index df17674e62961..6c63ceec6423c 100644 --- a/benchmarks/src/util/run.rs +++ b/benchmarks/src/util/run.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use super::memory_pool::PeakRecordingPool; +use datafusion::execution::memory_pool::MemoryPool; use datafusion::{DATAFUSION_VERSION, error::Result}; use datafusion_common::utils::get_available_parallelism; use serde::{Serialize, Serializer}; @@ -22,6 +24,7 @@ use serde_json::Value; use std::{ collections::HashMap, path::Path, + sync::Arc, time::{Duration, SystemTime}, }; @@ -91,6 +94,16 @@ pub struct BenchQuery { #[serde(serialize_with = "serialize_start_time")] start_time: SystemTime, success: bool, + /// Peak [`MemoryPool`] reservation observed while running this query, in + /// bytes. Recorded for failed queries too, since a query that ran out of + /// memory is one whose peak is worth seeing. + /// + /// `None` (and omitted from the JSON) only when the benchmark ran without a + /// memory limit, since there is then no pool to record. + /// + /// [`MemoryPool`]: datafusion::execution::memory_pool::MemoryPool + #[serde(skip_serializing_if = "Option::is_none")] + pool_peak_bytes: Option, } /// Internal representation of a single benchmark query iteration result. pub struct QueryResult { @@ -102,6 +115,10 @@ pub struct BenchmarkRun { context: RunContext, queries: Vec, current_case: Option, + /// The pool queries run against, when one was handed over with + /// [`BenchmarkRun::set_memory_pool`]. Only read through + /// [`BenchmarkRun::peak_recorder`]. + memory_pool: Option>, } impl Default for BenchmarkRun { @@ -117,15 +134,44 @@ impl BenchmarkRun { context: RunContext::new(), queries: vec![], current_case: None, + memory_pool: None, } } + + /// Report the peak reservation of `memory_pool` alongside each query. + /// + /// Call this with the pool of the [`RuntimeEnv`] the queries run against. + /// Has no effect unless a [`PeakRecordingPool`] is installed, which + /// [`CommonOpt::runtime_env_builder`] does whenever a memory limit is + /// configured; without one `pool_peak_bytes` is omitted from the results. + /// + /// Benchmarks that build a runtime per query should call this each time, so + /// each query reports against the pool it actually ran on. + /// + /// [`RuntimeEnv`]: datafusion::execution::runtime_env::RuntimeEnv + /// [`CommonOpt::runtime_env_builder`]: super::CommonOpt::runtime_env_builder + pub fn set_memory_pool(&mut self, memory_pool: &Arc) { + self.memory_pool = Some(Arc::clone(memory_pool)); + } + + /// The recorder in front of the pool set by [`Self::set_memory_pool`]. + fn peak_recorder(&self) -> Option<&PeakRecordingPool> { + PeakRecordingPool::from_pool(self.memory_pool.as_deref()?) + } + /// begin a new case. iterations added after this will be included in the new case pub fn start_new_case(&mut self, id: &str) { + // Give this query its own memory pool reading rather than inheriting + // the high-water mark of the queries that ran before it. + if let Some(recorder) = self.peak_recorder() { + recorder.reset_peak(); + } self.queries.push(BenchQuery { query: id.to_owned(), iterations: vec![], start_time: SystemTime::now(), success: true, + pool_peak_bytes: None, }); if let Some(c) = self.current_case.as_mut() { *c += 1; @@ -135,10 +181,14 @@ impl BenchmarkRun { } /// Write a new iteration to the current case pub fn write_iter(&mut self, elapsed: Duration, row_count: usize) { + // The peak is not reset between iterations, so this ends up holding the + // largest reservation seen across all of them. + let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved); if let Some(idx) = self.current_case { self.queries[idx] .iterations - .push(QueryIter { elapsed, row_count }) + .push(QueryIter { elapsed, row_count }); + self.queries[idx].pool_peak_bytes = pool_peak_bytes; } else { panic!("no cases existed yet"); } @@ -159,8 +209,12 @@ impl BenchmarkRun { /// Mark current query pub fn mark_failed(&mut self) { + // A query that failed under a memory limit wrote no iteration, so this + // is the only chance to record what it had reserved when it gave up. + let pool_peak_bytes = self.peak_recorder().map(PeakRecordingPool::peak_reserved); if let Some(idx) = self.current_case { self.queries[idx].success = false; + self.queries[idx].pool_peak_bytes = pool_peak_bytes; } else { unreachable!("Cannot mark failure: no current case"); } @@ -182,3 +236,87 @@ impl BenchmarkRun { Ok(()) } } + +#[cfg(test)] +mod tests { + use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryConsumer}; + + use super::*; + + fn recording_pool(limit: usize) -> Arc { + Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new( + limit, + )))) + } + + #[test] + fn each_case_reports_its_own_peak() { + let pool = recording_pool(1024); + let mut run = BenchmarkRun::new(); + run.set_memory_pool(&pool); + + run.start_new_case("q1"); + let reservation = MemoryConsumer::new("q1").register(&pool); + reservation.try_grow(600).unwrap(); + run.write_iter(Duration::from_millis(1), 1); + drop(reservation); + + // The second case must not inherit the first case's high-water mark. + run.start_new_case("q2"); + let reservation = MemoryConsumer::new("q2").register(&pool); + reservation.try_grow(100).unwrap(); + run.write_iter(Duration::from_millis(1), 1); + + assert_eq!(run.queries[0].pool_peak_bytes, Some(600)); + assert_eq!(run.queries[1].pool_peak_bytes, Some(100)); + } + + #[test] + fn a_later_pool_replaces_an_earlier_one() { + let first = recording_pool(1024); + let mut run = BenchmarkRun::new(); + run.set_memory_pool(&first); + MemoryConsumer::new("q1") + .register(&first) + .try_grow(600) + .unwrap(); + + // Benchmarks that build a runtime per query hand over the new pool + // before the next case; the reading follows it. + let second = recording_pool(1024); + run.set_memory_pool(&second); + run.start_new_case("q2"); + MemoryConsumer::new("q2") + .register(&second) + .try_grow(100) + .unwrap(); + run.write_iter(Duration::from_millis(1), 1); + + assert_eq!(run.queries[0].pool_peak_bytes, Some(100)); + } + + #[test] + fn a_failed_query_still_reports_its_peak() { + let pool = recording_pool(1024); + let mut run = BenchmarkRun::new(); + run.set_memory_pool(&pool); + + run.start_new_case("q1"); + let reservation = MemoryConsumer::new("q1").register(&pool); + reservation.try_grow(600).unwrap(); + // No `write_iter`: the query failed before completing an iteration. + run.mark_failed(); + + assert_eq!(run.queries[0].pool_peak_bytes, Some(600)); + } + + #[test] + fn the_peak_is_omitted_without_a_recording_pool() { + let mut run = BenchmarkRun::new(); + run.start_new_case("q1"); + run.write_iter(Duration::from_millis(1), 1); + + assert_eq!(run.queries[0].pool_peak_bytes, None); + assert!(!run.to_json().contains("pool_peak_bytes")); + } +} From 30ae8bf34070399966a7015678fe70bf31b3eeda Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:27:17 -0500 Subject: [PATCH 088/109] refactor(proto): move PartitionedFile / FileGroup serde into datafusion-datasource (#24006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #23494. Precursor for #23497 / #23683 (`DataSource` / `FileSource` proto hooks) and for #23752. ## Rationale for this change The protobuf conversions for the file-scan leaf types — `PartitionedFile`, `FileGroup`, `FileRange` — live in `datafusion-proto` as `TryFromProto` impls, because that is historically the only crate that can name both sides (the DataFusion type and the prost message are both foreign to it, hence the `TryFromProto` workaround trait in the first place). That placement means any *other* crate that needs those conversions has to reimplement them. #23683 hits exactly this: a `FileSource` serializing its own scan config needs to encode file groups, so the first cut of that PR grew a private second copy of the `PartitionedFile` wire logic inside `datafusion-datasource`, which can then drift from the central serializer. The same will be true of every source migrated under #23516–#23518. Nothing about these conversions needs `datafusion-proto`: they are plain data, with `ScalarValue` / `Statistics` / `Schema` going through `datafusion-proto-common`. They belong next to the types. ## What changes are included in this PR? - New `datafusion_datasource::proto` module, behind a new `proto` feature on `datafusion-datasource` (off by default; `datafusion-proto` enables it): - `FileRange::try_to_proto` / `try_from_proto` - `PartitionedFile::try_to_proto` / `try_from_proto` - `FileGroup` <-> `protobuf::FileGroup` - `datafusion-proto`'s `TryFromProto` impls for those types become one-line shims delegating to the new impls, so every existing caller keeps working and the two sides cannot disagree. ### Why these are `TryFrom` and not `try_to_proto` hooks `TryFromProto` exists because `datafusion-proto` owns neither side of the conversions it hosts: with both the DataFusion type and the prost message foreign to it, `impl TryFrom for X` is rejected by the orphan rule, so a local trait was the only way to say the same thing. Moving a conversion into the crate that owns the DataFusion type removes that constraint, and `&T` is `#[fundamental]`, so both directions are expressible with the standard trait (checked, not assumed): ```rust impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile // ok impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile // ok impl TryFrom<&[PartitionedFile]> for protobuf::FileGroup // E0117 ``` The last one is why `protobuf::FileGroup`'s *slice* conversion stays a `TryFromProto` shim: `&[PartitionedFile]` is not a type this crate owns, while `&FileGroup` is. Callers inside DataFusion go through `FileGroup`. So the rule this PR sets for the rest of #23494: **plain data uses `TryFrom`; anything needing an encode/decode context keeps the `try_to_proto(ctx)` / `try_from_proto(node, ctx)` hooks**, because the standard trait cannot carry that second argument. Usefully, none of the ~40 `TryFromProto`/`FromProto` impls needs a context, and nothing that needs one was ever a `TryFromProto` impl — the two categories are already disjoint, so the shape now tells a reader whether a conversion recurses. ### Why now `FromProto` / `TryFromProto` were added in #21929, *after* the 54.0.0 release, and 54.1.0 was cut before any of this landed — so they have never shipped in a release. Replacing them with the standard traits, and eventually deleting them, is a no-op for semver **today** and a major breaking change the moment 55.0.0 goes out. The same applies to the six inherent `try_to_proto` / `try_from_proto` methods this PR would otherwise have added: they are new, unreleased API, so choosing their final shape costs nothing right now. The other reason to settle it here rather than in a follow-up: this is the PR that establishes the pattern for the data-source family (#23516-#23519 and #23752 / #23781 are all queued behind it). Whichever shape merges first is the one they will copy. Retiring the remaining ~34 impls is still its own follow-up. Two notes for whoever picks it up: the sink and format-option conversions can move next to their types the same way, but the ones for `datafusion-common`-owned types (`JoinType`, `NullEquality`, `TableReference`, `UnnestOptions`, ...) cannot — `datafusion-common` cannot depend on `datafusion-proto-models` (it is underneath it via `datafusion-proto-common`). Their legal home is `proto-models` itself, implementing on the local proto type, which is already how `proto-common` hosts the `ScalarValue` / `Statistics` conversions. ## Are these changes tested? Yes. - New unit tests in `datafusion_datasource::proto` covering the `PartitionedFile` round trip (path, size, mtime, partition values, range, arrow schema, statistics), the `FileGroup` round trip, and the invalid-path error. - The existing `datafusion-proto` tests now exercise the delegating shims, so they also pin the shims themselves. - `datafusion-proto`, all features: 227 passed / 0 failed. - `datafusion-datasource` with `proto`: 180 passed / 0 failed. - Full workspace run: 10347 passed (`cargo test --profile ci --workspace --lib --tests --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption`). The only failures are 8 backtrace-symbolization tests in `datafusion-common`, a crate this PR does not touch and which sits below every crate it does; they fail the same way on the base commit on macOS. - `cargo fmt` and `ci/scripts/rust_clippy.sh` clean. ## Are there any user-facing changes? The protobuf wire format is unchanged, and no existing API changes shape. Additive: - New `proto` feature on `datafusion-datasource` (off by default). - New `TryFrom` impls in both directions between `FileRange`, `PartitionedFile`, `FileGroup` and their protobuf messages, under that feature. No new names are added to the crate's API surface: the trait is `core::convert::TryFrom`. Note for reviewers: while writing the round-trip test I found that `PartitionedFile` statistics do not round-trip cleanly on `main` — filed as #23998. This PR preserves that behavior exactly rather than changing decode semantics in a refactor; the test documents it. --------- Co-authored-by: Claude Opus 5 --- Cargo.lock | 1 + datafusion/datasource/Cargo.toml | 5 + datafusion/datasource/src/mod.rs | 4 + datafusion/datasource/src/proto.rs | 238 ++++++++++++++++++ datafusion/proto/Cargo.toml | 2 +- .../proto/src/physical_plan/from_proto.rs | 54 +--- .../proto/src/physical_plan/to_proto.rs | 47 +--- 7 files changed, 272 insertions(+), 79 deletions(-) create mode 100644 datafusion/datasource/src/proto.rs diff --git a/Cargo.lock b/Cargo.lock index 451ff70b1cd48..9cd5d12d5bca4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1944,6 +1944,7 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index 2ac42ed900095..459ca436f365d 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -34,6 +34,10 @@ all-features = true backtrace = ["datafusion-common/backtrace"] compression = ["async-compression", "liblzma", "bzip2", "flate2", "zstd", "tokio-util"] default = ["compression"] +# Enables the protobuf conversions for the file-scan leaf types owned by this +# crate (`FileRange`, `PartitionedFile`, `FileGroup`). Off by default so +# consumers that never serialize plans pay nothing. +proto = ["dep:datafusion-proto-models"] [dependencies] arrow = { workspace = true } @@ -56,6 +60,7 @@ datafusion-physical-expr = { workspace = true } datafusion-physical-expr-adapter = { workspace = true } datafusion-physical-expr-common = { workspace = true } datafusion-physical-plan = { workspace = true } +datafusion-proto-models = { workspace = true, optional = true } datafusion-session = { workspace = true } flate2 = { workspace = true, optional = true } futures = { workspace = true } diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rs index 7c8cae337f1eb..e415b3e48a02a 100644 --- a/datafusion/datasource/src/mod.rs +++ b/datafusion/datasource/src/mod.rs @@ -41,6 +41,10 @@ pub mod file_stream; pub mod memory; pub mod morsel; pub mod projection; +/// Protobuf conversions for [`FileRange`], [`PartitionedFile`] and +/// [`FileGroup`](crate::file_groups::FileGroup), gated on the `proto` feature. +#[cfg(feature = "proto")] +mod proto; pub mod schema_adapter; pub mod sink; pub mod source; diff --git a/datafusion/datasource/src/proto.rs b/datafusion/datasource/src/proto.rs new file mode 100644 index 0000000000000..cf48a461655c7 --- /dev/null +++ b/datafusion/datasource/src/proto.rs @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Protobuf conversions for the file-scan leaf types owned by this crate: +//! [`FileRange`], [`PartitionedFile`] and [`FileGroup`]. +//! +//! These are the single copy of that wire logic. `datafusion-proto`'s +//! `TryFromProto` implementations for the same types are thin shims that +//! delegate here, so the format cannot drift between the central serializer and +//! the per-source `try_to_proto` hooks. +//! +//! None of these conversions need a codec or an encode/decode context: every +//! field is plain data or goes through `datafusion-proto-common`. That is why +//! they are plain [`TryFrom`] impls rather than the `try_to_proto(ctx)` / +//! `try_from_proto(node, ctx)` hooks used for plans, expressions and scan +//! configs: the standard trait can express a conversion that takes nothing but +//! the value, and the orphan rule allows it here because one side of each +//! conversion is a type this crate owns. + +use std::sync::Arc; + +use chrono::{TimeZone, Utc}; +use datafusion_common::{DataFusionError, Result, internal_datafusion_err}; +use datafusion_proto_models::protobuf; +use object_store::ObjectMeta; +use object_store::path::Path; + +use crate::file_groups::FileGroup; +use crate::{FileRange, PartitionedFile}; + +impl TryFrom<&FileRange> for protobuf::FileRange { + type Error = DataFusionError; + + fn try_from(range: &FileRange) -> Result { + Ok(protobuf::FileRange { + start: range.start, + end: range.end, + }) + } +} + +impl TryFrom<&protobuf::FileRange> for FileRange { + type Error = DataFusionError; + + fn try_from(range: &protobuf::FileRange) -> Result { + Ok(FileRange { + start: range.start, + end: range.end, + }) + } +} + +impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile { + type Error = DataFusionError; + + fn try_from(file: &PartitionedFile) -> Result { + let last_modified = file.object_meta.last_modified; + let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| { + DataFusionError::Plan(format!( + "Invalid timestamp on PartitionedFile::ObjectMeta: {last_modified}" + )) + })? as u64; + Ok(protobuf::PartitionedFile { + arrow_schema: file + .arrow_schema + .as_ref() + .map(|s| s.as_ref().try_into()) + .transpose()?, + path: file.object_meta.location.as_ref().to_owned(), + size: file.object_meta.size, + last_modified_ns, + partition_values: file + .partition_values + .iter() + .map(|v| v.try_into()) + .collect::, _>>()?, + range: file.range.as_ref().map(TryInto::try_into).transpose()?, + statistics: file.statistics.as_ref().map(|s| s.as_ref().into()), + }) + } +} + +impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile { + type Error = DataFusionError; + + fn try_from(file: &protobuf::PartitionedFile) -> Result { + let mut pf = PartitionedFile::new_from_meta(ObjectMeta { + location: Path::parse(file.path.as_str()).map_err(|e| { + internal_datafusion_err!("Invalid object_store path: {e}") + })?, + last_modified: Utc.timestamp_nanos(file.last_modified_ns as i64), + size: file.size, + e_tag: None, + version: None, + }) + .with_partition_values( + file.partition_values + .iter() + .map(|v| v.try_into()) + .collect::, _>>()?, + ); + if let Some(proto_schema) = file.arrow_schema.as_ref() { + pf = pf.with_arrow_schema(Arc::new( + proto_schema.try_into().map_err(DataFusionError::from)?, + )); + } + if let Some(range) = file.range.as_ref() { + let range = FileRange::try_from(range)?; + pf = pf.with_range(range.start, range.end); + } + if let Some(proto_stats) = file.statistics.as_ref() { + // The wire format carries statistics for the full table schema (file + partition + // columns), so assign directly — `with_statistics` would append the partition + // column stats a second time. + pf.statistics = Some(Arc::new(proto_stats.try_into()?)); + } + Ok(pf) + } +} + +impl TryFrom<&FileGroup> for protobuf::FileGroup { + type Error = DataFusionError; + + fn try_from(group: &FileGroup) -> Result { + Ok(protobuf::FileGroup { + files: group + .files() + .iter() + .map(TryInto::try_into) + .collect::>>()?, + }) + } +} + +impl TryFrom<&protobuf::FileGroup> for FileGroup { + type Error = DataFusionError; + + fn try_from(group: &protobuf::FileGroup) -> Result { + Ok(FileGroup::new( + group + .files + .iter() + .map(TryInto::try_into) + .collect::>>()?, + )) + } +} + +#[cfg(test)] +mod tests { + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_common::{ScalarValue, Statistics}; + + use super::*; + + #[test] + fn partitioned_file_roundtrip_preserves_all_fields() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); + let pf = PartitionedFile::new_from_meta(ObjectMeta { + location: Path::parse("foo/bar.parquet")?, + last_modified: Utc.timestamp_nanos(1_000_000_000), + size: 1234, + e_tag: None, + version: None, + }) + .with_partition_values(vec![ScalarValue::from("2024-01-01")]) + .with_range(10, 20) + .with_arrow_schema(Arc::clone(&schema)) + .with_statistics(Arc::new(Statistics::new_unknown(&schema))); + + let encoded = protobuf::PartitionedFile::try_from(&pf)?; + let decoded = PartitionedFile::try_from(&encoded)?; + + assert_eq!(decoded.object_meta.location, pf.object_meta.location); + assert_eq!(decoded.object_meta.size, pf.object_meta.size); + assert_eq!( + decoded.object_meta.last_modified, + pf.object_meta.last_modified + ); + assert_eq!(decoded.partition_values, pf.partition_values); + assert_eq!(decoded.range, pf.range); + assert_eq!(decoded.arrow_schema.as_deref(), Some(schema.as_ref())); + // Statistics span the full table schema (file columns followed by one + // entry per partition column), and survive the round trip intact. + assert_eq!( + pf.statistics.as_ref().unwrap().column_statistics.len(), + schema.fields().len() + pf.partition_values.len() + ); + assert_eq!(decoded.statistics, pf.statistics); + Ok(()) + } + + #[test] + fn partitioned_file_from_proto_rejects_invalid_path() { + let proto = protobuf::PartitionedFile { + path: "foo//bar.parquet".to_string(), + ..Default::default() + }; + + let err = PartitionedFile::try_from(&proto).unwrap_err(); + assert!( + err.to_string().contains("Invalid object_store path"), + "unexpected error: {err}" + ); + } + + #[test] + fn file_group_roundtrip() -> Result<()> { + let group = FileGroup::new(vec![ + PartitionedFile::new("a.parquet", 1), + PartitionedFile::new("b.parquet", 2), + ]); + + let encoded = protobuf::FileGroup::try_from(&group)?; + let decoded = FileGroup::try_from(&encoded)?; + + assert_eq!(decoded.len(), 2); + assert_eq!( + decoded.files()[1].object_meta.location, + group.files()[1].object_meta.location + ); + Ok(()) + } +} diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index cfff8a949418a..037be27769f4d 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -54,7 +54,7 @@ chrono = { workspace = true } datafusion-catalog = { workspace = true } datafusion-catalog-listing = { workspace = true } datafusion-common = { workspace = true } -datafusion-datasource = { workspace = true } +datafusion-datasource = { workspace = true, features = ["proto"] } datafusion-datasource-arrow = { workspace = true } datafusion-datasource-avro = { workspace = true, optional = true } datafusion-datasource-csv = { workspace = true } diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs index 7aa6376313c96..34ad8c7a62fc7 100644 --- a/datafusion/proto/src/physical_plan/from_proto.rs +++ b/datafusion/proto/src/physical_plan/from_proto.rs @@ -23,7 +23,6 @@ use arrow::array::RecordBatch; use arrow::compute::SortOptions; use arrow::datatypes::{Field, Schema}; use arrow::ipc::reader::StreamReader; -use chrono::{TimeZone, Utc}; use datafusion_common::{ DataFusionError, Result, ScalarValue, internal_datafusion_err, not_impl_err, }; @@ -56,8 +55,6 @@ use datafusion_physical_plan::{ Partitioning, PhysicalExpr, RangePartitioning, SplitPoint, WindowExpr, }; use datafusion_proto_common::common::proto_error; -use object_store::ObjectMeta; -use object_store::path::Path; use super::{ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalPlanDecodeContext, @@ -632,64 +629,30 @@ pub fn parse_record_batches(buf: &[u8]) -> Result> { Ok(batches) } +/// Thin shim over `TryFrom<&protobuf::PartitionedFile>`, which owns the wire logic. impl TryFromProto<&protobuf::PartitionedFile> for PartitionedFile { type Error = DataFusionError; fn try_from_proto(val: &protobuf::PartitionedFile) -> Result { - let mut pf = PartitionedFile::new_from_meta(ObjectMeta { - location: Path::parse(val.path.as_str()) - .map_err(|e| proto_error(format!("Invalid object_store path: {e}")))?, - last_modified: Utc.timestamp_nanos(val.last_modified_ns as i64), - size: val.size, - e_tag: None, - version: None, - }) - .with_partition_values( - val.partition_values - .iter() - .map(|v| v.try_into()) - .collect::, _>>()?, - ); - if let Some(proto_schema) = val.arrow_schema.as_ref() { - pf = pf.with_arrow_schema(Arc::new( - proto_schema.try_into().map_err(DataFusionError::from)?, - )); - } - if let Some(range) = val.range.as_ref() { - let file_range = FileRange::try_from_proto(range)?; - pf = pf.with_range(file_range.start, file_range.end); - } - if let Some(proto_stats) = val.statistics.as_ref() { - // The wire format carries statistics for the full table schema (file + partition - // columns), so assign directly — `with_statistics` would append the partition - // column stats a second time. - pf.statistics = Some(Arc::new(proto_stats.try_into()?)); - } - Ok(pf) + PartitionedFile::try_from(val) } } +/// Thin shim over `TryFrom<&protobuf::FileRange>`, which owns the wire logic. impl TryFromProto<&protobuf::FileRange> for FileRange { type Error = DataFusionError; fn try_from_proto(value: &protobuf::FileRange) -> Result { - Ok(FileRange { - start: value.start, - end: value.end, - }) + FileRange::try_from(value) } } +/// Thin shim over `TryFrom<&protobuf::FileGroup>`, which owns the wire logic. impl TryFromProto<&protobuf::FileGroup> for FileGroup { type Error = DataFusionError; fn try_from_proto(val: &protobuf::FileGroup) -> Result { - let files = val - .files - .iter() - .map(PartitionedFile::try_from_proto) - .collect::, _>>()?; - Ok(FileGroup::new(files)) + FileGroup::try_from(val) } } @@ -734,7 +697,7 @@ impl TryFromProto<&protobuf::FileSinkConfig> for FileSinkConfig { let file_group = FileGroup::new( conf.file_groups .iter() - .map(PartitionedFile::try_from_proto) + .map(TryInto::try_into) .collect::>>()?, ); let table_paths = conf @@ -812,6 +775,9 @@ impl datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprD mod tests { use super::*; use arrow::datatypes::{DataType, Field, Schema}; + use chrono::{TimeZone, Utc}; + use object_store::ObjectMeta; + use object_store::path::Path; #[test] fn partitioned_file_path_roundtrip_percent_encoded() { diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs index e13923dbb9519..5189972f0e200 100644 --- a/datafusion/proto/src/physical_plan/to_proto.rs +++ b/datafusion/proto/src/physical_plan/to_proto.rs @@ -424,51 +424,30 @@ fn serialize_range_split_point( }) } +/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic. impl TryFromProto<&PartitionedFile> for protobuf::PartitionedFile { type Error = DataFusionError; fn try_from_proto(pf: &PartitionedFile) -> Result { - let last_modified = pf.object_meta.last_modified; - let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| { - DataFusionError::Plan(format!( - "Invalid timestamp on PartitionedFile::ObjectMeta: {last_modified}" - )) - })? as u64; - Ok(protobuf::PartitionedFile { - arrow_schema: pf - .arrow_schema - .as_ref() - .map(|s| s.as_ref().try_into()) - .transpose()?, - path: pf.object_meta.location.as_ref().to_owned(), - size: pf.object_meta.size, - last_modified_ns, - partition_values: pf - .partition_values - .iter() - .map(|v| v.try_into()) - .collect::, _>>()?, - range: pf - .range - .as_ref() - .map(protobuf::FileRange::try_from_proto) - .transpose()?, - statistics: pf.statistics.as_ref().map(|s| s.as_ref().into()), - }) + pf.try_into() } } +/// Thin shim over `TryFrom<&FileRange>`, which owns the wire logic. impl TryFromProto<&FileRange> for protobuf::FileRange { type Error = DataFusionError; fn try_from_proto(value: &FileRange) -> Result { - Ok(protobuf::FileRange { - start: value.start, - end: value.end, - }) + value.try_into() } } +/// Thin shim over `TryFrom<&PartitionedFile>`, which owns the wire logic. +/// +/// The slice form cannot be a `TryFrom` impl: the orphan rule only accepts a +/// type this crate owns, and `&[PartitionedFile]` is not one (`&FileGroup` is, +/// hence the impl next to the type). Callers inside DataFusion go through +/// `FileGroup`; this stays for downstream users of the published signature. impl TryFromProto<&[PartitionedFile]> for protobuf::FileGroup { type Error = DataFusionError; @@ -476,8 +455,8 @@ impl TryFromProto<&[PartitionedFile]> for protobuf::FileGroup { Ok(protobuf::FileGroup { files: gr .iter() - .map(protobuf::PartitionedFile::try_from_proto) - .collect::, _>>()?, + .map(TryInto::try_into) + .collect::>>()?, }) } } @@ -490,7 +469,7 @@ pub fn serialize_file_scan_config( let file_groups = conf .file_groups .iter() - .map(|p| protobuf::FileGroup::try_from_proto(p.files())) + .map(TryInto::try_into) .collect::, _>>()?; let mut output_orderings = vec![]; From 1ae6b872aa222e3f88c9a85aa39b6314f245e9d6 Mon Sep 17 00:00:00 2001 From: Neil Conway Date: Thu, 30 Jul 2026 14:28:51 -0400 Subject: [PATCH 089/109] perf: Replace SipHash with foldhash in `BoundedWindowAggExec` (#23984) ## Which issue does this PR close? - Closes: #23983 ## Rationale for this change `PartitionWindowAggStates` and `PartitionBatches` were using IndexMap's default SipHash hasher. Switching to foldhash yields a small but measurable performance improvement. Benchmark below. Note that the improvement is modest in part because linear mode is very inefficient when there are many partitions (#23982); I will send a PR for that separately. Benchmarks: - linear, 10,000 partitions: 216.7 ms -> 204.1 ms (~6%) - linear, 100 partitions: 45.0 ms -> 45.4 ms (within drift) - sorted, 10,000 partitions: 35.0 ms -> 34.8 ms (control) ## What changes are included in this PR? * Add new benchmark * Switch from using SipHash to foldhash ## Are these changes tested? Yes, covered by existing tests. ## Are there any user-facing changes? No functional change at all. --- .../physical-expr/src/window/window_expr.rs | 5 +- datafusion/physical-plan/Cargo.toml | 4 + .../physical-plan/benches/bounded_window.rs | 183 ++++++++++++++++++ .../src/windows/bounded_window_agg_exec.rs | 4 +- 4 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 datafusion/physical-plan/benches/bounded_window.rs diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 0f0ec647a50ae..8db5651346e8f 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -30,6 +30,7 @@ use arrow::compute::kernels::sort::SortColumn; use arrow::datatypes::FieldRef; use arrow::record_batch::RecordBatch; use datafusion_common::cast::as_boolean_array; +use datafusion_common::hash_utils::RandomState; use datafusion_common::utils::compare_rows; use datafusion_common::{ Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, internal_err, @@ -609,10 +610,10 @@ pub struct WindowState { pub state: WindowAggState, pub window_fn: WindowFn, } -pub type PartitionWindowAggStates = IndexMap; +pub type PartitionWindowAggStates = IndexMap; /// The IndexMap (i.e. an ordered HashMap) where record batches are separated for each partition. -pub type PartitionBatches = IndexMap; +pub type PartitionBatches = IndexMap; #[cfg(test)] mod tests { diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 58c2f0d7da537..0f72b74840d01 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -143,3 +143,7 @@ required-features = ["test_utils"] harness = false name = "multi_group_by" required-features = ["test_utils"] + +[[bench]] +harness = false +name = "bounded_window" diff --git a/datafusion/physical-plan/benches/bounded_window.rs b/datafusion/physical-plan/benches/bounded_window.rs new file mode 100644 index 0000000000000..f704a86287163 --- /dev/null +++ b/datafusion/physical-plan/benches/bounded_window.rs @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmark for `BoundedWindowAggExec` with many partitions. +//! +//! The streaming window operator keeps per-partition state keyed by +//! `PartitionKey` (`Vec`) and probes it for every buffered +//! partition on every batch, so its performance is sensitive to both the +//! number of live partitions and the cost of hashing the keys. `Linear` +//! mode (input sorted by the ORDER BY column but not by the partition +//! columns) keeps every partition live until the input is exhausted and is +//! the stress case; `Sorted` mode prunes finished partitions eagerly and +//! serves as the control. + +use std::sync::Arc; + +use arrow::array::UInt64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::ScalarValue; +use datafusion_execution::TaskContext; +use datafusion_expr::{ + WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, +}; +use datafusion_functions_aggregate::count::count_udaf; +use datafusion_physical_expr::expressions::col; +use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr}; +use datafusion_physical_plan::test::TestMemoryExec; +use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr}; +use datafusion_physical_plan::{ExecutionPlan, InputOrderMode, collect}; + +const BATCH_SIZE: usize = 8192; +const N_BATCHES: usize = 16; + +fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("pk", DataType::UInt64, false), + Field::new("ts", DataType::UInt64, false), + ])) +} + +/// Batches with `ts` ascending across the whole input. When +/// `partitions_sorted` is false, partition keys round-robin over +/// `n_partitions` (the `Linear` layout); when true, the input is laid out +/// partition-by-partition (the `Sorted` layout). +fn make_batches(n_partitions: usize, partitions_sorted: bool) -> Vec { + let total = BATCH_SIZE * N_BATCHES; + let rows_per_partition = total / n_partitions; + (0..N_BATCHES) + .map(|b| { + let start = b * BATCH_SIZE; + let pk: UInt64Array = (start..start + BATCH_SIZE) + .map(|i| { + if partitions_sorted { + Some((i / rows_per_partition) as u64) + } else { + Some((i % n_partitions) as u64) + } + }) + .collect(); + let ts: UInt64Array = (start..start + BATCH_SIZE) + .map(|i| Some(i as u64)) + .collect(); + RecordBatch::try_new(schema(), vec![Arc::new(pk), Arc::new(ts)]).unwrap() + }) + .collect() +} + +fn sort_expr(name: &str) -> PhysicalSortExpr { + PhysicalSortExpr { + expr: col(name, &schema()).unwrap(), + options: Default::default(), + } +} + +/// `COUNT(ts) OVER (PARTITION BY pk ORDER BY ts +/// RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING)` +fn window_exec( + batches: Vec, + mode: InputOrderMode, + input_ordering: Vec, +) -> Arc { + let schema = schema(); + let source = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None) + .expect("memory exec") + .try_with_sort_information(LexOrdering::new(input_ordering).into_iter().collect()) + .expect("sort information"); + let input = Arc::new(TestMemoryExec::update_cache(&Arc::new(source))); + let args = vec![col("ts", &schema).unwrap()]; + let partitionby_exprs = vec![col("pk", &schema).unwrap()]; + let orderby_exprs = vec![PhysicalSortExpr { + expr: col("ts", &schema).unwrap(), + options: Default::default(), + }]; + let window_frame = WindowFrame::new_bounds( + WindowFrameUnits::Range, + WindowFrameBound::CurrentRow, + WindowFrameBound::Following(ScalarValue::UInt64(Some(10))), + ); + let window_expr = create_window_expr( + &WindowFunctionDefinition::AggregateUDF(count_udaf()), + "count".to_string(), + &args, + &partitionby_exprs, + &orderby_exprs, + Arc::new(window_frame), + input.schema(), + false, + false, + None, + ) + .expect("window expr"); + Arc::new( + BoundedWindowAggExec::try_new(vec![window_expr], input, mode, true) + .expect("bounded window exec"), + ) +} + +fn bounded_window_benchmark(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut group = c.benchmark_group("bounded_window_partitions"); + group.sample_size(10); + + for n_partitions in [100, 10_000] { + let plan = window_exec( + make_batches(n_partitions, false), + InputOrderMode::Linear, + vec![sort_expr("ts")], + ); + group.bench_function(format!("linear {n_partitions} partitions"), |b| { + b.iter(|| { + let task_ctx = Arc::new(TaskContext::default()); + let batches = rt + .block_on(collect(Arc::clone(&plan), task_ctx)) + .expect("execution"); + assert_eq!( + batches.iter().map(|b| b.num_rows()).sum::(), + BATCH_SIZE * N_BATCHES + ); + }) + }); + } + + // Control: the same query over partition-sorted input, where finished + // partitions are pruned eagerly and the state maps stay small. + let plan = window_exec( + make_batches(10_000, true), + InputOrderMode::Sorted, + vec![sort_expr("pk"), sort_expr("ts")], + ); + group.bench_function("sorted 10000 partitions", |b| { + b.iter(|| { + let task_ctx = Arc::new(TaskContext::default()); + let batches = rt + .block_on(collect(Arc::clone(&plan), task_ctx)) + .expect("execution"); + assert_eq!( + batches.iter().map(|b| b.num_rows()).sum::(), + BATCH_SIZE * N_BATCHES + ); + }) + }); + + group.finish(); +} + +criterion_group!(benches, bounded_window_benchmark); +criterion_main!(benches); diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs index d5863080895f6..07751a70eceeb 100644 --- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs @@ -1054,13 +1054,13 @@ impl BoundedWindowAggStream { baseline_metrics: BaselineMetrics, search_mode: Box, ) -> Result { - let state = window_expr.iter().map(|_| IndexMap::new()).collect(); + let state = window_expr.iter().map(|_| IndexMap::default()).collect(); let empty_batch = RecordBatch::new_empty(Arc::clone(&schema)); Ok(Self { schema, input, input_buffer: empty_batch, - partition_buffers: IndexMap::new(), + partition_buffers: IndexMap::default(), window_agg_states: state, finished: false, window_expr, From 39d50641424dfbc9fadad9cad17619c64d06076b Mon Sep 17 00:00:00 2001 From: Raz Luvaton <16746759+rluvaton@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:58:45 +0300 Subject: [PATCH 090/109] chore: refactor `MaterializingSortMergeJoinStream` into generators and simplify code to be textbook like as possible (#23976) ## Which issue does this PR close? Part of: - https://github.com/apache/datafusion/issues/23974 ## Rationale for this change Simplify the code and make code similar to textbook so it is easier to read ## What changes are included in this PR? Changed to async generators + simplified to be similar to textbook Most of the code was written by Claude Fable 5 since it was just too large to comprehend in head ## Are these changes tested? existing tests + a little more tests to verify we are not counting the childs in the join time ## Are there any user-facing changes? The join_time now includes the time to read from the async spill stream between pending which is arguable more correct since this time is part of the operator, although long waits between pending calls will be counted in the op `join_time` while the alternative is not counting the read from file and decoding... --- .../src/joins/sort_merge_join/exec.rs | 10 +- .../sort_merge_join/materializing_stream.rs | 1193 ++++++++--------- .../src/joins/sort_merge_join/tests.rs | 141 ++ 3 files changed, 720 insertions(+), 624 deletions(-) diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs index 3b597323b2e7b..1abcd9d6c7ce4 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs @@ -81,8 +81,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen /// on the output batch size of the execution plan. There is no spilling support for streamed input. /// The comparisons are performed from values of join keys in streamed input with the values of /// join keys in buffered input. One row in streamed record batch could be matched with multiple rows in -/// buffered input batches. The streamed input is managed through the states in `StreamedState` -/// and streamed input batches are represented by `StreamedBatch`. +/// buffered input batches. Streamed input batches are represented by `StreamedBatch`. /// /// Buffered input is buffered for all record batches having the same value of join key. /// If the memory limit increases beyond the specified value and spilling is enabled, @@ -92,8 +91,7 @@ use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequiremen /// memory/disk depends on the number of rows of buffered input having the same value /// of join key as that of streamed input rows currently present in memory. Due to pre-sorted inputs, /// the algorithm understands when it is not needed anymore, and releases the buffered batches -/// from memory/disk. The buffered input is managed through the states in `BufferedState` -/// and buffered input batches are represented by `BufferedBatch`. +/// from memory/disk. Buffered input batches are represented by `BufferedBatch`. /// /// Depending on the type of join, left or right input may be selected as streamed or buffered /// respectively. For example, in a left-outer join, the left execution plan will be selected as @@ -546,7 +544,7 @@ impl ExecutionPlan for SortMergeJoinExec { context.runtime_env(), ) } else { - Ok(Box::pin(MaterializingSortMergeJoinStream::try_new( + MaterializingSortMergeJoinStream::try_new( Arc::clone(&self.schema), self.sort_options.clone(), self.null_equality, @@ -561,7 +559,7 @@ impl ExecutionPlan for SortMergeJoinExec { reservation, spill_manager, context.runtime_env(), - )?)) + ) } } diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs index 51cf38b9ab1f7..3baa0c4a3e792 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs @@ -17,20 +17,17 @@ //! Sort-Merge Join execution //! -//! This module implements the runtime state machine for the Sort-Merge Join -//! operator. It drives two sorted input streams (the *streamed* side and the -//! *buffered* side), compares join keys, and produces joined `RecordBatch`es. +//! This module implements the Sort-Merge Join operator as an async +//! generator running a merge scan: it drives two sorted input streams (the +//! *streamed* side and the *buffered* side), compares join keys, and +//! produces joined `RecordBatch`es. use std::cmp::Ordering; use std::collections::{HashMap, VecDeque}; use std::fmt::Debug; use std::mem::size_of; use std::ops::Range; -use std::pin::Pin; use std::sync::Arc; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering::Relaxed; -use std::task::{Context, Poll}; use crate::joins::sort_merge_join::filter::{ FilterMetadata, filter_record_batch_by_join_type, get_corrected_filter_mask, @@ -38,10 +35,10 @@ use crate::joins::sort_merge_join::filter::{ }; use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics; use crate::joins::utils::{JoinFilter, JoinKeyComparator}; -use crate::metrics::RecordOutput; +use crate::metrics::Time; use crate::spill::spill_manager::SpillManager; -use crate::stream::EmptyRecordBatchStream; -use crate::{PhysicalExpr, RecordBatchStream, SendableRecordBatchStream}; +use crate::stream::{EmptyRecordBatchStream, ObservedStream, RecordBatchStreamAdapter}; +use crate::{PhysicalExpr, SendableRecordBatchStream}; use arrow::array::{types::UInt64Type, *}; use arrow::compute::{ @@ -50,56 +47,16 @@ use arrow::compute::{ }; use arrow::datatypes::SchemaRef; use datafusion_common::cast::as_uint64_array; -use datafusion_common::{JoinType, NullEquality, Result, exec_err, internal_err}; -use datafusion_execution::SpillFile; +use datafusion_common::instant::Instant; +use datafusion_common::{ + DataFusionError, JoinType, NullEquality, Result, exec_err, internal_err, +}; use datafusion_execution::memory_pool::MemoryReservation; use datafusion_execution::runtime_env::RuntimeEnv; +use datafusion_execution::{SpillFile, TryEmitter, async_try_stream}; use datafusion_physical_expr_common::physical_expr::PhysicalExprRef; -use futures::{Stream, StreamExt, ready}; - -/// State of SMJ stream -#[derive(Debug, PartialEq, Eq)] -pub(super) enum SortMergeJoinState { - /// Init joining with a new streamed row or a new buffered batches - Init, - /// Polling one streamed row or one buffered batch, or both - Polling, - /// Joining polled data and making output - JoinOutput, - /// Emit ready data if have any and then go back to [`Self::Init`] state - EmitReadyThenInit, - /// No more output - Exhausted, -} - -/// State of streamed data stream -#[derive(Debug, PartialEq, Eq)] -pub(super) enum StreamedState { - /// Init polling - Init, - /// Polling one streamed row - Polling, - /// Ready to produce one streamed row - Ready, - /// No more streamed row - Exhausted, -} - -/// State of buffered data stream -#[derive(Debug, PartialEq, Eq)] -pub(super) enum BufferedState { - /// Init polling - Init, - /// Polling first row in the next batch - PollingFirst, - /// Polling rest rows in the next batch - PollingRest, - /// Ready to produce one batch - Ready, - /// No more buffered batches - Exhausted, -} +use futures::StreamExt; /// Represents a chunk of joined data from streamed and buffered side pub(super) struct StreamedJoinedChunk { @@ -335,6 +292,9 @@ pub(super) struct MaterializingSortMergeJoinStream { pub filter: Option, /// How the join is performed pub join_type: JoinType, + /// Cached `needs_deferred_filtering(filter, join_type)` — both inputs + /// are fixed at construction time. + pub deferred_filtering: bool, /// Target output batch size pub batch_size: usize, @@ -348,10 +308,8 @@ pub(super) struct MaterializingSortMergeJoinStream { pub streamed: SendableRecordBatchStream, /// Current processing record batch of streamed pub streamed_batch: StreamedBatch, - /// (used in outer join) Is current streamed row joined at least once? - pub streamed_joined: bool, - /// State of streamed - pub streamed_state: StreamedState, + /// True once the streamed input has no more rows + pub streamed_exhausted: bool, /// Join key columns of streamed pub on_streamed: Vec, @@ -365,10 +323,11 @@ pub(super) struct MaterializingSortMergeJoinStream { pub buffered: SendableRecordBatchStream, /// Current buffered data pub buffered_data: BufferedData, - /// (used in outer join) Is current buffered batches joined at least once? - pub buffered_joined: bool, - /// State of buffered - pub buffered_state: BufferedState, + /// Has any streamed row matched the current buffered key group? + /// (FULL join: an unmatched group is emitted null-joined when passed.) + pub buffered_group_matched: bool, + /// True once the buffered input has no more rows and no group remains + pub buffered_exhausted: bool, /// Join key columns of buffered pub on_buffered: Vec, @@ -377,23 +336,26 @@ pub(super) struct MaterializingSortMergeJoinStream { // These fields track the execution state of merge join and are updated // during the execution. // ======================================================================== - /// Current state of the stream - pub state: SortMergeJoinState, /// Staging output array builders pub joined_record_batches: JoinedRecordBatches, /// Output buffer. Currently used by filtering as it requires double buffering - /// to avoid small/empty batches. Non-filtered join outputs directly from `staging_output_record_batches.batches` + /// to avoid small/empty batches. Non-filtered joins output directly from + /// `joined_record_batches.joined_batches` pub output: BatchCoalescer, - /// The comparison result of current streamed row and buffered batches - pub current_ordering: Ordering, /// Manages the process of spilling and reading back intermediate data pub spill_manager: SpillManager, - /// Tracks the active stream when loading spilled buffered batches back in memory - pub spill_stream: Option, /// Tracks the number of batches currently spilled pub spilled_batch_count: usize, + /// Time spent doing the join's own work (including spill write and + /// read-back). The clock is stopped while awaiting the child inputs or + /// the consumer taking an emitted batch — see [`Self::stop_join_time`]. + pub join_time: Time, + /// Start of the currently running `join_time` span; `None` while the + /// clock is stopped. + pub join_time_start: Option, + // ======================================================================== // CACHED COMPARATORS: // Pre-built comparators to avoid per-row type dispatch in hot loops. @@ -413,8 +375,9 @@ pub(super) struct MaterializingSortMergeJoinStream { pub reservation: MemoryReservation, /// Runtime env pub runtime_env: Arc, - /// A unique number for each batch - pub streamed_batch_counter: AtomicUsize, + /// A unique id per streamed batch, tagging deferred-filter metadata so + /// `get_corrected_filter_mask` can group output rows by input batch. + pub streamed_batch_counter: usize, } /// Staging area for joined data before output @@ -560,264 +523,6 @@ impl JoinedRecordBatches { self.debug_assert_empty_consistency(); } } -impl RecordBatchStream for MaterializingSortMergeJoinStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} - -impl Stream for MaterializingSortMergeJoinStream { - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let join_time = self.join_metrics.join_time().clone(); - let _timer = join_time.timer(); - loop { - match &self.state { - SortMergeJoinState::Init => { - let streamed_exhausted = - self.streamed_state == StreamedState::Exhausted; - let buffered_exhausted = - self.buffered_state == BufferedState::Exhausted; - self.state = if streamed_exhausted && buffered_exhausted { - SortMergeJoinState::Exhausted - } else { - match self.current_ordering { - Ordering::Less | Ordering::Equal => { - if !streamed_exhausted { - // Batch deferred filtering: process_filtered_batches() - // only when >= batch_size rows have accumulated. - // Without this gate, unique keys cause per-row pipeline - // execution (concat + correct_mask + filter_by_type), - // which dominates runtime. - // - // Accumulated rows are bounded to ~2*batch_size: - // one batch_size worth from freeze_dequeuing_buffered() - // (when an input batch is fully consumed), plus up to - // batch_size pairs accumulating toward the next freeze. - // This does not reintroduce the unbounded buffering - // fixed by PR #20482. Exhausted state flushes remainder. - if needs_deferred_filtering( - &self.filter, - self.join_type, - ) { - let accumulated = self.num_unfrozen_pairs() - + self - .joined_record_batches - .filter_metadata - .filter_mask - .len(); - if accumulated >= self.batch_size { - // Ensure required spilled batches are restored to memory - // before processing, as this path invokes freeze_all(). - let needed = self.get_required_batch_indices( - self.buffered_data.batches.len(), - ); - if let Err(e) = ready!( - self.poll_spilled_batches(cx, &needed) - ) { - return Poll::Ready(Some(Err(e))); - } - match self.process_filtered_batches()? { - Poll::Ready(Some(batch)) => { - return Poll::Ready(Some(Ok(batch))); - } - Poll::Ready(None) | Poll::Pending => {} - } - } - } - - self.streamed_joined = false; - self.streamed_state = StreamedState::Init; - } - } - Ordering::Greater => { - if !buffered_exhausted { - self.buffered_joined = false; - self.buffered_state = BufferedState::Init; - } - } - } - SortMergeJoinState::Polling - }; - } - SortMergeJoinState::Polling => { - if ![StreamedState::Exhausted, StreamedState::Ready] - .contains(&self.streamed_state) - { - match self.poll_streamed_row(cx)? { - Poll::Ready(_) => {} - Poll::Pending => return Poll::Pending, - } - } - - if ![BufferedState::Exhausted, BufferedState::Ready] - .contains(&self.buffered_state) - { - match self.poll_buffered_batches(cx)? { - Poll::Ready(_) => {} - Poll::Pending => return Poll::Pending, - } - } - let streamed_exhausted = - self.streamed_state == StreamedState::Exhausted; - let buffered_exhausted = - self.buffered_state == BufferedState::Exhausted; - if streamed_exhausted && buffered_exhausted { - self.state = SortMergeJoinState::Exhausted; - continue; - } - self.current_ordering = self.compare_streamed_buffered()?; - self.state = SortMergeJoinState::JoinOutput; - } - SortMergeJoinState::EmitReadyThenInit => { - // If have data to emit, emit it and if no more, change to next - - // Verify metadata alignment before checking if we have batches to output - self.joined_record_batches - .filter_metadata - .debug_assert_metadata_aligned(); - - // For filtered joins, skip output and let Init state handle it - if needs_deferred_filtering(&self.filter, self.join_type) { - self.state = SortMergeJoinState::Init; - continue; - } - - // For non-filtered joins, only output if we have a completed batch - // (opportunistic output when target batch size is reached) - if self - .joined_record_batches - .joined_batches - .has_completed_batch() - { - let record_batch = self - .joined_record_batches - .joined_batches - .next_completed_batch() - .expect("has_completed_batch was true"); - (&record_batch) - .record_output(&self.join_metrics.baseline_metrics()); - return Poll::Ready(Some(Ok(record_batch))); - } - self.state = SortMergeJoinState::Init; - } - SortMergeJoinState::JoinOutput => { - // If the batch size limit is reached, restore required spilled batches to memory and freeze. - // Guarding at the top of the loop safely handles re-entry from Poll::Pending. - if self.num_unfrozen_pairs() >= self.batch_size { - let needed = self - .get_required_batch_indices(self.buffered_data.batches.len()); - ready!(self.poll_spilled_batches(cx, &needed))?; - - self.freeze_all()?; - - // Verify metadata alignment before checking if we have batches to output - self.joined_record_batches - .filter_metadata - .debug_assert_metadata_aligned(); - - // For filtered joins, skip output and let Init state handle it - if needs_deferred_filtering(&self.filter, self.join_type) { - continue; - } - - // For non-filtered joins, only output if we have a completed batch - if self - .joined_record_batches - .joined_batches - .has_completed_batch() - { - let record_batch = self - .joined_record_batches - .joined_batches - .next_completed_batch() - .expect("has_completed_batch was true"); - (&record_batch) - .record_output(&self.join_metrics.baseline_metrics()); - return Poll::Ready(Some(Ok(record_batch))); - } - - // Otherwise keep buffering (don't output yet) - continue; - } - - self.join_partial()?; - - if self.num_unfrozen_pairs() < self.batch_size - && self.buffered_data.scanning_finished() - { - self.buffered_data.scanning_reset(); - self.state = SortMergeJoinState::EmitReadyThenInit; - } - // Note: If join_partial() reached the batch size, the loop repeats to freeze the data. - } - SortMergeJoinState::Exhausted => { - let needed = - self.get_required_batch_indices(self.buffered_data.batches.len()); - ready!(self.poll_spilled_batches(cx, &needed))?; - - self.freeze_all()?; - - // Verify metadata alignment before final output - self.joined_record_batches - .filter_metadata - .debug_assert_metadata_aligned(); - - // For filtered joins, must concat and filter ALL data at once - if needs_deferred_filtering(&self.filter, self.join_type) - && !self.joined_record_batches.joined_batches.is_empty() - { - let record_batch = self.filter_joined_batch()?; - (&record_batch) - .record_output(&self.join_metrics.baseline_metrics()); - return Poll::Ready(Some(Ok(record_batch))); - } - - // For non-filtered joins, finish buffered data first - if !self.joined_record_batches.joined_batches.is_empty() { - self.joined_record_batches - .joined_batches - .finish_buffered_batch()?; - } - - // Output one completed batch at a time (stay in Exhausted until empty) - if self - .joined_record_batches - .joined_batches - .has_completed_batch() - { - let record_batch = self - .joined_record_batches - .joined_batches - .next_completed_batch() - .expect("has_completed_batch was true"); - (&record_batch) - .record_output(&self.join_metrics.baseline_metrics()); - return Poll::Ready(Some(Ok(record_batch))); - } - - // Finally check self.output BatchCoalescer (used by filtered joins) - return if !self.output.is_empty() { - self.output.finish_buffered_batch()?; - let record_batch = self - .output - .next_completed_batch() - .expect("Failed to get last batch"); - (&record_batch) - .record_output(&self.join_metrics.baseline_metrics()); - Poll::Ready(Some(Ok(record_batch))) - } else { - Poll::Ready(None) - }; - } - } - } - } -} impl MaterializingSortMergeJoinStream { #[expect(clippy::too_many_arguments)] @@ -836,7 +541,7 @@ impl MaterializingSortMergeJoinStream { reservation: MemoryReservation, spill_manager: SpillManager, runtime_env: Arc, - ) -> Result { + ) -> Result { let streamed_schema = streamed.schema(); let buffered_schema = buffered.schema(); debug_assert!( @@ -847,8 +552,8 @@ impl MaterializingSortMergeJoinStream { "MaterializingSortMergeJoinStream does not handle {join_type:?}; \ semi/anti/mark joins use BitwiseSortMergeJoinStream" ); - Ok(Self { - state: SortMergeJoinState::Init, + let join_time = join_metrics.join_time(); + let mut this = Self { sort_options, null_equality, schema: Arc::clone(&schema), @@ -858,13 +563,12 @@ impl MaterializingSortMergeJoinStream { buffered, streamed_batch: StreamedBatch::new_empty(streamed_schema), buffered_data: BufferedData::default(), - streamed_joined: false, - buffered_joined: false, - streamed_state: StreamedState::Init, - buffered_state: BufferedState::Init, - current_ordering: Ordering::Equal, + buffered_group_matched: false, + streamed_exhausted: false, + buffered_exhausted: false, on_streamed, on_buffered, + deferred_filtering: needs_deferred_filtering(&filter, join_type), filter, joined_record_batches: JoinedRecordBatches { joined_batches: BatchCoalescer::new(Arc::clone(&schema), batch_size) @@ -879,12 +583,299 @@ impl MaterializingSortMergeJoinStream { reservation, runtime_env, spill_manager, - spill_stream: None, spilled_batch_count: 0, + join_time, + join_time_start: None, streamed_buffered_cmp: None, buffered_equality_cmp: None, - streamed_batch_counter: AtomicUsize::new(0), - }) + streamed_batch_counter: 0, + }; + + let schema = Arc::clone(&this.schema); + let baseline_metrics = this.join_metrics.baseline_metrics(); + + let stream = async_try_stream(|mut emitter| async move { + this.start_join_time(); + let result = this.join(&mut emitter).await; + this.stop_join_time(); + result + }); + // ObservedStream records the baseline metrics (output rows/batches, + // end time). + Ok(Box::pin(ObservedStream::new( + Box::pin(RecordBatchStreamAdapter::new(schema, stream)), + baseline_metrics, + None, + ))) + } + + /// Main loop: the textbook sort-merge join. + /// + /// Both inputs arrive sorted on the join keys. The streamed side is + /// consumed one row at a time; the buffered side one key *group* (all + /// contiguous rows sharing a key) at a time + async fn join( + &mut self, + emitter: &mut TryEmitter, + ) -> Result<()> { + // 1. Load the first streamed row and the first buffered key group. + self.load_next_streamed_batch().await?; + self.advance_buffered_group().await?; + + // 2. Merge-scan while either input still has rows. + while !(self.streamed_exhausted && self.buffered_exhausted) { + // Flush the deferred-filtering pipeline once a full batch of + // rows accumulated (filtered outer joins output through it). + if self.deferred_filtering + && self.deferred_rows_accumulated() >= self.batch_size + { + self.emit_deferred_output(emitter).await?; + } + + // 3. Compare the join keys at both cursors. An exhausted side + // compares as the larger one, so the other side keeps + // draining through its own arm. + match self.compare_streamed_buffered()? { + // 3a. The streamed row can never match: null-join it (outer + // joins emit it; inner joins drop it), then advance. + Ordering::Less => { + self.null_join_streamed_row(); + if self.num_unfrozen_pairs() >= self.batch_size { + self.freeze_and_emit(emitter).await?; + } + if !self.try_advance_streamed_row() { + self.load_next_streamed_batch().await?; + } + } + // 3b. The buffered group can never match again: null-join + // it if nothing matched it (FULL join), then advance to + // the next key group. + Ordering::Greater => { + self.null_join_buffered_group(); + if !self.try_advance_buffered_group()? { + self.advance_buffered_group().await?; + } + } + // 3c. Match: pair the streamed row with the whole group — + // materializing ("freezing") mid-scan whenever a full + // batch of pairs accumulates — then advance streamed. + // The group stays for the next streamed row. + Ordering::Equal => { + while !self.pair_streamed_row_with_group() { + self.freeze_and_emit(emitter).await?; + } + if !self.try_advance_streamed_row() { + self.load_next_streamed_batch().await?; + } + } + } + + // 4. Emit completed output batches (filtered joins emit + // through the deferred-filtering pipeline above instead). + if !self.deferred_filtering + && self + .joined_record_batches + .joined_batches + .has_completed_batch() + { + self.emit_completed_joined_batches(emitter).await; + } + } + + // 5. Flush everything that remains. + self.on_children_exhausted(emitter).await + } + + /// `Equal`: pair the current streamed row with every row of the + /// buffered key group, and mark the group as matched. + /// + /// Returns false when a full batch of pairs has accumulated (the scan + /// may or may not be complete): the caller must materialize + /// (`freeze_and_emit`) and call again, which resumes the scan where it + /// paused. Returns true when the group scan is complete and there is + /// room for more pairs. + fn pair_streamed_row_with_group(&mut self) -> bool { + while !self.buffered_data.scanning_finished() + && self.num_unfrozen_pairs() < self.batch_size + { + let scanning_idx = self.buffered_data.scanning_idx(); + self.streamed_batch.append_output_pair( + Some(self.buffered_data.scanning_batch_idx), + Some(scanning_idx), + self.batch_size, + ); + self.buffered_data.scanning_advance(); + } + if self.num_unfrozen_pairs() >= self.batch_size { + return false; + } + + self.buffered_group_matched = true; + self.buffered_data.scanning_reset(); + true + } + + /// `Less` (outer joins): no buffered row matches the current streamed + /// row — emit it joined to NULLs. Inner joins emit nothing. + fn null_join_streamed_row(&mut self) { + if matches!( + self.join_type, + JoinType::Left | JoinType::Right | JoinType::Full + ) { + let scanning_batch_idx = if self.buffered_data.scanning_finished() { + None + } else { + Some(self.buffered_data.scanning_batch_idx) + }; + self.streamed_batch.append_output_pair( + scanning_batch_idx, + None, + self.batch_size, + ); + } + self.buffered_data.scanning_reset(); + } + + /// `Greater` (FULL join): the buffered group can never match a streamed + /// row anymore — if nothing matched it, mark all its rows for + /// null-joined output (produced when the group's batches are dequeued). + fn null_join_buffered_group(&mut self) { + if self.join_type == JoinType::Full && !self.buffered_group_matched { + while !self.buffered_data.scanning_finished() { + let scanning_idx = self.buffered_data.scanning_idx(); + self.buffered_data + .scanning_batch_mut() + .null_joined + .push(scanning_idx); + self.buffered_data.scanning_advance(); + } + } + self.buffered_data.scanning_reset(); + } + + /// Start (resume) the `join_time` clock. + fn start_join_time(&mut self) { + debug_assert!(self.join_time_start.is_none(), "join_time already running"); + self.join_time_start = Some(Instant::now()); + } + + /// Stop (pause) the `join_time` clock, accumulating the elapsed span. + /// + /// Called around awaits whose duration is not the join's own work: the + /// child input streams' `next()` and `emitter.emit()` (where the + /// consumer processes the batch). The join's own spill write and + /// read-back are NOT excluded — that time is join work. + fn stop_join_time(&mut self) { + if let Some(start) = self.join_time_start.take() { + self.join_time.add_elapsed(start); + } + } + + /// Number of rows currently waiting in the deferred-filtering pipeline. + /// + /// Typically bounded to ~2*batch_size: one batch_size worth from + /// freeze_dequeuing_buffered() (when an input batch is fully consumed), + /// plus up to batch_size pairs accumulating toward the next freeze. A + /// single streamed row matching a very large key group can exceed that + /// (its pairs freeze into the pipeline before the gate runs again — same + /// as the pre-generator design). This does not reintroduce the unbounded + /// buffering fixed by PR #20482; `on_children_exhausted` flushes the + /// remainder. + fn deferred_rows_accumulated(&self) -> usize { + self.num_unfrozen_pairs() + + self.joined_record_batches.filter_metadata.filter_mask.len() + } + + /// Run the deferred-filtering pipeline over everything accumulated so + /// far and emit its completed output, if any. Clears the accumulation + /// it processed. + /// + /// The caller gates this on `deferred_rows_accumulated() >= batch_size`: + /// running the pipeline per row instead (concat + correct_mask + + /// filter_by_type) would dominate runtime for unique keys. + async fn emit_deferred_output( + &mut self, + emitter: &mut TryEmitter, + ) -> Result<()> { + // Ensure required spilled batches are restored to memory before + // processing, as this path invokes freeze_all(). + self.restore_spilled_batches_for_freeze().await?; + if let Some(batch) = self.process_filtered_batches()? { + // While the emitted batch is in the consumer's hands the join + // isn't doing any work. + self.stop_join_time(); + emitter.emit(batch).await; + self.start_join_time(); + } + Ok(()) + } + + /// Restore every spilled buffered batch that the next freeze needs. + async fn restore_spilled_batches_for_freeze(&mut self) -> Result<()> { + let needed = self.get_required_batch_indices(self.buffered_data.batches.len()); + self.restore_spilled_batches(&needed).await + } + + /// Emit all completed joined batches to the stream consumer. + async fn emit_completed_joined_batches( + &mut self, + emitter: &mut TryEmitter, + ) { + while let Some(record_batch) = self + .joined_record_batches + .joined_batches + .next_completed_batch() + { + // While the emitted batch is in the consumer's hands the join + // isn't doing any work. + self.stop_join_time(); + emitter.emit(record_batch).await; + self.start_join_time(); + } + } + + /// Flush everything that remains once both inputs are exhausted. + async fn on_children_exhausted( + &mut self, + emitter: &mut TryEmitter, + ) -> Result<()> { + // Freeze the remaining pairs, restoring any spilled batches needed. + self.restore_spilled_batches_for_freeze().await?; + self.freeze_all()?; + + // Verify metadata alignment before final output + self.joined_record_batches + .filter_metadata + .debug_assert_metadata_aligned(); + + if self.deferred_filtering { + // Filtered joins must concat and filter ALL remaining data at once + if !self.joined_record_batches.joined_batches.is_empty() { + let record_batch = self.filter_joined_batch()?; + self.stop_join_time(); + emitter.emit(record_batch).await; + self.start_join_time(); + } + } else if !self.joined_record_batches.joined_batches.is_empty() { + // For non-filtered joins, finish buffered data first, then emit + // every completed batch. + self.joined_record_batches + .joined_batches + .finish_buffered_batch()?; + self.emit_completed_joined_batches(emitter).await; + } + + // Drain the double-buffering coalescer used by filtered joins. + if !self.output.is_empty() { + self.output.finish_buffered_batch()?; + while let Some(record_batch) = self.output.next_completed_batch() { + self.stop_join_time(); + emitter.emit(record_batch).await; + self.start_join_time(); + } + } + + Ok(()) } /// Build a comparator for streamed vs buffered head batch keys. @@ -927,9 +918,9 @@ impl MaterializingSortMergeJoinStream { /// Process accumulated batches for filtered joins /// - /// Freezes unfrozen pairs, applies deferred filtering, and outputs if ready. - /// Returns Poll::Ready with a batch if one is available, otherwise Poll::Pending. - fn process_filtered_batches(&mut self) -> Poll>> { + /// Freezes unfrozen pairs, applies deferred filtering, and returns a + /// completed output batch if one is ready. + fn process_filtered_batches(&mut self) -> Result> { self.freeze_all()?; self.joined_record_batches @@ -947,12 +938,11 @@ impl MaterializingSortMergeJoinStream { .output .next_completed_batch() .expect("Failed to get output batch"); - (&record_batch).record_output(&self.join_metrics.baseline_metrics()); - return Poll::Ready(Some(Ok(record_batch))); + return Ok(Some(record_batch)); } } - Poll::Pending + Ok(None) } /// Identifies which buffered batches are needed for the upcoming freeze operation @@ -981,11 +971,10 @@ impl MaterializingSortMergeJoinStream { /// Asynchronously reads spilled batches back into memory. /// Only processes the required indices to avoid OOMs. - fn poll_spilled_batches( + async fn restore_spilled_batches( &mut self, - cx: &mut Context<'_>, required_indices: &[usize], - ) -> Poll> { + ) -> Result<()> { for &idx in required_indices { // Guard against indices that might be out of bounds if the queue was cleared if idx >= self.buffered_data.batches.len() { @@ -995,15 +984,12 @@ impl MaterializingSortMergeJoinStream { let bb = &mut self.buffered_data.batches[idx]; if let BufferedBatchState::Spilled(spill_file) = &bb.batch { - if self.spill_stream.is_none() { - let stream = self - .spill_manager - .read_spill_as_stream(Arc::clone(spill_file), None)?; - self.spill_stream = Some(stream); - } + let mut spill_stream = self + .spill_manager + .read_spill_as_stream(Arc::clone(spill_file), None)?; - match ready!(self.spill_stream.as_mut().unwrap().poll_next_unpin(cx)) { - Some(Ok(batch)) => { + match spill_stream.next().await.transpose()? { + Some(batch) => { // Transition the batch back to InMemory bb.batch = BufferedBatchState::InMemory(batch); self.spilled_batch_count -= 1; @@ -1016,78 +1002,65 @@ impl MaterializingSortMergeJoinStream { self.join_metrics .peak_mem_used() .set_max(self.reservation.size()); - - self.spill_stream = None; - } - Some(Err(e)) => { - self.spill_stream = None; - return Poll::Ready(Err(e)); } None => { - self.spill_stream = None; - return Poll::Ready(internal_err!("Spill file was empty")); + return internal_err!("Spill file was empty"); } } } } - Poll::Ready(Ok(())) + + Ok(()) } - /// Poll next streamed row - fn poll_streamed_row(&mut self, cx: &mut Context) -> Poll>> { + /// Sync fast path of advancing the streamed cursor: move to the next row + /// of the current batch. Returns false at the batch boundary, where the + /// caller must load the next batch via + /// [`Self::load_next_streamed_batch`]. + fn try_advance_streamed_row(&mut self) -> bool { + if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows() { + self.streamed_batch.idx += 1; + return true; + } + false + } + + /// Load the next streamed batch (freezing the finished one) and point + /// the streamed cursor at its first row. Sets `streamed_exhausted` when + /// the streamed input has no more rows. + async fn load_next_streamed_batch(&mut self) -> Result<()> { loop { - match &self.streamed_state { - StreamedState::Init => { - if self.streamed_batch.idx + 1 < self.streamed_batch.batch.num_rows() - { - self.streamed_batch.idx += 1; - self.streamed_state = StreamedState::Ready; - return Poll::Ready(Some(Ok(()))); - } else { - self.streamed_state = StreamedState::Polling; - } + // Loading a new streamed batch freezes the current one, which + // materializes buffered columns — restore any spilled buffered + // batches it needs first. + self.restore_spilled_batches_for_freeze().await?; + + // The child's execution time is its own, not join_time. + self.stop_join_time(); + let item = self.streamed.next().await.transpose(); + self.start_join_time(); + match item? { + None => { + // Release the streamed input pipeline's resources. + let streamed_schema = self.streamed.schema(); + self.streamed = + Box::pin(EmptyRecordBatchStream::new(streamed_schema)); + self.streamed_exhausted = true; + return Ok(()); } - StreamedState::Polling => { - let needed = - self.get_required_batch_indices(self.buffered_data.batches.len()); - if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed)) { - return Poll::Ready(Some(Err(e))); - } - - match self.streamed.poll_next_unpin(cx)? { - Poll::Pending => { - return Poll::Pending; - } - Poll::Ready(None) => { - // Release the streamed input pipeline's resources. - let streamed_schema = self.streamed.schema(); - self.streamed = - Box::pin(EmptyRecordBatchStream::new(streamed_schema)); - self.streamed_state = StreamedState::Exhausted; - } - Poll::Ready(Some(batch)) => { - if batch.num_rows() > 0 { - self.freeze_streamed()?; - self.join_metrics.input_batches().add(1); - self.join_metrics.input_rows().add(batch.num_rows()); - self.streamed_batch = - StreamedBatch::new(batch, &self.on_streamed); - self.rebuild_streamed_buffered_cmp()?; - // Every incoming streaming batch should have its unique id - // Check `JoinedRecordBatches.self.streamed_batch_counter` documentation - self.streamed_batch_counter - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.streamed_state = StreamedState::Ready; - } - } + Some(batch) => { + if batch.num_rows() > 0 { + self.freeze_streamed()?; + self.join_metrics.input_batches().add(1); + self.join_metrics.input_rows().add(batch.num_rows()); + self.streamed_batch = + StreamedBatch::new(batch, &self.on_streamed); + self.rebuild_streamed_buffered_cmp()?; + // Every incoming streamed batch gets a unique id. + self.streamed_batch_counter += 1; + return Ok(()); } } - StreamedState::Ready => { - return Poll::Ready(Some(Ok(()))); - } - StreamedState::Exhausted => { - return Poll::Ready(None); - } } } } @@ -1150,146 +1123,193 @@ impl MaterializingSortMergeJoinStream { Ok(()) } - /// Poll next buffered batches - fn poll_buffered_batches(&mut self, cx: &mut Context) -> Poll>> { - loop { - match &self.buffered_state { - BufferedState::Init => { - // pop previous buffered batches - let mut head_changed = false; - while !self.buffered_data.batches.is_empty() { - let head_batch = self.buffered_data.head_batch(); - // If the head batch is fully processed, dequeue it and produce output of it. - if head_batch.range.end == head_batch.num_rows { - // load the spilled head batch before dequeuing - let needed = self.get_required_batch_indices(1); - if let Err(e) = ready!(self.poll_spilled_batches(cx, &needed)) - { - return Poll::Ready(Some(Err(e))); - } + /// Sync fast path of [`Self::advance_buffered_group`]: when the next + /// group starts in the single remaining buffered batch and provably ends + /// within it (the common case — a group only reaches a batch boundary + /// once per batch), advance entirely synchronously. Returns false — + /// leaving all state unchanged — when the async path must run instead. + fn try_advance_buffered_group(&mut self) -> Result { + if self.buffered_data.batches.len() != 1 { + return Ok(false); + } + let head_batch = self.buffered_data.head_batch(); + if head_batch.range.end == head_batch.num_rows { + // Fully consumed — needs dequeuing (and loading the next batch). + return Ok(false); + } - self.freeze_dequeuing_buffered()?; - if let Some(mut buffered_batch) = - self.buffered_data.batches.pop_front() - { - self.produce_buffered_not_matched(&mut buffered_batch)?; - self.free_reservation(&buffered_batch); - if matches!( - buffered_batch.batch, - BufferedBatchState::Spilled(_) - ) { - self.spilled_batch_count -= 1; - } - head_changed = true; - } - } else { - // If the head batch is not fully processed, break the loop. - // Streamed batch will be joined with the head batch in the next step. - break; - } - } - if head_changed { + if self.buffered_equality_cmp.is_none() { + self.rebuild_buffered_equality_cmp()?; + } + let cmp = self.buffered_equality_cmp.as_ref().unwrap(); + + // Scan the next group's extent before committing any state, so a + // bail-out (the group may span into the next batch) leaves + // everything untouched for the async path. + let batch = self.buffered_data.head_batch(); + let group_start = batch.range.end; + let mut group_end = group_start + 1; + while group_end < batch.num_rows && cmp.is_equal(group_start, group_end) { + group_end += 1; + } + if group_end == batch.num_rows { + return Ok(false); + } + + let batch = self.buffered_data.tail_batch_mut(); + batch.range.start = group_start; + batch.range.end = group_end; + self.buffered_group_matched = false; + Ok(true) + } + + /// Advance the buffered side to the next key group: dequeue batches + /// fully consumed by the previous group, then collect all contiguous + /// rows sharing the next join key (the group may span multiple buffered + /// batches). Sets `buffered_exhausted` when no group remains. + async fn advance_buffered_group(&mut self) -> Result<()> { + self.buffered_group_matched = false; + self.dequeue_consumed_buffered_batches().await?; + + if self.buffered_data.batches.is_empty() { + // Load the batch holding the first row of the next group. + if !self.load_next_buffered_batch().await? { + self.buffered_exhausted = true; + return Ok(()); + } + } else { + // Seed the next group at the first unconsumed row of the + // remaining batch. + let tail_batch = self.buffered_data.tail_batch_mut(); + tail_batch.range.start = tail_batch.range.end; + tail_batch.range.end += 1; + } + + self.extend_buffered_group().await + } + + /// Dequeue buffered batches fully consumed by the previous group, + /// producing their pending output (e.g. Full-join null-joined rows). + async fn dequeue_consumed_buffered_batches(&mut self) -> Result<()> { + let mut head_changed = false; + while !self.buffered_data.batches.is_empty() { + let head_batch = self.buffered_data.head_batch(); + if head_batch.range.end != head_batch.num_rows { + // The next group starts within the head batch: streamed rows + // will be joined with the head batch in the next step. + break; + } + // load the spilled head batch before dequeuing + let needed = self.get_required_batch_indices(1); + self.restore_spilled_batches(&needed).await?; + + self.freeze_dequeuing_buffered()?; + if let Some(mut buffered_batch) = self.buffered_data.batches.pop_front() { + self.produce_buffered_not_matched(&mut buffered_batch)?; + self.free_reservation(&buffered_batch); + if matches!(buffered_batch.batch, BufferedBatchState::Spilled(_)) { + self.spilled_batch_count -= 1; + } + head_changed = true; + } + } + if head_changed { + self.streamed_buffered_cmp = None; + self.buffered_equality_cmp = None; + } + Ok(()) + } + + /// Load the next non-empty buffered batch and seed a new group with its + /// first row. Returns false when the buffered input is exhausted. + async fn load_next_buffered_batch(&mut self) -> Result { + loop { + // The child's execution time is its own, not join_time. + self.stop_join_time(); + let item = self.buffered.next().await.transpose(); + self.start_join_time(); + match item? { + None => { + // Release the buffered input pipeline's resources. + let buffered_schema = self.buffered.schema(); + self.buffered = + Box::pin(EmptyRecordBatchStream::new(buffered_schema)); + return Ok(false); + } + Some(batch) => { + self.join_metrics.input_batches().add(1); + self.join_metrics.input_rows().add(batch.num_rows()); + + if batch.num_rows() > 0 { + let buffered_batch = + BufferedBatch::new(batch, 0..1, &self.on_buffered); + self.allocate_reservation(buffered_batch)?; self.streamed_buffered_cmp = None; - self.buffered_equality_cmp = None; + return Ok(true); } - if self.buffered_data.batches.is_empty() { - self.buffered_state = BufferedState::PollingFirst; + } + } + } + } + + /// Extend the current group with every following row that shares its + /// key, loading more buffered batches as needed. + async fn extend_buffered_group(&mut self) -> Result<()> { + loop { + if self.buffered_data.tail_batch().range.end + < self.buffered_data.tail_batch().num_rows + { + if self.buffered_equality_cmp.is_none() { + self.rebuild_buffered_equality_cmp()?; + } + while self.buffered_data.tail_batch().range.end + < self.buffered_data.tail_batch().num_rows + { + if self.buffered_equality_cmp.as_ref().unwrap().is_equal( + self.buffered_data.head_batch().range.start, + self.buffered_data.tail_batch().range.end, + ) { + self.buffered_data.tail_batch_mut().range.end += 1; } else { - let tail_batch = self.buffered_data.tail_batch_mut(); - tail_batch.range.start = tail_batch.range.end; - tail_batch.range.end += 1; - self.buffered_state = BufferedState::PollingRest; + // Group complete within the current batch. + return Ok(()); } } - BufferedState::PollingFirst => match self.buffered.poll_next_unpin(cx)? { - Poll::Pending => { - return Poll::Pending; - } - Poll::Ready(None) => { + } else { + // The child's execution time is its own, not join_time. + self.stop_join_time(); + let item = self.buffered.next().await.transpose(); + self.start_join_time(); + match item? { + None => { + // Group complete; the input is done but the group is + // still valid — `buffered_exhausted` is only set once + // it has been fully consumed and dequeued. // Release the buffered input pipeline's resources. let buffered_schema = self.buffered.schema(); self.buffered = Box::pin(EmptyRecordBatchStream::new(buffered_schema)); - self.buffered_state = BufferedState::Exhausted; - return Poll::Ready(None); + return Ok(()); } - Poll::Ready(Some(batch)) => { + Some(batch) => { + // Polling batches coming concurrently as multiple partitions self.join_metrics.input_batches().add(1); self.join_metrics.input_rows().add(batch.num_rows()); - if batch.num_rows() > 0 { let buffered_batch = - BufferedBatch::new(batch, 0..1, &self.on_buffered); - + BufferedBatch::new(batch, 0..0, &self.on_buffered); self.allocate_reservation(buffered_batch)?; - self.streamed_buffered_cmp = None; - self.buffered_state = BufferedState::PollingRest; - } - } - }, - BufferedState::PollingRest => { - if self.buffered_data.tail_batch().range.end - < self.buffered_data.tail_batch().num_rows - { - if self.buffered_equality_cmp.is_none() { - self.rebuild_buffered_equality_cmp()?; - } - while self.buffered_data.tail_batch().range.end - < self.buffered_data.tail_batch().num_rows - { - if self.buffered_equality_cmp.as_ref().unwrap().is_equal( - self.buffered_data.head_batch().range.start, - self.buffered_data.tail_batch().range.end, - ) { - self.buffered_data.tail_batch_mut().range.end += 1; - } else { - self.buffered_state = BufferedState::Ready; - return Poll::Ready(Some(Ok(()))); - } - } - } else { - match self.buffered.poll_next_unpin(cx)? { - Poll::Pending => { - return Poll::Pending; - } - Poll::Ready(None) => { - // Release the buffered input pipeline's resources. - let buffered_schema = self.buffered.schema(); - self.buffered = Box::pin(EmptyRecordBatchStream::new( - buffered_schema, - )); - self.buffered_state = BufferedState::Ready; - } - Poll::Ready(Some(batch)) => { - // Polling batches coming concurrently as multiple partitions - self.join_metrics.input_batches().add(1); - self.join_metrics.input_rows().add(batch.num_rows()); - if batch.num_rows() > 0 { - let buffered_batch = BufferedBatch::new( - batch, - 0..0, - &self.on_buffered, - ); - self.allocate_reservation(buffered_batch)?; - self.buffered_equality_cmp = None; - } - } + self.buffered_equality_cmp = None; } } } - BufferedState::Ready => { - return Poll::Ready(Some(Ok(()))); - } - BufferedState::Exhausted => { - return Poll::Ready(None); - } } } } /// Get comparison result of streamed row and buffered batches fn compare_streamed_buffered(&mut self) -> Result { - if self.streamed_state == StreamedState::Exhausted { + if self.streamed_exhausted { return Ok(Ordering::Greater); } if !self.buffered_data.has_buffered_rows() { @@ -1305,81 +1325,23 @@ impl MaterializingSortMergeJoinStream { )) } - /// Produce join and fill output buffer until reaching target batch size - /// or the join is finished - fn join_partial(&mut self) -> Result<()> { - // Whether to join streamed rows - let mut join_streamed = false; - // Whether to join buffered rows - let mut join_buffered = false; - - // determine whether we need to join streamed/buffered rows - match self.current_ordering { - Ordering::Less => { - if matches!( - self.join_type, - JoinType::Left | JoinType::Right | JoinType::Full - ) { - join_streamed = !self.streamed_joined; - } - } - Ordering::Equal => { - join_streamed = true; - join_buffered = true; - } - Ordering::Greater => { - if self.join_type == JoinType::Full { - join_buffered = !self.buffered_joined; - }; - } - } - if !join_streamed && !join_buffered { - // no joined data - self.buffered_data.scanning_finish(); - return Ok(()); - } - - if join_buffered { - // joining streamed/nulls and buffered - while !self.buffered_data.scanning_finished() - && self.num_unfrozen_pairs() < self.batch_size - { - let scanning_idx = self.buffered_data.scanning_idx(); - if join_streamed { - // Join streamed row and buffered row - self.streamed_batch.append_output_pair( - Some(self.buffered_data.scanning_batch_idx), - Some(scanning_idx), - self.batch_size, - ); - } else { - // Join nulls and buffered row for FULL join - self.buffered_data - .scanning_batch_mut() - .null_joined - .push(scanning_idx); - } - self.buffered_data.scanning_advance(); + /// Materialize ("freeze") the accumulated pairs — restoring any spilled + /// batches they reference first — and emit completed output batches + /// (filtered joins emit through the deferred-filtering gate instead). + async fn freeze_and_emit( + &mut self, + emitter: &mut TryEmitter, + ) -> Result<()> { + self.restore_spilled_batches_for_freeze().await?; + self.freeze_all()?; - if self.buffered_data.scanning_finished() { - self.streamed_joined = join_streamed; - self.buffered_joined = true; - } - } - } else { - // joining streamed and nulls - let scanning_batch_idx = if self.buffered_data.scanning_finished() { - None - } else { - Some(self.buffered_data.scanning_batch_idx) - }; - self.streamed_batch.append_output_pair( - scanning_batch_idx, - None, - self.batch_size, - ); - self.buffered_data.scanning_finish(); - self.streamed_joined = true; + if !self.deferred_filtering + && self + .joined_record_batches + .joined_batches + .has_completed_batch() + { + self.emit_completed_joined_batches(emitter).await; } Ok(()) } @@ -1513,7 +1475,7 @@ impl MaterializingSortMergeJoinStream { // but must flow through the same pipeline as matched rows to // preserve output ordering. Use null metadata as a sentinel so // get_corrected_filter_mask() passes them through unchanged. - if needs_deferred_filtering(&self.filter, self.join_type) { + if self.deferred_filtering { self.joined_record_batches .push_batch_with_null_metadata(batch, self.join_type); } else { @@ -1616,12 +1578,12 @@ impl MaterializingSortMergeJoinStream { filter_result_mask.clone() }; - if needs_deferred_filtering(&self.filter, self.join_type) { + if self.deferred_filtering { self.joined_record_batches.push_batch_with_filter_metadata( output_batch, &combined_left_indices, &mask, - self.streamed_batch_counter.load(Relaxed), + self.streamed_batch_counter, self.join_type, ); } else { @@ -1968,9 +1930,9 @@ fn fetch_right_columns_from_batch_by_idxs( pub(super) struct BufferedData { /// Buffered batches with the same key pub batches: VecDeque, - /// current scanning batch index used in join_partial() + /// current scanning batch index used by the group-scan phase pub scanning_batch_idx: usize, - /// current scanning offset used in join_partial() + /// current scanning offset used by the group-scan phase pub scanning_offset: usize, } @@ -2023,11 +1985,6 @@ impl BufferedData { pub fn scanning_finished(&self) -> bool { self.scanning_batch_idx == self.batches.len() } - - pub fn scanning_finish(&mut self) { - self.scanning_batch_idx = self.batches.len(); - self.scanning_offset = 0; - } } /// Get join array refs of given batch and join columns diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs index 313818cf6c6b5..3dbb50eba07d9 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs @@ -4315,6 +4315,147 @@ async fn join_time_excludes_consumer_wait() -> Result<()> { .await } +/// Three 2-row batches with unique matching keys, right-side column names. +fn join_time_batches_right() -> Vec { + vec![ + build_table_i32( + ("a2", &vec![0, 1]), + ("b2", &vec![1, 2]), + ("c2", &vec![7, 8]), + ), + build_table_i32( + ("a2", &vec![2, 3]), + ("b2", &vec![3, 4]), + ("c2", &vec![7, 8]), + ), + build_table_i32( + ("a2", &vec![4, 5]), + ("b2", &vec![5, 6]), + ("c2", &vec![7, 8]), + ), + ] +} + +/// Build a no-filter Inner materializing join over the given input streams. +/// The small batch size makes the output surface as multiple batches, so a +/// slow consumer test sees multiple emits. +fn materializing_join_time_test_join( + streamed: SendableRecordBatchStream, + buffered: SendableRecordBatchStream, +) -> (SendableRecordBatchStream, ExecutionPlanMetricsSet) { + use crate::joins::sort_merge_join::materializing_stream::MaterializingSortMergeJoinStream; + use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics; + + let metrics = ExecutionPlanMetricsSet::new(); + let out_schema = Arc::new(Schema::new( + streamed + .schema() + .fields() + .iter() + .chain(buffered.schema().fields().iter()) + .map(|f| f.as_ref().clone()) + .collect::>(), + )); + let (reservation, spill_manager, runtime_env) = + test_stream_resources(buffered.schema(), &metrics); + let stream = MaterializingSortMergeJoinStream::try_new( + out_schema, + vec![SortOptions::default()], + NullEquality::NullEqualsNothing, + streamed, + buffered, + vec![Arc::new(Column::new("b1", 1)) as _], + vec![Arc::new(Column::new("b2", 1)) as _], + None, + Inner, + 2, + SortMergeJoinMetrics::new(0, &metrics), + reservation, + spill_manager, + runtime_env, + ) + .unwrap(); + (stream, metrics) +} + +/// join_time must not include time spent waiting for the streamed input. +#[tokio::test] +async fn materializing_join_time_excludes_streamed_input_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let streamed = delayed_stream(join_time_batches(), delay); + let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO); + let (stream, metrics) = materializing_join_time_test_join(streamed, buffered); + + let start = Instant::now(); + let batches = collect_stream(stream).await?; + let wall = start.elapsed(); + + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 6, "all rows should match"); + assert!( + wall >= delay * 3, + "streamed delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + +/// join_time must not include time spent waiting for the buffered input. +#[tokio::test] +async fn materializing_join_time_excludes_buffered_input_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let streamed = delayed_stream(join_time_batches(), Duration::ZERO); + let buffered = delayed_stream(join_time_batches_right(), delay); + let (stream, metrics) = materializing_join_time_test_join(streamed, buffered); + + let start = Instant::now(); + let batches = collect_stream(stream).await?; + let wall = start.elapsed(); + + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, 6, "all rows should match"); + assert!( + wall >= delay * 3, + "buffered delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + +/// join_time must not include time the consumer spends holding an emitted +/// batch (the generator is suspended inside `emitter.emit` meanwhile). +#[tokio::test] +async fn materializing_join_time_excludes_consumer_wait() -> Result<()> { + check_join_time_excluded(|delay| async move { + let streamed = delayed_stream(join_time_batches(), Duration::ZERO); + let buffered = delayed_stream(join_time_batches_right(), Duration::ZERO); + let (mut stream, metrics) = materializing_join_time_test_join(streamed, buffered); + + let start = Instant::now(); + let mut output_batches = 0u32; + while let Some(batch) = stream.next().await { + batch?; + output_batches += 1; + // Simulate a slow consumer between emitted batches. + tokio::time::sleep(delay).await; + } + let wall = start.elapsed(); + + assert!( + output_batches >= 3, + "expected multiple emitted batches, got {output_batches}" + ); + assert!( + wall >= delay * output_batches, + "consumer delays should dominate wall time, got {wall:?}" + ); + Ok((join_time_of(&metrics), wall)) + }) + .await +} + /// An inner key group spanning multiple inner batches must survive the inner /// input returning Pending mid-way: inner rows delivered before the Pending /// still take part in the filter evaluation. From 317b324675b1c2b174c7876b098477cc2b3b5638 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:03:40 -0400 Subject: [PATCH 091/109] Own target-safe Parquet and compression features Signed-off-by: Ethan Urbanski (cherry picked from commit a1513a828c9d37e80e73c928794e03ca466657bc) --- Cargo.lock | 1 + ci/scripts/check_wasm_dependency_policy.sh | 83 +++++++++++ datafusion/common/Cargo.toml | 13 +- datafusion/core/Cargo.toml | 24 ++- datafusion/datasource/Cargo.toml | 11 +- .../datasource/src/file_compression_type.rs | 137 ++++++++++++++++-- 6 files changed, 250 insertions(+), 19 deletions(-) create mode 100755 ci/scripts/check_wasm_dependency_policy.sh diff --git a/Cargo.lock b/Cargo.lock index 9cd5d12d5bca4..793b420b99dfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1748,6 +1748,7 @@ dependencies = [ "env_logger", "flate2", "futures", + "getrandom 0.3.4", "glob", "indexmap 2.14.0", "insta", diff --git a/ci/scripts/check_wasm_dependency_policy.sh b/ci/scripts/check_wasm_dependency_policy.sh new file mode 100755 index 0000000000000..6e5e404ad345b --- /dev/null +++ b/ci/scripts/check_wasm_dependency_policy.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" + +target="wasm32-unknown-unknown" +graph="$( + cargo tree -p datafusion --target "$target" --locked \ + --prefix none -e normal,build +)" +feature_graph="$( + cargo tree -p datafusion --target "$target" --locked \ + --prefix none -e normal,build,features +)" + +denied_packages=( + aws-lc-sys + hyper + liblzma-sys + native-tls + openssl-sys + ring + tempfile + walkdir + zstd-sys +) + +for package in "${denied_packages[@]}"; do + if grep -Eq "^${package} v" <<<"$graph"; then + printf 'denied package in %s graph: %s\n' "$target" "$package" >&2 + cargo tree -p datafusion --target "$target" --locked -i "$package" >&2 || true + exit 1 + fi +done + +denied_features=( + 'object_store feature "aws"' + 'object_store feature "azure"' + 'object_store feature "fs"' + 'object_store feature "gcp"' + 'tokio feature "rt-multi-thread"' +) + +for feature in "${denied_features[@]}"; do + if grep -Fq "$feature" <<<"$feature_graph"; then + printf 'denied feature in %s graph: %s\n' "$target" "$feature" >&2 + exit 1 + fi +done + +duplicates="$( + cargo tree -p datafusion --target "$target" --locked \ + -d --prefix none -e normal,build +)" +for package in arrow parquet object_store; do + if grep -Eq "^${package} v" <<<"$duplicates"; then + printf 'duplicate %s source/version universe in %s graph\n' "$package" "$target" >&2 + grep -E "^${package} v" <<<"$duplicates" >&2 + exit 1 + fi +done + +printf 'DataFusion %s dependency policy passed\n' "$target" diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 1eb23089a4021..8c90b72fb3d9e 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -79,7 +79,18 @@ libc = "0.2.185" log = { workspace = true } num-traits = { workspace = true } object_store = { workspace = true, optional = true } -parquet = { workspace = true, optional = true, default-features = true } +parquet = { workspace = true, optional = true, default-features = false, features = [ + "arrow", + "async", + "object_store", + "snap", + "brotli", + "flate2-zlib-rs", + "lz4", + "zstd", + "base64", + "simdutf8", +] } recursive = { workspace = true, optional = true } sqlparser = { workspace = true, optional = true } tokio = { workspace = true } diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 8679dad9f9a32..7547c43b071a7 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -146,19 +146,37 @@ flate2 = { workspace = true, optional = true } futures = { workspace = true } indexmap = { workspace = true } itertools = { workspace = true } -liblzma = { workspace = true, optional = true } log = { workspace = true } object_store = { workspace = true } parking_lot = { workspace = true } -parquet = { workspace = true, optional = true, default-features = true } +parquet = { workspace = true, optional = true, default-features = false, features = [ + "arrow", + "async", + "object_store", + "snap", + "brotli", + "flate2-zlib-rs", + "lz4", + "zstd", + "base64", + "simdutf8", +] } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } sqlparser = { workspace = true, optional = true } -tempfile = { workspace = true } tokio = { workspace = true } url = { workspace = true } uuid = { workspace = true, features = ["v4", "js"] } + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] +liblzma = { workspace = true, optional = true } +tempfile = { workspace = true } zstd = { workspace = true, optional = true } +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +# Select getrandom's supported JavaScript backend through Cargo features. This +# avoids process-global RUSTFLAGS and keeps native entropy behavior unchanged. +getrandom = { version = "0.3", features = ["wasm_js"] } + [dev-dependencies] async-trait = { workspace = true } criterion = { workspace = true, features = ["async_tokio", "async_futures"] } diff --git a/datafusion/datasource/Cargo.toml b/datafusion/datasource/Cargo.toml index 459ca436f365d..e90ac2fdcb2ae 100644 --- a/datafusion/datasource/Cargo.toml +++ b/datafusion/datasource/Cargo.toml @@ -44,8 +44,6 @@ arrow = { workspace = true } async-compression = { version = "0.4.40", features = [ "bzip2", "gzip", - "xz", - "zstd", "tokio", ], optional = true } async-trait = { workspace = true } @@ -66,15 +64,20 @@ flate2 = { workspace = true, optional = true } futures = { workspace = true } glob = { workspace = true } itertools = { workspace = true } -liblzma = { workspace = true, optional = true } log = { workspace = true } object_store = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } -tempfile = { workspace = true, optional = true } tokio = { workspace = true } tokio-util = { version = "0.7.17", features = ["io"], optional = true } url = { workspace = true } + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] +async-compression = { version = "0.4.40", features = [ + "xz", + "zstd", +], optional = true } +liblzma = { workspace = true, optional = true } zstd = { workspace = true, optional = true } [dev-dependencies] diff --git a/datafusion/datasource/src/file_compression_type.rs b/datafusion/datasource/src/file_compression_type.rs index 89efb580652b1..b6adb50327333 100644 --- a/datafusion/datasource/src/file_compression_type.rs +++ b/datafusion/datasource/src/file_compression_type.rs @@ -28,12 +28,23 @@ use datafusion_common::parsers::CompressionTypeVariant::{self, *}; use async_compression::tokio::bufread::{ BzDecoder as AsyncBzDecoder, BzEncoder as AsyncBzEncoder, GzipDecoder as AsyncGzDecoder, GzipEncoder as AsyncGzEncoder, +}; +#[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] +use async_compression::tokio::bufread::{ XzDecoder as AsyncXzDecoder, XzEncoder as AsyncXzEncoder, ZstdDecoder as AsyncZstdDecoer, ZstdEncoder as AsyncZstdEncoder, }; #[cfg(feature = "compression")] -use async_compression::tokio::write::{BzEncoder, GzipEncoder, XzEncoder, ZstdEncoder}; +use async_compression::tokio::write::{BzEncoder, GzipEncoder}; +#[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] +use async_compression::tokio::write::{XzEncoder, ZstdEncoder}; use bytes::Bytes; #[cfg(feature = "compression")] use bzip2::read::MultiBzDecoder; @@ -43,15 +54,28 @@ use futures::StreamExt; #[cfg(feature = "compression")] use futures::TryStreamExt; use futures::stream::BoxStream; -#[cfg(feature = "compression")] +#[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] use liblzma::read::XzDecoder; use object_store::buffered::BufWriter; use tokio::io::AsyncWrite; #[cfg(feature = "compression")] use tokio_util::io::{ReaderStream, StreamReader}; -#[cfg(feature = "compression")] +#[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) +))] use zstd::Decoder as ZstdDecoder; +#[cfg(all(feature = "compression", target_arch = "wasm32", target_os = "unknown"))] +fn target_unavailable(operation: &str, codec: &str) -> DataFusionError { + DataFusionError::NotImplemented(format!( + "{operation} with {codec} compression is unavailable on target wasm32-unknown-unknown" + )) +} + /// Readable file compression type #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FileCompressionType { @@ -136,14 +160,36 @@ impl FileCompressionType { BZIP2 => ReaderStream::new(AsyncBzEncoder::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] XZ => ReaderStream::new(AsyncXzEncoder::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] ZSTD => ReaderStream::new(AsyncZstdEncoder::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + XZ => { + return Err(target_unavailable("stream compression", "xz")); + } + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + ZSTD => { + return Err(target_unavailable("stream compression", "zstd")); + } #[cfg(not(feature = "compression"))] GZIP | BZIP2 | XZ | ZSTD => { return Err(DataFusionError::NotImplemented( @@ -191,20 +237,45 @@ impl FileCompressionType { } None => Box::new(BzEncoder::new(w)), }, - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] XZ => match compression_level { Some(level) => { Box::new(XzEncoder::with_quality(w, Level::Precise(level as i32))) } None => Box::new(XzEncoder::new(w)), }, - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] ZSTD => match compression_level { Some(level) => { Box::new(ZstdEncoder::with_quality(w, Level::Precise(level as i32))) } None => Box::new(ZstdEncoder::new(w)), }, + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + XZ => { + return Err(target_unavailable("asynchronous writer compression", "xz")); + } + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + ZSTD => { + return Err(target_unavailable( + "asynchronous writer compression", + "zstd", + )); + } #[cfg(not(feature = "compression"))] GZIP | BZIP2 | XZ | ZSTD => { // compression_level is not used when compression feature is disabled @@ -236,14 +307,36 @@ impl FileCompressionType { BZIP2 => ReaderStream::new(AsyncBzDecoder::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] XZ => ReaderStream::new(AsyncXzDecoder::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] ZSTD => ReaderStream::new(AsyncZstdDecoer::new(StreamReader::new(s))) .map_err(DataFusionError::from) .boxed(), + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + XZ => { + return Err(target_unavailable("stream decompression", "xz")); + } + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + ZSTD => { + return Err(target_unavailable("stream decompression", "zstd")); + } #[cfg(not(feature = "compression"))] GZIP | BZIP2 | XZ | ZSTD => { return Err(DataFusionError::NotImplemented( @@ -264,13 +357,35 @@ impl FileCompressionType { GZIP => Box::new(MultiGzDecoder::new(r)), #[cfg(feature = "compression")] BZIP2 => Box::new(MultiBzDecoder::new(r)), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] XZ => Box::new(XzDecoder::new_multi_decoder(r)), - #[cfg(feature = "compression")] + #[cfg(all( + feature = "compression", + not(all(target_arch = "wasm32", target_os = "unknown")) + ))] ZSTD => match ZstdDecoder::new(r) { Ok(decoder) => Box::new(decoder), Err(e) => return Err(DataFusionError::External(Box::new(e))), }, + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + XZ => { + return Err(target_unavailable("reader decompression", "xz")); + } + #[cfg(all( + feature = "compression", + target_arch = "wasm32", + target_os = "unknown" + ))] + ZSTD => { + return Err(target_unavailable("reader decompression", "zstd")); + } #[cfg(not(feature = "compression"))] GZIP | BZIP2 | XZ | ZSTD => { return Err(DataFusionError::NotImplemented( From b5ac43bdb2b77485a89c38df3d8b72acc1daf77e Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 03:51:58 -0400 Subject: [PATCH 092/109] Declare the target feature selector intentional Signed-off-by: Ethan Urbanski --- datafusion/core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 7547c43b071a7..31c56b933625a 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -202,7 +202,7 @@ test-utils = { path = "../../test-utils" } tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot", "fs"] } [package.metadata.cargo-machete] -ignored = ["datafusion-doc", "datafusion-macros", "dashmap"] +ignored = ["datafusion-doc", "datafusion-macros", "dashmap", "getrandom"] [target.'cfg(not(target_os = "windows"))'.dev-dependencies] nix = { version = "0.31.3", features = ["fs"] } From 65a1ef7ae243e844e895bc8295e255dbfa9cd2ba Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 04:32:48 -0400 Subject: [PATCH 093/109] Keep filesystem test helpers off browser targets Signed-off-by: Ethan Urbanski --- datafusion/core/src/test_util/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index d70c0d186d007..66097b87631d6 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -25,7 +25,9 @@ pub mod csv; use futures::Stream; use std::collections::HashMap; use std::fmt::Formatter; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use std::fs::File; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -54,6 +56,7 @@ use std::pin::Pin; use async_trait::async_trait; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use tempfile::TempDir; // backwards compatibility #[cfg(feature = "parquet")] @@ -148,6 +151,7 @@ pub async fn plan_and_collect( } /// Generate CSV partitions within the supplied directory +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub fn populate_csv_partitions( tmp_dir: &TempDir, partition_count: usize, From 6329bccdb84ff09f7d897093a172c64acf40ae27 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 04:44:30 -0400 Subject: [PATCH 094/109] Test browser compression as unavailable Signed-off-by: Ethan Urbanski --- datafusion/wasmtest/src/lib.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs index f545ccf19306a..e27e2c51f9980 100644 --- a/datafusion/wasmtest/src/lib.rs +++ b/datafusion/wasmtest/src/lib.rs @@ -263,8 +263,9 @@ mod test { ); } + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] #[wasm_bindgen_test(unsupported = tokio::test)] - async fn test_csv_read_xz_compressed() { + async fn test_native_csv_read_xz_compressed() { let csv_data = "id,value\n1,a\n2,b\n3,c\n"; let input = Bytes::from(csv_data.as_bytes().to_vec()); let input_stream = @@ -313,4 +314,26 @@ mod test { +----+-------+" ); } + + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + #[wasm_bindgen_test] + fn unsupported_compression_reports_codec_operation_and_target() { + for (codec, compression) in [ + ("xz", FileCompressionType::XZ), + ("zstd", FileCompressionType::ZSTD), + ] { + let stream = stream::iter(vec![Ok::( + Bytes::from_static(b"test"), + )]) + .boxed(); + let error = match compression.convert_to_compress_stream(stream) { + Ok(_) => panic!("native compression backend must be unavailable"), + Err(error) => error, + }; + let message = error.to_string(); + assert!(message.contains("stream compression"), "{message}"); + assert!(message.contains(codec), "{message}"); + assert!(message.contains("wasm32-unknown-unknown"), "{message}"); + } + } } From 8171028112751d8c915fee8de742bb04bf3db45c Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:04:06 -0400 Subject: [PATCH 095/109] Define the DataFusion browser runtime profile Signed-off-by: Ethan Urbanski (cherry picked from commit f6421dd7c6bec64e03fdb6ae02655875ec950d4c) --- .github/workflows/wasm-browser.yml | 96 +++++++++++++++++ datafusion/core/src/lib.rs | 1 + datafusion/execution/Cargo.toml | 10 +- datafusion/execution/src/disk_manager.rs | 97 ++++++++++++++--- datafusion/wasmtest/Cargo.toml | 2 +- datafusion/wasmtest/src/lib.rs | 101 ++++++++++++++++-- datafusion/wasmtest/tests/browser-server.mjs | 104 +++++++++++++++++++ docs/source/user-guide/features.md | 3 + docs/source/user-guide/wasm.md | 41 ++++++++ 9 files changed, 423 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/wasm-browser.yml create mode 100644 datafusion/wasmtest/tests/browser-server.mjs create mode 100644 docs/source/user-guide/wasm.md diff --git a/.github/workflows/wasm-browser.yml b/.github/workflows/wasm-browser.yml new file mode 100644 index 0000000000000..b6a6da565a8e1 --- /dev/null +++ b/.github/workflows/wasm-browser.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Browser WASM + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + minimal-toolchain: + name: No native compiler + runs-on: ubuntu-latest + container: + image: debian:bookworm-slim + env: + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: ld.lld + steps: + - name: Install transport tools + run: | + apt-get update + apt-get install --yes --no-install-recommends ca-certificates curl git libc6-dev lld + rm -rf /var/lib/apt/lists/* + - uses: actions/checkout@v6 + - name: Install Rust and wasm32 + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | + sh -s -- -y --profile minimal --default-toolchain 1.94.0 + "$HOME/.cargo/bin/rustup" target add wasm32-unknown-unknown + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Verify minimal toolchain + run: | + test -z "${RUSTFLAGS:-}" + for tool in cc gcc clang c++ g++ cmake; do + if command -v "$tool"; then + echo "unexpected native build tool: $tool" >&2 + exit 1 + fi + done + command -v ld.lld + - name: Check browser graph + run: | + cargo check -p datafusion --target wasm32-unknown-unknown --locked + ci/scripts/check_wasm_dependency_policy.sh + + browser-runtime: + name: ${{ matrix.browser }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + browser: + - chrome + - firefox + steps: + - uses: actions/checkout@v6 + - run: rustup target add wasm32-unknown-unknown + - uses: taiki-e/install-action@v2 + with: + tool: wasm-pack + - name: Run browser runtime profile + env: + RUSTFLAGS: -C debuginfo=none + run: | + node datafusion/wasmtest/tests/browser-server.mjs > browser-server.log 2>&1 & + server_pid=$! + trap 'kill "$server_pid"' EXIT + for attempt in $(seq 1 30); do + if curl --fail --silent --head \ + http://127.0.0.1:9876/tpch_region_small.parquet >/dev/null; then + break + fi + if [ "$attempt" -eq 30 ]; then + cat browser-server.log + exit 1 + fi + sleep 1 + done + DATAFUSION_WASM_HTTP_PARQUET_URL=\ +http://127.0.0.1:9876/tpch_region_small.parquet \ + wasm-pack test --headless --${{ matrix.browser }} datafusion/wasmtest diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index 3170f4be7f683..6e8f10de1596f 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -899,6 +899,7 @@ pub mod variable { pub mod test; mod schema_equivalence; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub mod test_util; #[cfg(doctest)] diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.toml index c9d4acd3644ba..8fcd264bb8f1c 100644 --- a/datafusion/execution/Cargo.toml +++ b/datafusion/execution/Cargo.toml @@ -62,16 +62,18 @@ datafusion-expr = { workspace = true, default-features = false } datafusion-physical-expr-common = { workspace = true, default-features = false } futures = { workspace = true } log = { workspace = true } -object_store = { workspace = true, features = ["fs"] } +object_store = { workspace = true } parking_lot = { workspace = true } parquet = { workspace = true, optional = true } pin-project-lite = { workspace = true } -rand = { workspace = true } -tempfile = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true, features = ["io"] } url = { workspace = true } -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] + +[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies] +object_store = { workspace = true, features = ["fs"] } +rand = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = ["fs"] } [dev-dependencies] diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs index 313379f01291f..869d22550984b 100644 --- a/datafusion/execution/src/disk_manager.rs +++ b/datafusion/execution/src/disk_manager.rs @@ -17,20 +17,34 @@ //! [`DiskManager`]: Manages files generated during query execution -use crate::spill_file::{SpillFile, SpillWriter, TempFileFactory}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +use crate::spill_file::SpillWriter; +use crate::spill_file::{SpillFile, TempFileFactory}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use bytes::Bytes; -use datafusion_common::human_readable_size; -use datafusion_common::{DataFusionError, Result, config_err, resources_datafusion_err}; -#[cfg(not(target_arch = "wasm32"))] +use datafusion_common::{DataFusionError, Result, config_err}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +use datafusion_common::{human_readable_size, resources_datafusion_err}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use futures::StreamExt; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use log::debug; use parking_lot::Mutex; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use rand::{Rng, rng}; use std::fmt::Debug; -use std::path::{Path, PathBuf}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use tempfile::{Builder, NamedTempFile, TempDir}; + +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +type LocalDirectories = Vec>; +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] +type LocalDirectories = (); pub const DEFAULT_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB pub const DEFAULT_MAX_SPILL_MERGE_FAN_IN: usize = 0; @@ -57,7 +71,16 @@ impl Debug for DiskManagerBuilder { impl Default for DiskManagerBuilder { fn default() -> Self { Self { - mode: DiskManagerMode::OsTmpDirectory, + mode: { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + { + DiskManagerMode::OsTmpDirectory + } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + { + DiskManagerMode::Disabled + } + }, max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE, max_spill_merge_fan_in: DEFAULT_MAX_SPILL_MERGE_FAN_IN, } @@ -115,6 +138,7 @@ impl DiskManagerBuilder { /// Create a DiskManager given the builder pub fn build(self) -> Result { match self.mode { + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] DiskManagerMode::OsTmpDirectory => Ok(DiskManager { local_dirs: Mutex::new(Some(vec![])), max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size), @@ -123,6 +147,7 @@ impl DiskManagerBuilder { active_files_count: Arc::new(AtomicUsize::new(0)), factory: None, }), + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] DiskManagerMode::Directories(conf_dirs) => { let local_dirs = create_local_dirs(&conf_dirs)?; debug!( @@ -137,6 +162,13 @@ impl DiskManagerBuilder { factory: None, }) } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + DiskManagerMode::OsTmpDirectory | DiskManagerMode::Directories(_) => { + Err(DataFusionError::NotImplemented( + "Filesystem-backed spilling is unavailable on target wasm32-unknown-unknown" + .to_owned(), + )) + } DiskManagerMode::Disabled => Ok(DiskManager { local_dirs: Mutex::new(None), max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size), @@ -194,7 +226,7 @@ pub struct DiskManager { /// /// If `Some(vec![])` a new OS specified temporary directory will be created /// If `None` an error will be returned (configured not to spill) - local_dirs: Mutex>>>, + local_dirs: Mutex>, /// The maximum amount of data (in bytes) stored inside the temporary directories. /// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime /// without requiring exclusive (`&mut`) access to the `DiskManager`. @@ -317,15 +349,22 @@ impl DiskManager { /// Returns the temporary directory paths pub fn temp_dir_paths(&self) -> Vec { - self.local_dirs - .lock() - .as_ref() - .map(|dirs| { - dirs.iter() - .map(|temp_dir| temp_dir.path().to_path_buf()) - .collect() - }) - .unwrap_or_default() + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + { + self.local_dirs + .lock() + .as_ref() + .map(|dirs| { + dirs.iter() + .map(|temp_dir| temp_dir.path().to_path_buf()) + .collect() + }) + .unwrap_or_default() + } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + { + Vec::new() + } } /// Return true if this disk manager supports creating temporary @@ -339,6 +378,7 @@ impl DiskManager { /// /// If the file can not be created for some reason, returns an /// error message referencing the request description + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub fn create_tmp_file( self: &Arc, request_description: &str, @@ -380,6 +420,19 @@ impl DiskManager { disk_manager: Arc::clone(self), })) } + + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + pub fn create_tmp_file( + self: &Arc, + request_description: &str, + ) -> Result> { + if let Some(factory) = &self.factory { + return factory.create_temp_file(request_description); + } + Err(DataFusionError::NotImplemented(format!( + "Temporary spill file creation for {request_description} is unavailable on target wasm32-unknown-unknown; the browser profile is memory-only" + ))) + } } /// A wrapper around a [`NamedTempFile`] that also contains @@ -394,6 +447,7 @@ impl DiskManager { /// Once all references to this file are dropped, the file is deleted, and the /// disk usage is subtracted from the disk manager's total. #[derive(Debug)] +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub struct RefCountedTempFile { /// The reference to the directory in which temporary files are created to ensure /// it is not cleaned up prior to the NamedTempFile @@ -409,6 +463,7 @@ pub struct RefCountedTempFile { disk_manager: Arc, } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl Clone for RefCountedTempFile { fn clone(&self) -> Self { Self { @@ -420,6 +475,7 @@ impl Clone for RefCountedTempFile { } } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl RefCountedTempFile { pub fn path(&self) -> &Path { self.tempfile.path() @@ -435,6 +491,7 @@ impl RefCountedTempFile { } /// When the temporary file is dropped, subtract its disk usage from the disk manager's total +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl Drop for RefCountedTempFile { fn drop(&mut self) { // Only subtract disk usage when this is the last reference to the file @@ -453,6 +510,7 @@ impl Drop for RefCountedTempFile { } /// Setup local dirs by creating one new dir in each of the given dirs +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] fn create_local_dirs(local_dirs: &[PathBuf]) -> Result>> { local_dirs .iter() @@ -469,12 +527,14 @@ fn create_local_dirs(local_dirs: &[PathBuf]) -> Result>> { .collect() } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub struct FileSpillWriter { file: std::fs::File, disk_manager: Arc, current_file_disk_usage: Arc, } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl std::io::Write for FileSpillWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { let len = buf.len() as u64; @@ -515,6 +575,7 @@ impl std::io::Write for FileSpillWriter { } } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl SpillWriter for FileSpillWriter { fn finish(&mut self) -> Result<()> { // flush() is handled by Arrow, nothing left to do here @@ -522,6 +583,7 @@ impl SpillWriter for FileSpillWriter { } } +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] impl SpillFile for RefCountedTempFile { fn path(&self) -> Option<&Path> { Some(self.tempfile.path()) @@ -585,7 +647,8 @@ impl SpillFile for RefCountedTempFile { })) } } -#[cfg(test)] + +#[cfg(all(test, not(all(target_arch = "wasm32", target_os = "unknown"))))] mod tests { use super::*; diff --git a/datafusion/wasmtest/Cargo.toml b/datafusion/wasmtest/Cargo.toml index d6cea2e68d384..ef85dd1384180 100644 --- a/datafusion/wasmtest/Cargo.toml +++ b/datafusion/wasmtest/Cargo.toml @@ -62,7 +62,7 @@ wasm-bindgen = "0.2.99" [dev-dependencies] bytes = { workspace = true } futures = { workspace = true } -object_store = { workspace = true } +object_store = { workspace = true, features = ["http"] } # needs to be compiled tokio = { workspace = true } url = { workspace = true } diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs index e27e2c51f9980..99a822679aace 100644 --- a/datafusion/wasmtest/src/lib.rs +++ b/datafusion/wasmtest/src/lib.rs @@ -99,6 +99,8 @@ mod test { use datafusion_physical_plan::collect; use datafusion_sql::parser::DFParser; use futures::{StreamExt, TryStreamExt, stream}; + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + use object_store::{ClientOptions, http::HttpBuilder}; use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path}; use url::Url; use wasm_bindgen_test::wasm_bindgen_test; @@ -234,7 +236,7 @@ mod test { writer.write(&batch).unwrap(); writer.close().unwrap(); - let session_ctx = SessionContext::new(); + let session_ctx = get_ctx(); let store = InMemory::new(); let path = Path::from("a.parquet"); @@ -247,19 +249,24 @@ mod test { .await .unwrap(); - let df = session_ctx.sql("SELECT * FROM a").await.unwrap(); + let df = session_ctx + .sql( + "SELECT value, SUM(id) AS total \ + FROM a WHERE id >= 2 GROUP BY value ORDER BY total DESC", + ) + .await + .unwrap(); let result = df.collect().await.unwrap(); assert_eq!( batches_to_string(&result), - "+----+-------+\n\ - | id | value |\n\ - +----+-------+\n\ - | 1 | a |\n\ - | 2 | b |\n\ - | 3 | c |\n\ - +----+-------+" + "+-------+-------+\n\ + | value | total |\n\ + +-------+-------+\n\ + | c | 3 |\n\ + | b | 2 |\n\ + +-------+-------+" ); } @@ -314,7 +321,6 @@ mod test { +----+-------+" ); } - #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] #[wasm_bindgen_test] fn unsupported_compression_reports_codec_operation_and_target() { @@ -336,4 +342,79 @@ mod test { assert!(message.contains("wasm32-unknown-unknown"), "{message}"); } } + + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + #[wasm_bindgen_test] + fn browser_profile_rejects_filesystem_spilling() { + let configured = DiskManagerBuilder::default() + .with_mode(DiskManagerMode::OsTmpDirectory) + .build() + .expect_err("browser profile must reject a filesystem disk manager"); + assert!( + configured + .to_string() + .contains("Filesystem-backed spilling is unavailable"), + "{configured}" + ); + + let disabled = Arc::new( + DiskManagerBuilder::default() + .with_mode(DiskManagerMode::Disabled) + .build() + .unwrap(), + ); + assert!(!disabled.tmp_files_enabled()); + let spill = disabled + .create_tmp_file("browser query") + .expect_err("browser spill must fail before filesystem access"); + assert!(spill.to_string().contains("browser profile is memory-only")); + } + + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + #[wasm_bindgen_test] + async fn browser_http_parquet_projection_filter_order_and_aggregate() { + let Some(table_url) = option_env!("DATAFUSION_WASM_HTTP_PARQUET_URL") else { + return; + }; + + let table_url = Url::parse(table_url).unwrap(); + let mut origin = table_url.clone(); + origin.set_path("/"); + origin.set_query(None); + origin.set_fragment(None); + + let store = HttpBuilder::new() + .with_url(origin.as_str()) + .with_client_options(ClientOptions::new().with_allow_http(true)) + .build() + .unwrap(); + + let ctx = get_ctx(); + ctx.register_object_store(&origin, Arc::new(store)); + ctx.register_parquet("region", table_url.as_str(), Default::default()) + .await + .unwrap(); + + let rows = ctx + .sql( + "SELECT r_name, SUM(r_regionkey) AS total \ + FROM region WHERE r_regionkey >= 3 \ + GROUP BY r_name ORDER BY total DESC", + ) + .await + .unwrap() + .collect() + .await + .unwrap(); + + assert_eq!( + batches_to_string(&rows), + "+-------------+-------+\n\ + | r_name | total |\n\ + +-------------+-------+\n\ + | MIDDLE EAST | 4 |\n\ + | EUROPE | 3 |\n\ + +-------------+-------+" + ); + } } diff --git a/datafusion/wasmtest/tests/browser-server.mjs b/datafusion/wasmtest/tests/browser-server.mjs new file mode 100644 index 0000000000000..4fc0ad6bc5225 --- /dev/null +++ b/datafusion/wasmtest/tests/browser-server.mjs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; + +const fixtureUrl = new URL( + "../../core/tests/data/tpch_region_small.parquet", + import.meta.url, +); +const fixture = await readFile(fixtureUrl); +const etag = `"${createHash("sha256").update(fixture).digest("hex")}"`; +const port = Number.parseInt(process.env.PORT ?? "9876", 10); + +function setCorsHeaders(response) { + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setHeader("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS"); + response.setHeader("Access-Control-Allow-Headers", "Range, If-Range"); + response.setHeader( + "Access-Control-Expose-Headers", + "Accept-Ranges, Content-Encoding, Content-Length, Content-Range, ETag", + ); +} + +const server = createServer((request, response) => { + setCorsHeaders(response); + + if (request.method === "OPTIONS") { + response.writeHead(204); + response.end(); + return; + } + + if (request.url !== "/tpch_region_small.parquet") { + response.writeHead(404); + response.end(); + return; + } + + response.setHeader("Accept-Ranges", "bytes"); + response.setHeader("Content-Encoding", "identity"); + response.setHeader("Content-Type", "application/vnd.apache.parquet"); + response.setHeader("ETag", etag); + + if (request.method === "HEAD") { + response.setHeader("Content-Length", fixture.length); + response.writeHead(200); + response.end(); + return; + } + + const range = request.headers.range; + const ifRange = request.headers["if-range"]; + if (range && (!ifRange || ifRange === etag)) { + const match = /^bytes=(\d+)-(\d+)$/.exec(range); + if (!match) { + response.writeHead(416); + response.end(); + return; + } + + const start = Number.parseInt(match[1], 10); + const end = Math.min(Number.parseInt(match[2], 10), fixture.length - 1); + if (start > end || start >= fixture.length) { + response.setHeader("Content-Range", `bytes */${fixture.length}`); + response.writeHead(416); + response.end(); + return; + } + + const body = fixture.subarray(start, end + 1); + response.setHeader("Content-Length", body.length); + response.setHeader( + "Content-Range", + `bytes ${start}-${end}/${fixture.length}`, + ); + response.writeHead(206); + response.end(body); + return; + } + + response.setHeader("Content-Length", fixture.length); + response.writeHead(200); + response.end(fixture); +}); + +server.listen(port, "127.0.0.1", () => { + console.log(`browser parquet server listening on http://127.0.0.1:${port}`); +}); diff --git a/docs/source/user-guide/features.md b/docs/source/user-guide/features.md index 967e81e681f50..a2acca2c5f4b0 100644 --- a/docs/source/user-guide/features.md +++ b/docs/source/user-guide/features.md @@ -19,6 +19,9 @@ # Features +See [WebAssembly browser profile](./wasm.md) for the narrower set of +capabilities tested on `wasm32-unknown-unknown`. + ## General - [x] SQL Parser diff --git a/docs/source/user-guide/wasm.md b/docs/source/user-guide/wasm.md new file mode 100644 index 0000000000000..287f159484724 --- /dev/null +++ b/docs/source/user-guide/wasm.md @@ -0,0 +1,41 @@ + + +# WebAssembly browser profile + +DataFusion has a deliberately narrow browser profile for +`wasm32-unknown-unknown`. It covers in-memory execution and Parquet reads from +an asynchronously registered object store. The tested session uses one target +partition and a disabled disk manager. + +The browser profile currently verifies projection, filtering, ordering, and +aggregation over Snappy Parquet in Chrome and Firefox. A browser HTTP store must +be registered explicitly, and its server must expose the HTTP range and +validator headers required by the object-store client. + +Filesystem-backed object stores, temporary files, spilling, Tokio's +multi-threaded runtime, and generalized multi-partition execution are outside +this profile. Configuring a filesystem disk manager or requesting a spill +returns an operation-time error before filesystem access. + +Gzip and bzip2 file streams remain available. Xz and zstd file-stream formats +are recognized, but compression or decompression returns an error that names +the operation, codec, and `wasm32-unknown-unknown` target. Parquet and Arrow IPC +have their own codec availability rules; a file's metadata may be readable even +when a compressed data page is not. From 0cb694d17af11e0954d6a935e59bdfed5cf383cf Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:07:49 -0400 Subject: [PATCH 096/109] Link minimal WASM CI without a host compiler Signed-off-by: Ethan Urbanski (cherry picked from commit 54dde47ec65b76402f7229e59ae262c10e543dee) --- .github/workflows/wasm-browser.yml | 5 +-- ci/scripts/compiler-free-linker.sh | 49 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100755 ci/scripts/compiler-free-linker.sh diff --git a/.github/workflows/wasm-browser.yml b/.github/workflows/wasm-browser.yml index b6a6da565a8e1..8089e2458a7c7 100644 --- a/.github/workflows/wasm-browser.yml +++ b/.github/workflows/wasm-browser.yml @@ -29,12 +29,12 @@ jobs: container: image: debian:bookworm-slim env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: ld.lld + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: ${{ github.workspace }}/ci/scripts/compiler-free-linker.sh steps: - name: Install transport tools run: | apt-get update - apt-get install --yes --no-install-recommends ca-certificates curl git libc6-dev lld + apt-get install --yes --no-install-recommends ca-certificates curl git libc6-dev libgcc-12-dev lld rm -rf /var/lib/apt/lists/* - uses: actions/checkout@v6 - name: Install Rust and wasm32 @@ -53,6 +53,7 @@ jobs: fi done command -v ld.lld + test -x "$CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER" - name: Check browser graph run: | cargo check -p datafusion --target wasm32-unknown-unknown --locked diff --git a/ci/scripts/compiler-free-linker.sh b/ci/scripts/compiler-free-linker.sh new file mode 100755 index 0000000000000..3060d2eec865f --- /dev/null +++ b/ci/scripts/compiler-free-linker.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# rustc normally invokes a C compiler as its GNU linker driver. The POC's +# minimal-toolchain job intentionally has no compiler, so provide the startup +# objects and system search paths that a driver would otherwise add. +set -eu + +readonly system_lib=/usr/lib/x86_64-linux-gnu +readonly gcc_lib=/usr/lib/gcc/x86_64-linux-gnu/12 +readonly dynamic_linker=/lib64/ld-linux-x86-64.so.2 + +for object in \ + "$system_lib/Scrt1.o" \ + "$system_lib/crti.o" \ + "$gcc_lib/crtbeginS.o" \ + "$gcc_lib/crtendS.o" \ + "$system_lib/crtn.o" +do + test -f "$object" +done + +exec ld.lld \ + "$system_lib/Scrt1.o" \ + "$system_lib/crti.o" \ + "$gcc_lib/crtbeginS.o" \ + -L/lib/x86_64-linux-gnu \ + -L"$system_lib" \ + -L"$gcc_lib" \ + "$@" \ + "$gcc_lib/crtendS.o" \ + "$system_lib/crtn.o" \ + --dynamic-linker="$dynamic_linker" From 79eec52627231c1a2e8be30359fd9ca254cd283c Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:15:57 -0400 Subject: [PATCH 097/109] Count distinct dependency universes in WASM policy Signed-off-by: Ethan Urbanski (cherry picked from commit 0941b326bc4d5a2a37feea0e99d96955bfa7eb99) --- ci/scripts/check_wasm_dependency_policy.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/scripts/check_wasm_dependency_policy.sh b/ci/scripts/check_wasm_dependency_policy.sh index 6e5e404ad345b..fe422d7a98a4e 100755 --- a/ci/scripts/check_wasm_dependency_policy.sh +++ b/ci/scripts/check_wasm_dependency_policy.sh @@ -68,14 +68,20 @@ for feature in "${denied_features[@]}"; do fi done -duplicates="$( +package_universes="$( cargo tree -p datafusion --target "$target" --locked \ - -d --prefix none -e normal,build + --prefix none -e normal,build --format '{p}' | + sed 's/ (\\*)$//' | + LC_ALL=C sort -u )" for package in arrow parquet object_store; do - if grep -Eq "^${package} v" <<<"$duplicates"; then + universe_count="$( + awk -v package="$package" '$1 == package { count++ } END { print count + 0 }' \ + <<<"$package_universes" + )" + if (( universe_count > 1 )); then printf 'duplicate %s source/version universe in %s graph\n' "$package" "$target" >&2 - grep -E "^${package} v" <<<"$duplicates" >&2 + awk -v package="$package" '$1 == package' <<<"$package_universes" >&2 exit 1 fi done From 653b46df8d041f06947867b8dd257a47c5d9a044 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:16:45 -0400 Subject: [PATCH 098/109] Normalize repeated Cargo tree entries Signed-off-by: Ethan Urbanski (cherry picked from commit 23adfabdbd32e9ccdad114626151f224008d2e8d) --- ci/scripts/check_wasm_dependency_policy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/check_wasm_dependency_policy.sh b/ci/scripts/check_wasm_dependency_policy.sh index fe422d7a98a4e..6f4116d611b1f 100755 --- a/ci/scripts/check_wasm_dependency_policy.sh +++ b/ci/scripts/check_wasm_dependency_policy.sh @@ -71,7 +71,7 @@ done package_universes="$( cargo tree -p datafusion --target "$target" --locked \ --prefix none -e normal,build --format '{p}' | - sed 's/ (\\*)$//' | + sed 's/ (\*)$//' | LC_ALL=C sort -u )" for package in arrow parquet object_store; do From 5b746f60a79da501fdd370a658a609c606252d07 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:19:06 -0400 Subject: [PATCH 099/109] Translate compiler-driver flags for raw LLD Signed-off-by: Ethan Urbanski (cherry picked from commit 43ebca177b148219455be1af57a56029a60afd9a) --- ci/scripts/compiler-free-linker.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/ci/scripts/compiler-free-linker.sh b/ci/scripts/compiler-free-linker.sh index 3060d2eec865f..6b3e1dabb256a 100755 --- a/ci/scripts/compiler-free-linker.sh +++ b/ci/scripts/compiler-free-linker.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,7 +20,7 @@ # rustc normally invokes a C compiler as its GNU linker driver. The POC's # minimal-toolchain job intentionally has no compiler, so provide the startup # objects and system search paths that a driver would otherwise add. -set -eu +set -euo pipefail readonly system_lib=/usr/lib/x86_64-linux-gnu readonly gcc_lib=/usr/lib/gcc/x86_64-linux-gnu/12 @@ -36,6 +36,25 @@ do test -f "$object" done +linker_args=() +for argument in "$@"; do + case "$argument" in + -m64) + linker_args+=("-m" "elf_x86_64") + ;; + -nodefaultlibs) + # The driver below supplies the startup objects and libraries explicitly. + ;; + -Wl,*) + IFS=',' read -r -a forwarded <<<"${argument#-Wl,}" + linker_args+=("${forwarded[@]}") + ;; + *) + linker_args+=("$argument") + ;; + esac +done + exec ld.lld \ "$system_lib/Scrt1.o" \ "$system_lib/crti.o" \ @@ -43,7 +62,7 @@ exec ld.lld \ -L/lib/x86_64-linux-gnu \ -L"$system_lib" \ -L"$gcc_lib" \ - "$@" \ + "${linker_args[@]}" \ "$gcc_lib/crtendS.o" \ "$system_lib/crtn.o" \ --dynamic-linker="$dynamic_linker" From 13126f0b437182c33c09048222a2ddfecb5068c5 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:20:05 -0400 Subject: [PATCH 100/109] Keep the browser workflow valid YAML Signed-off-by: Ethan Urbanski (cherry picked from commit 717ff90d4b3590310c83076f0e0b7d87b502de9e) --- .github/workflows/wasm-browser.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/wasm-browser.yml b/.github/workflows/wasm-browser.yml index 8089e2458a7c7..e853edbb8e95b 100644 --- a/.github/workflows/wasm-browser.yml +++ b/.github/workflows/wasm-browser.yml @@ -92,6 +92,5 @@ jobs: fi sleep 1 done - DATAFUSION_WASM_HTTP_PARQUET_URL=\ -http://127.0.0.1:9876/tpch_region_small.parquet \ + DATAFUSION_WASM_HTTP_PARQUET_URL=http://127.0.0.1:9876/tpch_region_small.parquet \ wasm-pack test --headless --${{ matrix.browser }} datafusion/wasmtest From 3f02e4de785ad9941a850e8b85041f14ea3fed74 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:39:36 -0400 Subject: [PATCH 101/109] Link host proc macros as shared libraries Signed-off-by: Ethan Urbanski (cherry picked from commit bedca33601931e104f16d791a0116ff77865b1b1) --- ci/scripts/compiler-free-linker.sh | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ci/scripts/compiler-free-linker.sh b/ci/scripts/compiler-free-linker.sh index 6b3e1dabb256a..aaec81ccec263 100755 --- a/ci/scripts/compiler-free-linker.sh +++ b/ci/scripts/compiler-free-linker.sh @@ -37,8 +37,13 @@ do done linker_args=() +shared_output=false for argument in "$@"; do case "$argument" in + -shared) + shared_output=true + linker_args+=("$argument") + ;; -m64) linker_args+=("-m" "elf_x86_64") ;; @@ -55,14 +60,23 @@ for argument in "$@"; do esac done +startup_objects=( + "$system_lib/crti.o" + "$gcc_lib/crtbeginS.o" +) +linker_tail=( + "$gcc_lib/crtendS.o" + "$system_lib/crtn.o" +) +if [[ "$shared_output" == false ]]; then + startup_objects=("$system_lib/Scrt1.o" "${startup_objects[@]}") + linker_tail+=("--dynamic-linker=$dynamic_linker") +fi + exec ld.lld \ - "$system_lib/Scrt1.o" \ - "$system_lib/crti.o" \ - "$gcc_lib/crtbeginS.o" \ + "${startup_objects[@]}" \ -L/lib/x86_64-linux-gnu \ -L"$system_lib" \ -L"$gcc_lib" \ "${linker_args[@]}" \ - "$gcc_lib/crtendS.o" \ - "$system_lib/crtn.o" \ - --dynamic-linker="$dynamic_linker" + "${linker_tail[@]}" From 761833336c9879afb7c0d27a75eba94954158dc2 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 00:41:37 -0400 Subject: [PATCH 102/109] Grant loopback access in headless Firefox tests Signed-off-by: Ethan Urbanski (cherry picked from commit db3fb062b6e908a83c086817dcf816271213be5a) --- datafusion/wasmtest/webdriver.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/wasmtest/webdriver.json b/datafusion/wasmtest/webdriver.json index f59a2be9955f1..e0592e93a530e 100644 --- a/datafusion/wasmtest/webdriver.json +++ b/datafusion/wasmtest/webdriver.json @@ -2,7 +2,8 @@ "moz:firefoxOptions": { "prefs": { "media.navigator.streams.fake": true, - "media.navigator.permission.disabled": true + "media.navigator.permission.disabled": true, + "permissions.default.loopback-network": 1 }, "args": [] }, @@ -12,4 +13,4 @@ "--use-fake-ui-for-media-stream" ] } -} \ No newline at end of file +} From 70a462331fb2e05ead3325db427645c09b0c035d Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 02:46:12 -0400 Subject: [PATCH 103/109] Allow Firefox Fetch adapter preflights Signed-off-by: Ethan Urbanski (cherry picked from commit 2d7bac5b0435042e5342d99cda7b9344fd4e291d) --- .github/workflows/wasm-browser.yml | 1 + datafusion/wasmtest/tests/browser-server.mjs | 5 +- .../wasmtest/tests/browser-server.test.mjs | 65 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 datafusion/wasmtest/tests/browser-server.test.mjs diff --git a/.github/workflows/wasm-browser.yml b/.github/workflows/wasm-browser.yml index e853edbb8e95b..1f3d09d8b0a47 100644 --- a/.github/workflows/wasm-browser.yml +++ b/.github/workflows/wasm-browser.yml @@ -78,6 +78,7 @@ jobs: env: RUSTFLAGS: -C debuginfo=none run: | + node --test datafusion/wasmtest/tests/browser-server.test.mjs node datafusion/wasmtest/tests/browser-server.mjs > browser-server.log 2>&1 & server_pid=$! trap 'kill "$server_pid"' EXIT diff --git a/datafusion/wasmtest/tests/browser-server.mjs b/datafusion/wasmtest/tests/browser-server.mjs index 4fc0ad6bc5225..082efb2e65745 100644 --- a/datafusion/wasmtest/tests/browser-server.mjs +++ b/datafusion/wasmtest/tests/browser-server.mjs @@ -30,7 +30,10 @@ const port = Number.parseInt(process.env.PORT ?? "9876", 10); function setCorsHeaders(response) { response.setHeader("Access-Control-Allow-Origin", "*"); response.setHeader("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS"); - response.setHeader("Access-Control-Allow-Headers", "Range, If-Range"); + response.setHeader( + "Access-Control-Allow-Headers", + "Range, If-Range, User-Agent", + ); response.setHeader( "Access-Control-Expose-Headers", "Accept-Ranges, Content-Encoding, Content-Length, Content-Range, ETag", diff --git a/datafusion/wasmtest/tests/browser-server.test.mjs b/datafusion/wasmtest/tests/browser-server.test.mjs new file mode 100644 index 0000000000000..4910f82e33257 --- /dev/null +++ b/datafusion/wasmtest/tests/browser-server.test.mjs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +async function unusedLoopbackPort() { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address(); + server.close(); + await once(server, "close"); + return port; +} + +test("preflight permits the headers used by browser Fetch adapters", async () => { + const port = await unusedLoopbackPort(); + const server = spawn(process.execPath, ["browser-server.mjs"], { + cwd: new URL(".", import.meta.url), + env: { ...process.env, PORT: String(port) }, + stdio: ["ignore", "pipe", "pipe"], + }); + + try { + await once(server.stdout, "data"); + const response = await fetch( + `http://127.0.0.1:${port}/tpch_region_small.parquet`, + { + method: "OPTIONS", + headers: { + Origin: "http://127.0.0.1:12345", + "Access-Control-Request-Headers": "range,if-range,user-agent", + "Access-Control-Request-Method": "HEAD", + }, + }, + ); + + assert.equal(response.status, 204); + assert.match( + response.headers.get("access-control-allow-headers") ?? "", + /(?:^|,\s*)User-Agent(?:,|$)/, + ); + } finally { + server.kill(); + await once(server, "exit"); + } +}); From 053d360077455000ed66d0a4dffe732332bdd07d Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 24 Jul 2026 02:46:12 -0400 Subject: [PATCH 104/109] Ignore compiler search paths in raw LLD Signed-off-by: Ethan Urbanski (cherry picked from commit 693aa0b5d2a3c925db963776a472d6144352116e) --- ci/scripts/compiler-free-linker.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/compiler-free-linker.sh b/ci/scripts/compiler-free-linker.sh index aaec81ccec263..324dfc91d05b0 100755 --- a/ci/scripts/compiler-free-linker.sh +++ b/ci/scripts/compiler-free-linker.sh @@ -50,6 +50,9 @@ for argument in "$@"; do -nodefaultlibs) # The driver below supplies the startup objects and libraries explicitly. ;; + -B*) + # rustc forwards its compiler search path, which raw ld.lld does not use. + ;; -Wl,*) IFS=',' read -r -a forwarded <<<"${argument#-Wl,}" linker_args+=("${forwarded[@]}") From d6c05b8ce0270d7530e4f9ee8658c77249a06d99 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 31 Jul 2026 00:30:28 -0400 Subject: [PATCH 105/109] Fix browser WASM spill assertion Signed-off-by: Ethan Urbanski --- datafusion/wasmtest/src/lib.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/datafusion/wasmtest/src/lib.rs b/datafusion/wasmtest/src/lib.rs index 99a822679aace..169ff8c36080f 100644 --- a/datafusion/wasmtest/src/lib.rs +++ b/datafusion/wasmtest/src/lib.rs @@ -81,6 +81,8 @@ mod test { use bytes::Bytes; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + use datafusion::prelude::CsvReadOptions; use datafusion::{ arrow::{ array::{ArrayRef, Int32Array, RecordBatch, StringArray}, @@ -88,7 +90,6 @@ mod test { }, datasource::MemTable, execution::context::SessionContext, - prelude::CsvReadOptions, }; use datafusion_common::{DataFusionError, test_util::batches_to_string}; use datafusion_execution::{ @@ -98,10 +99,14 @@ mod test { }; use datafusion_physical_plan::collect; use datafusion_sql::parser::DFParser; - use futures::{StreamExt, TryStreamExt, stream}; + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + use futures::TryStreamExt; + use futures::{StreamExt, stream}; + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + use object_store::PutPayload; #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] use object_store::{ClientOptions, http::HttpBuilder}; - use object_store::{ObjectStoreExt, PutPayload, memory::InMemory, path::Path}; + use object_store::{ObjectStoreExt, memory::InMemory, path::Path}; use url::Url; use wasm_bindgen_test::wasm_bindgen_test; @@ -364,9 +369,10 @@ mod test { .unwrap(), ); assert!(!disabled.tmp_files_enabled()); - let spill = disabled - .create_tmp_file("browser query") - .expect_err("browser spill must fail before filesystem access"); + let spill = match disabled.create_tmp_file("browser query") { + Ok(_) => panic!("browser spill must fail before filesystem access"), + Err(spill) => spill, + }; assert!(spill.to_string().contains("browser profile is memory-only")); } From c4dbaf455281d127a4d2836069604d586ad66d6a Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 31 Jul 2026 00:38:33 -0400 Subject: [PATCH 106/109] Silence browser WASM target warnings Signed-off-by: Ethan Urbanski --- datafusion/datasource-arrow/src/file_format.rs | 2 ++ datafusion/datasource-csv/src/source.rs | 7 ++++++- datafusion/datasource-json/src/source.rs | 1 + datafusion/datasource/src/boundary_stream.rs | 4 +++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rs index c50ad98dfca0b..b5b78af08bfaf 100644 --- a/datafusion/datasource-arrow/src/file_format.rs +++ b/datafusion/datasource-arrow/src/file_format.rs @@ -21,12 +21,14 @@ use std::collections::HashMap; use std::fmt::{self, Debug}; +#[cfg(not(target_arch = "wasm32"))] use std::io::{Seek, SeekFrom}; use std::sync::Arc; use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::ArrowError; use arrow::ipc::convert::fb_to_schema; +#[cfg(not(target_arch = "wasm32"))] use arrow::ipc::reader::{FileReader, StreamReader}; use arrow::ipc::writer::IpcWriteOptions; use arrow::ipc::{CompressionType, root_as_message}; diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 25ec311880405..73a831932315e 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -21,6 +21,7 @@ use datafusion_datasource::boundary_stream::AlignedBoundaryStream; use datafusion_datasource::projection::{ProjectionOpener, SplitProjection}; use datafusion_physical_plan::projection::ProjectionExprs; use std::fmt; +#[cfg(not(target_arch = "wasm32"))] use std::io::Read; use std::sync::Arc; @@ -38,7 +39,9 @@ use datafusion_common_runtime::JoinSet; use datafusion_datasource::file::FileSource; use datafusion_datasource::file_scan_config::FileScanConfig; use datafusion_execution::TaskContext; -use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet}; +#[cfg(not(target_arch = "wasm32"))] +use datafusion_physical_plan::metrics::BaselineMetrics; +use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_physical_plan::{ DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, }; @@ -179,6 +182,7 @@ impl CsvSource { } impl CsvSource { + #[cfg(not(target_arch = "wasm32"))] fn open(&self, reader: R) -> Result> { Ok(self.builder().build(reader)?) } @@ -361,6 +365,7 @@ impl FileOpener for CsvOpener { let store = Arc::clone(&self.object_store); let terminator = self.config.terminator(); + #[cfg(not(target_arch = "wasm32"))] let baseline_metrics = BaselineMetrics::new(&self.config.metrics, self.partition_index); diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rs index 8632d6b942bc1..e8425f43986b9 100644 --- a/datafusion/datasource-json/src/source.rs +++ b/datafusion/datasource-json/src/source.rs @@ -17,6 +17,7 @@ //! Execution plan for reading JSON files (line-delimited and array formats) +#[cfg(not(target_arch = "wasm32"))] use std::io::BufReader; use std::pin::Pin; use std::sync::Arc; diff --git a/datafusion/datasource/src/boundary_stream.rs b/datafusion/datasource/src/boundary_stream.rs index 7b1cfb814df31..53285fe69702c 100644 --- a/datafusion/datasource/src/boundary_stream.rs +++ b/datafusion/datasource/src/boundary_stream.rs @@ -28,7 +28,9 @@ use std::task::{Context, Poll}; use bytes::Bytes; use futures::stream::{BoxStream, Stream}; use futures::{StreamExt, TryFutureExt}; -use object_store::{GetOptions, GetRange, GetResultPayload, ObjectStore}; +#[cfg(not(target_arch = "wasm32"))] +use object_store::GetResultPayload; +use object_store::{GetOptions, GetRange, ObjectStore}; /// How far past `raw_end` the initial bounded fetch covers. If the terminating /// newline is not found within this window, `ScanningLastTerminator` issues From 7eaa3e6e1c4c23ba0c2061fbf21db29527dcf0b6 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 31 Jul 2026 00:39:58 -0400 Subject: [PATCH 107/109] Avoid unused browser CSV partition state Signed-off-by: Ethan Urbanski --- datafusion/datasource-csv/src/source.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 73a831932315e..9168b0bda8ea8 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -218,6 +218,7 @@ pub struct CsvOpener { config: Arc, file_compression_type: FileCompressionType, object_store: Arc, + #[cfg(not(target_arch = "wasm32"))] partition_index: usize, } @@ -232,6 +233,7 @@ impl CsvOpener { config, file_compression_type, object_store, + #[cfg(not(target_arch = "wasm32"))] partition_index: 0, } } @@ -248,13 +250,14 @@ impl FileSource for CsvSource { &self, object_store: Arc, base_config: &FileScanConfig, - partition_index: usize, + _partition_index: usize, ) -> Result> { let mut opener = Arc::new(CsvOpener { config: Arc::new(self.clone()), file_compression_type: base_config.file_compression_type, object_store, - partition_index, + #[cfg(not(target_arch = "wasm32"))] + partition_index: _partition_index, }) as Arc; opener = ProjectionOpener::try_new( self.projection.clone(), From 06a63c57c3c67047bb3bfc0110c08c598939d6ab Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 31 Jul 2026 00:52:21 -0400 Subject: [PATCH 108/109] poc: validate DataFusion on current WASM fork stack Signed-off-by: Ethan Urbanski --- Cargo.lock | 258 +++++++++++++----- Cargo.toml | 17 +- datafusion/common/src/nested_struct.rs | 12 +- .../core/src/datasource/file_format/csv.rs | 1 + datafusion/core/tests/sql/path_partition.rs | 2 + .../functions-aggregate/src/array_agg.rs | 2 +- datafusion/functions/src/core/arrow_field.rs | 4 +- .../functions/src/core/arrow_metadata.rs | 4 +- .../tests/cases/roundtrip_logical_plan.rs | 4 +- 9 files changed, 222 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 793b420b99dfb..8214a18837014 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,8 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b952ca5a8046ad741b60f142d6eca4aeebcad615694202bc64c5341f23e32c5b" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-arith", "arrow-array", @@ -188,8 +187,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a13b8d3008c4e9063c597a08f46446fe3fd5789277127672d6c0bdbb43b1ff" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -202,8 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9486151b2f0785bafc6fa04fc5c99fcb4495455662e58787ea32eaaed33c4192" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "ahash", "arrow-buffer", @@ -213,6 +210,7 @@ dependencies = [ "chrono-tz", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -221,8 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e4f9b23a0d7b613acb59fa20bdbe0f80ffdae6411498378340b3915e45f5b84" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,20 +242,18 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4776577a87794bfdf0b4e90e2ea12454fa7738ea2823c4be5b9d1851da7b434" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ad451ce4f98710828a455b96991b8f031deb2e67f5fcad6773f017e4a69c3a" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,7 +262,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.0", "chrono", "comfy-table", "half", @@ -279,8 +274,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa7bf96d6141a7bcca2eed57c7c9767d2a2175281857b8a7b68308992864784" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-cast", @@ -294,8 +288,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b38fe43e2e8704360f1464e6e8cc4fc381ef02cc4fb0192afa8df1aaa0115c66" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-buffer", "arrow-schema", @@ -307,8 +300,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42115e09dbb694b5955da998912121451c6910b338228cb80a5701370dba43ff" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-arith", "arrow-array", @@ -321,11 +313,10 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64 0.22.1", + "base64 0.23.0", "bytes", "futures", "once_cell", - "paste", "prost", "prost-types", "tonic", @@ -335,8 +326,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dac499fcbc6ba74ee0324057821d381929a48526a3966bd9dffb44aa06d98c" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,8 +341,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe05e916ddc50f4c7a363cd69c0ef5894fcee063517e9a0b8582f0c56746af6" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,8 +365,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e13dbdc2a9c053c10c7baa6e30faee04a180aa7ce88e471835850ce37abd20b" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -389,8 +377,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5a1f8c733d15260b305683472ee8ad89c62cbd706703ca873b90d051b41592" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,8 +389,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e4969dc350d571766247143ab36a5187d095d3d3690970408bc630d47c69e5" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "bitflags", "serde", @@ -414,8 +400,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402770dba90865359d98d1ef92ef16e23d75c0cca9c2c880c8a05468b7743bf9" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "ahash", "arrow-array", @@ -428,8 +413,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b0afbb8b9016700938291123df30838b89decc3213dba00852021988b170d3" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "arrow-array", "arrow-buffer", @@ -1013,7 +997,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -1378,6 +1362,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1512,6 +1506,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -3271,6 +3275,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.13" @@ -3845,6 +3861,55 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4065,9 +4130,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -4210,7 +4275,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ - "num-bigint", + "num-bigint 0.4.6", "num-complex", "num-integer", "num-iter", @@ -4228,6 +4293,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4269,7 +4344,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -4361,30 +4436,34 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +version = "0.14.1" +source = "git+https://github.com/daxis-io/arrow-rs-object-store?rev=d0066c218eaf3336bc6b5e5ca3141fe78e4fea8d#d0066c218eaf3336bc6b5e5ca3141fe78e4fea8d" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", "futures-util", + "gloo-timers", "http 1.4.0", + "http-body 1.0.1", "http-body-util", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "js-sys", + "md-5 0.11.0", + "nix", "parking_lot", "percent-encoding", "quick-xml", "rand 0.10.1", "reqwest", - "ring", "rustls-pki-types", "serde", "serde_json", @@ -4396,6 +4475,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -4476,8 +4556,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5302d4da74d6596a1f11f9928767995b53bca657cbeea1e4e8c5074f8a1157dd" +source = "git+https://github.com/daxis-io/arrow-rs?rev=3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578#3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" dependencies = [ "ahash", "arrow-array", @@ -4486,7 +4565,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.0", "brotli", "bytes", "chrono", @@ -4495,11 +4574,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "ring", "seq-macro", "simdutf8", @@ -4534,12 +4612,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pbjson" version = "0.8.0" @@ -4918,9 +4990,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -4952,6 +5024,7 @@ version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -5240,9 +5313,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -5261,11 +5334,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", + "rustls-platform-verifier", "sync_wrapper", "tokio", "tokio-rustls", @@ -5400,6 +5470,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -5761,6 +5858,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -5825,6 +5932,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqllogictest" version = "0.29.1" @@ -6944,9 +7057,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -6987,6 +7100,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "2.1.1" diff --git a/Cargo.toml b/Cargo.toml index 87c23cc456651..7afebedcc41c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,7 +176,7 @@ liblzma = { version = "0.4.6", features = ["static"] } log = "^0.4" memchr = "2.8.1" num-traits = { version = "0.2" } -object_store = { version = "0.13.2", default-features = false } +object_store = { version = "0.14.1", default-features = false, features = ["tokio", "web"] } parking_lot = "0.12" parquet = { version = "59.1.0", default-features = false, features = [ "arrow", @@ -295,3 +295,18 @@ debug = false debug-assertions = false strip = "debuginfo" incremental = false + +# Daxis-only integration pins. The upstream candidate branch intentionally +# remains free of fork URLs; this POC branch proves the complete dependency +# stack at immutable exact heads. +[patch.crates-io] +arrow = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-avro = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-buffer = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-data = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-flight = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-ipc = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-ord = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +arrow-schema = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } +object_store = { git = "https://github.com/daxis-io/arrow-rs-object-store", rev = "d0066c218eaf3336bc6b5e5ca3141fe78e4fea8d" } +parquet = { git = "https://github.com/daxis-io/arrow-rs", rev = "3ef75b2ac57fb1cbd40a98c2bf29adb86fbdb578" } diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index e915b91b911cc..da994f7a95196 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -1123,8 +1123,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1148,8 +1148,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1176,8 +1176,8 @@ mod tests { assert!(map.is_null(1)); let map0 = map.value(0); let entries = map0.as_any().downcast_ref::().unwrap(); - let keys = get_column_as!(entries, "keys", StringArray); - let vals = get_column_as!(entries, "values", Int32Array); + let keys = get_column_as!(entries, "key", StringArray); + let vals = get_column_as!(entries, "value", Int32Array); assert_eq!(keys.value(0), "a"); assert_eq!(vals.value(0), 1); } diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 90d7eb3b41388..b9cc06385996b 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -136,6 +136,7 @@ mod tests { }, range: Default::default(), attributes: Attributes::default(), + extensions: Default::default(), }) } diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs index 82a15eb401fc4..4fce490d3fce7 100644 --- a/datafusion/core/tests/sql/path_partition.rs +++ b/datafusion/core/tests/sql/path_partition.rs @@ -718,6 +718,7 @@ impl ObjectStore for MirroringObjectStore { payload, meta, attributes: Attributes::default(), + extensions: Default::default(), }) } @@ -789,6 +790,7 @@ impl ObjectStore for MirroringObjectStore { Ok(ListResult { common_prefixes: common_prefixes.into_iter().collect(), objects, + extensions: Default::default(), }) } diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index cfacd771968c2..eaf7f9addbc72 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -1741,7 +1741,7 @@ mod tests { acc2.update_batch(&[data(["b", "c", "a"])])?; acc1 = merge(acc1, acc2)?; - assert_eq!(acc1.size(), 282); + assert_eq!(acc1.size(), 290); Ok(()) } diff --git a/datafusion/functions/src/core/arrow_field.rs b/datafusion/functions/src/core/arrow_field.rs index dce7cff42ba80..1c1968cf14dde 100644 --- a/datafusion/functions/src/core/arrow_field.rs +++ b/datafusion/functions/src/core/arrow_field.rs @@ -80,8 +80,8 @@ impl ArrowFieldFunc { Arc::new(Field::new( "entries", DataType::Struct(Fields::from(vec![ - Field::new("keys", DataType::Utf8, false), - Field::new("values", DataType::Utf8, true), + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), ])), false, )), diff --git a/datafusion/functions/src/core/arrow_metadata.rs b/datafusion/functions/src/core/arrow_metadata.rs index a80f66f396731..d930f8cf78bdd 100644 --- a/datafusion/functions/src/core/arrow_metadata.rs +++ b/datafusion/functions/src/core/arrow_metadata.rs @@ -104,8 +104,8 @@ impl ScalarUDFImpl for ArrowMetadataFunc { Arc::new(Field::new( "entries", DataType::Struct(Fields::from(vec![ - Field::new("keys", DataType::Utf8, false), - Field::new("values", DataType::Utf8, true), + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), ])), false, )), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index 74f7253386764..ad213715e7bdf 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -2216,7 +2216,7 @@ fn round_trip_scalar_values_and_data_types() { Arc::new(Field::new( "entries", DataType::Struct(Fields::from(vec![ - Field::new("key", DataType::Int32, true), + Field::new("key", DataType::Int32, false), Field::new("value", DataType::Utf8, false), ])), false, @@ -2228,7 +2228,7 @@ fn round_trip_scalar_values_and_data_types() { Arc::new(Field::new( "entries", DataType::Struct(Fields::from(vec![ - Field::new("key", DataType::Int32, true), + Field::new("key", DataType::Int32, false), Field::new("value", DataType::Utf8, true), ])), false, From aebdb2958b5beb94f7891e3e8efbee169ac6f220 Mon Sep 17 00:00:00 2001 From: Ethan Urbanski Date: Fri, 31 Jul 2026 23:35:30 -0400 Subject: [PATCH 109/109] Preserve native CSV partition metrics lint Signed-off-by: Ethan Urbanski --- datafusion/datasource-csv/src/source.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 9168b0bda8ea8..64a125eddafbe 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -250,14 +250,17 @@ impl FileSource for CsvSource { &self, object_store: Arc, base_config: &FileScanConfig, - _partition_index: usize, + partition_index: usize, ) -> Result> { + #[cfg(target_arch = "wasm32")] + let _ = partition_index; + let mut opener = Arc::new(CsvOpener { config: Arc::new(self.clone()), file_compression_type: base_config.file_compression_type, object_store, #[cfg(not(target_arch = "wasm32"))] - partition_index: _partition_index, + partition_index, }) as Arc; opener = ProjectionOpener::try_new( self.projection.clone(),