Add allow_mixed_array_batches feature

CHANGELOG.md

@@ -25,6 +25,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * `From<isize>` and `From<usize>` implementations for `Scalar` and `Value`.
 * One-element `From` implementations for primitive numeric types on `Array` and `ArrayV`.
 * `TryFrom<Vec<Value>>` for `Value`, with mixed-variant collections rejected.
+* Fallible `try_from_arrays_with_field`, `try_from_field_array_chunks`,
+  `try_from_slices`, and `try_push_field_array` on `SuperArray`, returning
+  `Result` where the existing forms panic.
+* Support for **Mixed-type SuperArray chunks** behind the `allow_mixed_array_batches` feature flag. This feature is intended for transient works. Therefore, it:
+  * Relaxes the `ArrowType` homogeneity checks on the `SuperArray` constructors
+    and push methods so that separately typed chunks can be stored in one column.
+  * Adds `SuperArray::check_type_uniformity()`, gated to the same feature,
+    reporting whether all chunks have the same `ArrowType`.
+  * Ensures that a `Field`, when present, still guarantees that describes every chunk. Attaching a field to non-uniform chunks is rejected. Hence, it is not possible to create
+  a `SuperTable` with mixed `SuperArray`s either, as that would be contractually incorrect. 
+* With the feature off, behaviour is unchanged. 
 
 ### Fixed
 

Cargo.toml

@@ -122,6 +122,17 @@ value_type = []
 # over multiple inner objects of the same type, for memory-mapped streaming etc.
 chunked = []
 
+# Relaxes the assumption that SuperArray batch types must be homogenous,
+# allowing separately typed batches to form part of the one Vec<Array>.
+# Useful for dynamic workloads, but breaks from the official Arrow spec meaning
+# these will not convert over FFI unless first coerced.
+# Note this feature is specifically for when there is no `Field` annotation on
+# the `SuperArray`, and thus is intended for transient workloads for which it is
+# suitable. It is not possible to put these on a `SuperTable` and/or tabular data
+# as a permanent storage mechanism as that is contractually incorrect as far as 
+# tag-based `Field` semantics are concerned. 
+allow_mixed_array_batches = ["chunked"]
+
 # Int64-based string
 large_string = []
 

README.md

@@ -235,7 +235,7 @@ See `minarrow-pyo3/examples`
 
 ## Feature Flags
 
-Default features: `views`, `chunked`, `large_string`, `simd`, `select`.
+Default features: `views`, `chunked`, `large_string`, `simd`, `select`, `log`.
 
 | Feature | Description |
 |---------|-------------|
@@ -244,6 +244,7 @@ Default features: `views`, `chunked`, `large_string`, `simd`, `select`.
 | `large_string` | String arrays with 64-bit offsets |
 | `simd` | SIMD kernels for Bitmask and arithmetic |
 | `select` | Pandas-esque `.c()` / `.r()` selection |
+| `log` | Warning output via the `log` crate |
 
 Interop:
 
@@ -290,6 +291,7 @@ Extras:
 | `hash` | Hash and Eq for `Scalar` |
 | `size` | Byte size estimation |
 | `table_metadata` | Schema-level metadata map on `Table` |
+| `allow_mixed_array_batches` | Separately typed batches within one `SuperArray` |
 
 See [Cargo.toml](Cargo.toml) for the full list with detailed notes on each.
 

src/ffi/arrow_c_ffi.rs

@@ -2838,15 +2838,35 @@ unsafe extern "C" fn rb_producer_stream_release(stream: *mut ArrowArrayStream) {
 
 /// Creates an ArrowArrayStream that yields arrays (one per chunk).
 ///
-/// Used for SuperArray ("ChunkedArray") exchange.
+/// Used for SuperArray ("ChunkedArray") exchange. Every chunk must match
+/// `field`'s dtype, because the stream advertises one schema and consumers
+/// read each chunk's buffers against it.
 ///
 /// # Arguments
 /// * `chunks` - the arrays to yield
 /// * `field` - the field describing the array type
 ///
 /// # Returns
 /// A heap-allocated ArrowArrayStream.
+///
+/// # Panics
+/// With the `allow_mixed_array_batches` feature on, panics when a chunk's
+/// `ArrowType` differs from `field.dtype`. Callers holding mixed batches
+/// check `SuperArray::check_type_uniformity()` and coerce to one type
+/// before exporting.
 pub fn export_array_stream(chunks: Vec<Arc<Array>>, field: crate::Field) -> Box<ArrowArrayStream> {
+    #[cfg(feature = "allow_mixed_array_batches")]
+    for (i, chunk) in chunks.iter().enumerate() {
+        assert_eq!(
+            chunk.arrow_type(),
+            field.dtype,
+            "export_array_stream: chunk {i} ArrowType does not match the stream \
+             schema. Mixed-type batches cannot cross the FFI boundary: check \
+             `SuperArray::check_type_uniformity()` and coerce to one type \
+             before exporting."
+        );
+    }
+
     let holder = Box::new(ArrayStreamHolder {
         field,
         chunks,
@@ -2965,12 +2985,32 @@ impl HasLastError for ArrayViewStreamHolder {
 ///
 /// Used to export a windowed [`crate::SuperArrayV`] or any sequence of
 /// `ArrayV` chunks without materialising them. Each slice's `(offset, len)`
-/// is conveyed at the Arrow C layer.
+/// is conveyed at the Arrow C layer. Every slice must match `field`'s
+/// dtype, because the stream advertises one schema and consumers read each
+/// slice's buffers against it.
+///
+/// # Panics
+/// With the `allow_mixed_array_batches` feature on, panics when a slice's
+/// `ArrowType` differs from `field.dtype`. Callers holding mixed batches
+/// check `SuperArray::check_type_uniformity()` and coerce to one type
+/// before exporting.
 #[cfg(feature = "views")]
 pub fn export_array_view_stream(
     slices: Vec<crate::ArrayV>,
     field: crate::Field,
 ) -> Box<ArrowArrayStream> {
+    #[cfg(feature = "allow_mixed_array_batches")]
+    for (i, slice) in slices.iter().enumerate() {
+        assert_eq!(
+            slice.array.arrow_type(),
+            field.dtype,
+            "export_array_view_stream: slice {i} ArrowType does not match the \
+             stream schema. Mixed-type batches cannot cross the FFI boundary: \
+             check `SuperArray::check_type_uniformity()` and coerce to one \
+             type before exporting."
+        );
+    }
+
     let holder = Box::new(ArrayViewStreamHolder {
         field,
         slices,
@@ -4592,6 +4632,56 @@ mod tests {
         assert_eq!(inner1.data.as_slice(), &[10, 20]);
     }
 
+    #[cfg(all(feature = "views", feature = "allow_mixed_array_batches"))]
+    #[test]
+    #[should_panic(expected = "Mixed-type batches cannot cross the FFI boundary")]
+    fn test_export_array_view_stream_mixed_types_panics() {
+        use super::export_super_array_view_stream;
+        use crate::{ArrayV, SuperArrayV};
+
+        let mut ints = IntegerArray::<i32>::default();
+        for v in [1, 2, 3] {
+            ints.push(v);
+        }
+        let mut floats = FloatArray::<f64>::default();
+        for v in [1.0, 2.0, 3.0] {
+            floats.push(v);
+        }
+        let v1 = ArrayV::new(Array::from_int32(ints), 0, 3);
+        let v2 = ArrayV::new(Array::from_float64(floats), 0, 3);
+        let field = Arc::new(Field::new("x", ArrowType::Int32, false, None));
+        let super_view = SuperArrayV {
+            slices: vec![v1, v2],
+            len: 6,
+            field,
+        };
+
+        let _ = export_super_array_view_stream(&super_view);
+    }
+
+    #[cfg(feature = "allow_mixed_array_batches")]
+    #[test]
+    #[should_panic(expected = "Mixed-type batches cannot cross the FFI boundary")]
+    fn test_export_array_stream_mixed_types_panics() {
+        use super::export_array_stream;
+
+        let mut ints = IntegerArray::<i32>::default();
+        for v in [1, 2, 3] {
+            ints.push(v);
+        }
+        let mut floats = FloatArray::<f64>::default();
+        for v in [1.0, 2.0, 3.0] {
+            floats.push(v);
+        }
+        let chunks = vec![
+            Arc::new(Array::from_int32(ints)),
+            Arc::new(Array::from_float64(floats)),
+        ];
+        let field = Field::new("x", ArrowType::Int32, false, None);
+
+        let _ = export_array_stream(chunks, field);
+    }
+
     #[cfg(all(feature = "views", feature = "chunked"))]
     #[test]
     fn test_super_table_view_stream_round_trip() {