diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ec603..e9f75b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `From` and `From` implementations for `Scalar` and `Value`. * One-element `From` implementations for primitive numeric types on `Array` and `ArrayV`. * `TryFrom>` for `Value`, with mixed-variant collections rejected. +* Fallible `try_from_arrays_with_field`, `try_from_field_array_chunks`, + `try_from_slices`, and `try_push_field_array` on `SuperArray`, returning + `Result` where the existing forms panic. +* Support for **Mixed-type SuperArray chunks** behind the `allow_mixed_array_batches` feature flag. This feature is intended for transient works. Therefore, it: + * Relaxes the `ArrowType` homogeneity checks on the `SuperArray` constructors + and push methods so that separately typed chunks can be stored in one column. + * Adds `SuperArray::check_type_uniformity()`, gated to the same feature, + reporting whether all chunks have the same `ArrowType`. + * Ensures that a `Field`, when present, still guarantees that describes every chunk. Attaching a field to non-uniform chunks is rejected. Hence, it is not possible to create + a `SuperTable` with mixed `SuperArray`s either, as that would be contractually incorrect. +* With the feature off, behaviour is unchanged. ### Fixed diff --git a/Cargo.toml b/Cargo.toml index aec9b81..2f707c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,6 +122,17 @@ value_type = [] # over multiple inner objects of the same type, for memory-mapped streaming etc. chunked = [] +# Relaxes the assumption that SuperArray batch types must be homogenous, +# allowing separately typed batches to form part of the one Vec. +# Useful for dynamic workloads, but breaks from the official Arrow spec meaning +# these will not convert over FFI unless first coerced. +# Note this feature is specifically for when there is no `Field` annotation on +# the `SuperArray`, and thus is intended for transient workloads for which it is +# suitable. It is not possible to put these on a `SuperTable` and/or tabular data +# as a permanent storage mechanism as that is contractually incorrect as far as +# tag-based `Field` semantics are concerned. +allow_mixed_array_batches = ["chunked"] + # Int64-based string large_string = [] diff --git a/README.md b/README.md index 1ab88ed..f180910 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ See `minarrow-pyo3/examples` ## Feature Flags -Default features: `views`, `chunked`, `large_string`, `simd`, `select`. +Default features: `views`, `chunked`, `large_string`, `simd`, `select`, `log`. | Feature | Description | |---------|-------------| @@ -244,6 +244,7 @@ Default features: `views`, `chunked`, `large_string`, `simd`, `select`. | `large_string` | String arrays with 64-bit offsets | | `simd` | SIMD kernels for Bitmask and arithmetic | | `select` | Pandas-esque `.c()` / `.r()` selection | +| `log` | Warning output via the `log` crate | Interop: @@ -290,6 +291,7 @@ Extras: | `hash` | Hash and Eq for `Scalar` | | `size` | Byte size estimation | | `table_metadata` | Schema-level metadata map on `Table` | +| `allow_mixed_array_batches` | Separately typed batches within one `SuperArray` | See [Cargo.toml](Cargo.toml) for the full list with detailed notes on each. diff --git a/src/ffi/arrow_c_ffi.rs b/src/ffi/arrow_c_ffi.rs index d9f6e20..2a5a66e 100644 --- a/src/ffi/arrow_c_ffi.rs +++ b/src/ffi/arrow_c_ffi.rs @@ -2838,7 +2838,9 @@ unsafe extern "C" fn rb_producer_stream_release(stream: *mut ArrowArrayStream) { /// Creates an ArrowArrayStream that yields arrays (one per chunk). /// -/// Used for SuperArray ("ChunkedArray") exchange. +/// Used for SuperArray ("ChunkedArray") exchange. Every chunk must match +/// `field`'s dtype, because the stream advertises one schema and consumers +/// read each chunk's buffers against it. /// /// # Arguments /// * `chunks` - the arrays to yield @@ -2846,7 +2848,25 @@ unsafe extern "C" fn rb_producer_stream_release(stream: *mut ArrowArrayStream) { /// /// # Returns /// A heap-allocated ArrowArrayStream. +/// +/// # Panics +/// With the `allow_mixed_array_batches` feature on, panics when a chunk's +/// `ArrowType` differs from `field.dtype`. Callers holding mixed batches +/// check `SuperArray::check_type_uniformity()` and coerce to one type +/// before exporting. pub fn export_array_stream(chunks: Vec>, field: crate::Field) -> Box { + #[cfg(feature = "allow_mixed_array_batches")] + for (i, chunk) in chunks.iter().enumerate() { + assert_eq!( + chunk.arrow_type(), + field.dtype, + "export_array_stream: chunk {i} ArrowType does not match the stream \ + schema. Mixed-type batches cannot cross the FFI boundary: check \ + `SuperArray::check_type_uniformity()` and coerce to one type \ + before exporting." + ); + } + let holder = Box::new(ArrayStreamHolder { field, chunks, @@ -2965,12 +2985,32 @@ impl HasLastError for ArrayViewStreamHolder { /// /// Used to export a windowed [`crate::SuperArrayV`] or any sequence of /// `ArrayV` chunks without materialising them. Each slice's `(offset, len)` -/// is conveyed at the Arrow C layer. +/// is conveyed at the Arrow C layer. Every slice must match `field`'s +/// dtype, because the stream advertises one schema and consumers read each +/// slice's buffers against it. +/// +/// # Panics +/// With the `allow_mixed_array_batches` feature on, panics when a slice's +/// `ArrowType` differs from `field.dtype`. Callers holding mixed batches +/// check `SuperArray::check_type_uniformity()` and coerce to one type +/// before exporting. #[cfg(feature = "views")] pub fn export_array_view_stream( slices: Vec, field: crate::Field, ) -> Box { + #[cfg(feature = "allow_mixed_array_batches")] + for (i, slice) in slices.iter().enumerate() { + assert_eq!( + slice.array.arrow_type(), + field.dtype, + "export_array_view_stream: slice {i} ArrowType does not match the \ + stream schema. Mixed-type batches cannot cross the FFI boundary: \ + check `SuperArray::check_type_uniformity()` and coerce to one \ + type before exporting." + ); + } + let holder = Box::new(ArrayViewStreamHolder { field, slices, @@ -4592,6 +4632,56 @@ mod tests { assert_eq!(inner1.data.as_slice(), &[10, 20]); } + #[cfg(all(feature = "views", feature = "allow_mixed_array_batches"))] + #[test] + #[should_panic(expected = "Mixed-type batches cannot cross the FFI boundary")] + fn test_export_array_view_stream_mixed_types_panics() { + use super::export_super_array_view_stream; + use crate::{ArrayV, SuperArrayV}; + + let mut ints = IntegerArray::::default(); + for v in [1, 2, 3] { + ints.push(v); + } + let mut floats = FloatArray::::default(); + for v in [1.0, 2.0, 3.0] { + floats.push(v); + } + let v1 = ArrayV::new(Array::from_int32(ints), 0, 3); + let v2 = ArrayV::new(Array::from_float64(floats), 0, 3); + let field = Arc::new(Field::new("x", ArrowType::Int32, false, None)); + let super_view = SuperArrayV { + slices: vec![v1, v2], + len: 6, + field, + }; + + let _ = export_super_array_view_stream(&super_view); + } + + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + #[should_panic(expected = "Mixed-type batches cannot cross the FFI boundary")] + fn test_export_array_stream_mixed_types_panics() { + use super::export_array_stream; + + let mut ints = IntegerArray::::default(); + for v in [1, 2, 3] { + ints.push(v); + } + let mut floats = FloatArray::::default(); + for v in [1.0, 2.0, 3.0] { + floats.push(v); + } + let chunks = vec![ + Arc::new(Array::from_int32(ints)), + Arc::new(Array::from_float64(floats)), + ]; + let field = Field::new("x", ArrowType::Int32, false, None); + + let _ = export_array_stream(chunks, field); + } + #[cfg(all(feature = "views", feature = "chunked"))] #[test] fn test_super_table_view_stream_round_trip() { diff --git a/src/structs/chunked/super_array.rs b/src/structs/chunked/super_array.rs index c5e274e..f48615e 100644 --- a/src/structs/chunked/super_array.rs +++ b/src/structs/chunked/super_array.rs @@ -28,6 +28,28 @@ //! - Use `from_arrays()` when you don't need field metadata (e.g., Dam consolidation) //! - Use `from_arrays_with_field()` when field metadata is required //! +//! ## Mixed chunk types (`allow_mixed_array_batches` feature) +//! By default, all chunks in a `SuperArray` must have the same `ArrowType`. +//! The `allow_mixed_array_batches` feature permits different chunk types when +//! no `Field` is attached. +//! +//! A `Field` defines the type of every chunk and therefore always requires +//! type uniformity, regardless of this feature. Attaching a field to mixed +//! chunks is rejected. +//! +//! With `allow_mixed_array_batches` enabled, [`check_type_uniformity`] reports +//! whether all chunks have the same `ArrowType`. Operations that require a +//! uniform type, including consolidation, rechunking, FFI export, and field +//! attachment, can use this check before proceeding. +//! +//! Field-based constructors and `push_field_array` also provide `try_*` +//! variants that return `Result` when runtime data may not match the required +//! type. +//! +//! Mixed cases cannot be used in `SuperTable` and are rejected at the boundary +//! as it would violate contractual `Field`-based guarantees. Hence, these are +//! intended for transient workloads only. +//! //! ## Apache Arrow / Polars bridges (`cast_arrow` / `cast_polars` features) //! - `to_apache_arrow()` exports each chunk as an arrow-rs `ArrayRef`. //! - `to_polars()` builds a polars `Series` whose internal chunks mirror the SuperArray. @@ -149,8 +171,10 @@ impl SuperArray { /// Use this for streaming consolidation patterns where field metadata is not needed. /// /// # Panics - /// Panics if chunks have mismatched types. + /// Panics if chunks have mismatched types, unless the + /// `allow_mixed_array_batches` feature is on. pub fn from_arrays(chunks: Vec) -> Self { + #[cfg(not(feature = "allow_mixed_array_batches"))] if chunks.len() > 1 { let dtype = chunks[0].arrow_type(); for (i, chunk) in chunks.iter().enumerate().skip(1) { @@ -181,18 +205,41 @@ impl SuperArray { /// The field metadata applies to all chunks (they represent the same logical column). /// /// # Panics - /// Panics if chunks have mismatched types or don't match the field type. + /// Panics if any chunk does not match the field type. A `Field` present + /// on a SuperArray always describes every chunk, so this holds with the + /// `allow_mixed_array_batches` feature on as well. Mixed chunks belong + /// in the field-free constructors, and `try_from_arrays_with_field` is + /// the fallible equivalent. pub fn from_arrays_with_field(chunks: Vec, field: impl Into>) -> Self { + match Self::try_from_arrays_with_field(chunks, field) { + Ok(sa) => sa, + Err(e) => panic!("{e}"), + } + } + + /// Fallible form of `from_arrays_with_field`. + /// + /// Returns an error instead of panicking when a chunk does not match the + /// field type. This is appropriate for callers assembling from runtime data, where a + /// mismatch is an expected condition rather than a programming defect. + pub fn try_from_arrays_with_field( + chunks: Vec, + field: impl Into>, + ) -> Result { let field = field.into(); for (i, chunk) in chunks.iter().enumerate() { - assert_eq!( - chunk.arrow_type(), - field.dtype, - "Chunk {i} ArrowType mismatch (expected {:?}, got {:?})", - field.dtype, - chunk.arrow_type() - ); + if chunk.arrow_type() != field.dtype { + return Err(MinarrowError::IncompatibleTypeError { + from: "Array", + to: "SuperArray", + message: Some(format!( + "Chunk {i} ArrowType mismatch (expected {:?}, got {:?})", + field.dtype, + chunk.arrow_type() + )), + }); + } } #[cfg_attr(not(feature = "shared_dict"), allow(unused_mut))] @@ -205,13 +252,14 @@ impl SuperArray { }; #[cfg(feature = "shared_dict")] sa.rebuild_category_manager(); - sa + Ok(sa) } /// Constructs a SuperArray from raw `Array` chunks with null counts. /// /// # Panics - /// Panics if chunks have mismatched types or null_counts length doesn't match chunks length. + /// 1. If null_counts length does not match chunks length. + /// 2. On mismatched chunk types, unless the `allow_mixed_array_batches`feature is on. pub fn from_arrays_nc(chunks: Vec, null_counts: Vec) -> Self { assert_eq!( chunks.len(), @@ -221,6 +269,7 @@ impl SuperArray { chunks.len() ); + #[cfg(not(feature = "allow_mixed_array_batches"))] if chunks.len() > 1 { let dtype = chunks[0].arrow_type(); for (i, chunk) in chunks.iter().enumerate().skip(1) { @@ -251,31 +300,62 @@ impl SuperArray { /// /// Extracts field metadata and null counts from the chunks. /// - /// # Panics - /// Panics if chunks is empty or metadata/type/nullable mismatch is found. + /// ## Panics + /// Panics if chunks is empty, or on a type, nullability, or field name + /// mismatch between chunks. Every `FieldArray` carries a `Field`, and a + /// `Field` present on a SuperArray always describes every chunk, so these + /// checks hold with the `allow_mixed_array_batches` feature on as well. + /// `try_from_field_array_chunks` is the fallible equivalent. pub fn from_field_array_chunks(chunks: Vec) -> Self { - assert!( - !chunks.is_empty(), - "from_field_array_chunks: input chunks cannot be empty" - ); + match Self::try_from_field_array_chunks(chunks) { + Ok(sa) => sa, + Err(e) => panic!("{e}"), + } + } + + /// Fallible form of `from_field_array_chunks`. + /// + /// Returns an error instead of panicking when the chunk list is empty or + /// when the chunks' fields disagree on type, nullability, or name. This + /// suits callers assembling from runtime data, where a mismatch is an + /// expected condition rather than a programming defect. + pub fn try_from_field_array_chunks(chunks: Vec) -> Result { + if chunks.is_empty() { + return Err(MinarrowError::ShapeError { + message: "from_field_array_chunks: input chunks cannot be empty".to_string(), + }); + } let field = chunks[0].field.clone(); for (i, fa) in chunks.iter().enumerate().skip(1) { - assert_eq!( - fa.field.dtype, field.dtype, - "Chunk {i} ArrowType mismatch (expected {:?}, got {:?})", - field.dtype, fa.field.dtype - ); - assert_eq!( - fa.field.nullable, field.nullable, - "Chunk {i} nullability mismatch" - ); - assert_eq!( - fa.field.name, field.name, - "Chunk {i} field name mismatch (expected '{}', got '{}')", - field.name, fa.field.name - ); + if fa.field.dtype != field.dtype { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some(format!( + "Chunk {i} ArrowType mismatch (expected {:?}, got {:?})", + field.dtype, fa.field.dtype + )), + }); + } + if fa.field.nullable != field.nullable { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some(format!("Chunk {i} nullability mismatch")), + }); + } + if fa.field.name != field.name { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some(format!( + "Chunk {i} field name mismatch (expected '{}', got '{}')", + field.name, fa.field.name + )), + }); + } } let null_counts: Vec = chunks.iter().map(|fa| fa.null_count).collect(); @@ -291,7 +371,7 @@ impl SuperArray { }; #[cfg(feature = "shared_dict")] sa.rebuild_category_manager(); - sa + Ok(sa) } /// Construct from `Vec`. @@ -304,25 +384,51 @@ impl SuperArray { /// Materialises a `SuperArray` from an existing slice of `ArrayView` tuples, /// using the provided field metadata (applied to all slices). /// - /// Panics if the slice list is empty, or if any slice's type or nullability - /// does not match the provided field. + /// ## Panics + /// Panics if the slice list is empty, or if any slice's type or + /// nullability does not match the provided field. A `Field` present on a + /// SuperArray always describes every chunk, so these checks hold with the + /// `allow_mixed_array_batches` feature on as well. `try_from_slices` is + /// the fallible equivalent. #[cfg(feature = "views")] pub fn from_slices(slices: &[ArrayV], field: Arc) -> Self { - assert!(!slices.is_empty(), "from_slices requires non-empty slice"); + match Self::try_from_slices(slices, field) { + Ok(sa) => sa, + Err(e) => panic!("{e}"), + } + } + + /// Fallible form of `from_slices`. + /// + /// Returns an error instead of panicking when the slice list is empty or + /// when a slice's type or nullability does not match the provided field. + /// This is appropriate for callers assembling from runtime data, where a mismatch is + /// an expected condition rather than a programming defect. + #[cfg(feature = "views")] + pub fn try_from_slices(slices: &[ArrayV], field: Arc) -> Result { + if slices.is_empty() { + return Err(MinarrowError::ShapeError { + message: "from_slices requires non-empty slice".to_string(), + }); + } let mut arrays = Vec::with_capacity(slices.len()); let mut null_counts = Vec::with_capacity(slices.len()); for (i, view) in slices.iter().enumerate() { - assert_eq!( - view.array.arrow_type(), - field.dtype, - "Slice {i} ArrowType does not match field" - ); - assert_eq!( - view.array.is_nullable(), - field.nullable, - "Slice {i} nullability does not match field" - ); + if view.array.arrow_type() != field.dtype { + return Err(MinarrowError::IncompatibleTypeError { + from: "ArrayV", + to: "SuperArray", + message: Some(format!("Slice {i} ArrowType does not match field")), + }); + } + if view.array.is_nullable() != field.nullable { + return Err(MinarrowError::IncompatibleTypeError { + from: "ArrayV", + to: "SuperArray", + message: Some(format!("Slice {i} nullability does not match field")), + }); + } arrays.push(view.array.slice_clone(view.offset, view.len())); null_counts.push(view.null_count()); } @@ -337,7 +443,7 @@ impl SuperArray { }; #[cfg(feature = "shared_dict")] sa.rebuild_category_manager(); - sa + Ok(sa) } /// Returns a zero-copy view of this chunked array over the window `[offset..offset+len)`. @@ -474,20 +580,26 @@ impl SuperArray { /// `push_with_null_count()` to avoid recomputation. /// /// # Panics - /// Panics if the chunk type doesn't match existing chunks or field. + /// Panics if the chunk type does not match a `Field` this SuperArray + /// carries. A field-free SuperArray additionally panics on a mismatch + /// with the existing chunks, unless the `allow_mixed_array_batches` + /// feature is on. pub fn push(&mut self, chunk: Array) { - if let Some(first) = self.chunks.first() { - assert_eq!( - chunk.arrow_type(), - first.arrow_type(), - "Chunk ArrowType mismatch" - ); - } else if let Some(ref field) = self.field { + if let Some(ref field) = self.field { assert_eq!( chunk.arrow_type(), field.dtype, "Chunk ArrowType mismatch with field" ); + } else { + #[cfg(not(feature = "allow_mixed_array_batches"))] + if let Some(first) = self.chunks.first() { + assert_eq!( + chunk.arrow_type(), + first.arrow_type(), + "Chunk ArrowType mismatch" + ); + } } // If tracking null counts, compute from the array's null_mask if let Some(ref mut nc) = self.null_counts { @@ -503,19 +615,28 @@ impl SuperArray { /// Appends a raw array chunk with its null count. /// /// When the null count is already known this is slightly faster than `push` + /// + /// # Panics + /// Panics if the chunk type does not match a `Field` this SuperArray + /// carries. A field-free SuperArray additionally panics on a mismatch + /// with the existing chunks, unless the `allow_mixed_array_batches` + /// feature is on. pub fn push_with_null_count(&mut self, chunk: Array, null_count: usize) { - if let Some(first) = self.chunks.first() { - assert_eq!( - chunk.arrow_type(), - first.arrow_type(), - "Chunk ArrowType mismatch" - ); - } else if let Some(ref field) = self.field { + if let Some(ref field) = self.field { assert_eq!( chunk.arrow_type(), field.dtype, "Chunk ArrowType mismatch with field" ); + } else { + #[cfg(not(feature = "allow_mixed_array_batches"))] + if let Some(first) = self.chunks.first() { + assert_eq!( + chunk.arrow_type(), + first.arrow_type(), + "Chunk ArrowType mismatch" + ); + } } #[cfg_attr(not(feature = "shared_dict"), allow(unused_mut))] let mut chunk = chunk; @@ -534,21 +655,61 @@ impl SuperArray { /// If this SuperArray has no field metadata yet, it will be set from the chunk. /// /// # Panics - /// If the chunk does not match the expected type, nullability, or field name. + /// Panics if the chunk does not match the expected type, nullability, or + /// field name. The first push onto a field-free SuperArray attaches the + /// chunk's field, so every existing chunk must already match that type. A + /// `Field` present on a SuperArray always describes every chunk, so these + /// checks hold with the `allow_mixed_array_batches` feature on as well. + /// `try_push_field_array` is the fallible equivalent. pub fn push_field_array(&mut self, chunk: FieldArray) { + if let Err(e) = self.try_push_field_array(chunk) { + panic!("{e}"); + } + } + + /// Fallible form of `push_field_array`. + /// + /// Returns an error instead of panicking when the chunk does not match + /// the SuperArray's field, or when attaching the chunk's field to a + /// field-free SuperArray whose existing chunks are not uniformly of its + /// type. This is appropriate for callers appending runtime data, where a mismatch is + /// an expected condition rather than a programming defect. + pub fn try_push_field_array(&mut self, chunk: FieldArray) -> Result<(), MinarrowError> { if let Some(ref field) = self.field { - assert_eq!(chunk.field.dtype, field.dtype, "Chunk ArrowType mismatch"); - assert_eq!( - chunk.field.nullable, field.nullable, - "Chunk nullability mismatch" - ); - assert_eq!(chunk.field.name, field.name, "Chunk field name mismatch"); - } else if !self.chunks.is_empty() { - assert_eq!( - chunk.array.arrow_type(), - self.chunks[0].arrow_type(), - "Chunk ArrowType mismatch" - ); + if chunk.field.dtype != field.dtype { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some("Chunk ArrowType mismatch".to_string()), + }); + } + if chunk.field.nullable != field.nullable { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some("Chunk nullability mismatch".to_string()), + }); + } + if chunk.field.name != field.name { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some("Chunk field name mismatch".to_string()), + }); + } + } else { + for (i, existing) in self.chunks.iter().enumerate() { + if existing.arrow_type() != chunk.field.dtype { + return Err(MinarrowError::IncompatibleTypeError { + from: "FieldArray", + to: "SuperArray", + message: Some(format!( + "Chunk {i} ArrowType mismatch: a field cannot attach to a \ + SuperArray whose chunks are not uniformly of its type" + )), + }); + } + } } // Set field from first push if not already set @@ -566,6 +727,7 @@ impl SuperArray { } else { self.null_counts = Some(vec![chunk.null_count]); } + Ok(()) } /// Inserts rows from another SuperArray (or Array) at the specified index. @@ -959,6 +1121,24 @@ impl SuperArray { &self.chunks } + /// Reports whether all chunks have the same `ArrowType`. + /// + /// The `allow_mixed_array_batches` feature relaxes the homogeneity checks + /// at construction and on push, so separately typed chunks can be stored + /// in one column. Callers that require a uniform type (e.g. consolidation, + /// rechunking, FFI export) can check this method before proceeding. + /// + /// Returns `true` for an empty SuperArray. + #[cfg(feature = "allow_mixed_array_batches")] + pub fn check_type_uniformity(&self) -> bool { + let mut chunks = self.chunks.iter(); + let Some(first) = chunks.next() else { + return true; + }; + let dtype = first.arrow_type(); + chunks.all(|chunk| chunk.arrow_type() == dtype) + } + /// Borrow the column's `CategoryManagerT`, or `None` if the column /// is not categorical or no chunks have been pushed yet. /// @@ -1491,6 +1671,7 @@ mod tests { assert_eq!(ca.len(), 5); } + #[cfg(not(feature = "allow_mixed_array_batches"))] #[test] #[should_panic(expected = "Chunk ArrowType mismatch")] fn test_type_mismatch() { @@ -1503,6 +1684,158 @@ mod tests { ca.push(wrong); } + /// Without `allow_mixed_array_batches`, `from_arrays` rejects a chunk list + /// whose types differ. + #[cfg(not(feature = "allow_mixed_array_batches"))] + #[test] + #[should_panic(expected = "Chunk 1 ArrowType mismatch")] + fn test_from_arrays_mixed_types_panic() { + use crate::{arr_f64, arr_u64}; + + let _ = SuperArray::from_arrays(vec![arr_u64![1u64, 2, 3], arr_f64![4.0, 5.0]]); + } + + /// With `allow_mixed_array_batches`, `from_arrays` accepts chunks of + /// separate types and the logical length is still the sum of the chunk + /// lengths. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + fn test_from_arrays_mixed_types() { + use crate::{arr_f64, arr_str32, arr_u64}; + + let sa = SuperArray::from_arrays(vec![ + arr_u64![1u64, 2, 3], + arr_str32!["a", "b"], + arr_f64![4.0, 5.0, 6.0, 7.0], + ]); + + assert_eq!(sa.n_chunks(), 3); + assert_eq!(sa.len(), 9); + assert_eq!(sa.chunks[0].arrow_type(), ArrowType::UInt64); + assert_eq!(sa.chunks[1].arrow_type(), ArrowType::String); + assert_eq!(sa.chunks[2].arrow_type(), ArrowType::Float64); + } + + /// With `allow_mixed_array_batches`, `push` accepts a chunk whose type + /// differs from the chunks already held. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + fn test_push_mixed_types() { + use crate::arr_f64; + + let mut sa = SuperArray::new(); + sa.push(int_array(&[1, 2, 3])); + sa.push(arr_f64![4.0, 5.0]); + + assert_eq!(sa.n_chunks(), 2); + assert_eq!(sa.len(), 5); + } + + /// A `Field`-carrying constructor rejects mixed chunks with the feature + /// on: a present field always describes every chunk. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + #[should_panic(expected = "ArrowType mismatch")] + fn test_from_arrays_with_field_mixed_types_panics() { + use crate::{arr_f64, arr_u64}; + + let field = Field::new("x", ArrowType::UInt64, false, None); + let _ = SuperArray::from_arrays_with_field( + vec![arr_u64![1u64, 2, 3], arr_f64![4.0, 5.0]], + field, + ); + } + + /// A push onto a `Field`-carrying SuperArray rejects a mismatched chunk + /// with the feature on. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + #[should_panic(expected = "Chunk ArrowType mismatch with field")] + fn test_push_against_field_mixed_types_panics() { + use crate::{arr_f64, arr_u64}; + + let field = Field::new("x", ArrowType::UInt64, false, None); + let mut sa = SuperArray::from_arrays_with_field(vec![arr_u64![1u64, 2, 3]], field); + sa.push(arr_f64![4.0, 5.0]); + } + + /// Attaching a field to a mixed field-free SuperArray is rejected with + /// the feature on: the first `push_field_array` validates every existing + /// chunk against the incoming field's type. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + #[should_panic(expected = "a field cannot attach")] + fn test_push_field_array_onto_mixed_panics() { + use crate::{arr_f64, arr_u64, fa_u64}; + + let mut sa = SuperArray::from_arrays(vec![arr_u64![1u64, 2, 3], arr_f64![4.0, 5.0]]); + sa.push_field_array(fa_u64!("x", 6u64, 7)); + } + + /// The `try_*` constructor forms return `Err` on the conditions their + /// panicking counterparts reject, and `Ok` on valid input. + #[test] + fn test_try_constructor_forms() { + use crate::{arr_f64, arr_u64, fa_u64}; + + let field = Field::new("x", ArrowType::UInt64, false, None); + assert!( + SuperArray::try_from_arrays_with_field(vec![arr_u64![1u64, 2]], field.clone()).is_ok() + ); + assert!( + SuperArray::try_from_arrays_with_field(vec![arr_f64![1.0, 2.0]], field.clone()) + .is_err() + ); + + assert!(SuperArray::try_from_field_array_chunks(vec![]).is_err()); + assert!( + SuperArray::try_from_field_array_chunks(vec![fa_u64!("x", 1u64, 2)]).is_ok() + ); + + let mut sa = SuperArray::try_from_arrays_with_field(vec![arr_u64![1u64, 2]], field).unwrap(); + assert!(sa.try_push_field_array(fa_u64!("x", 3u64, 4)).is_ok()); + assert!(sa + .try_push_field_array(crate::fa_f64!("x", 5.0, 6.0)) + .is_err()); + assert_eq!(sa.n_chunks(), 2); + } + + /// `try_from_slices` returns `Err` on an empty slice list or a slice + /// whose type does not match the field, and `Ok` on valid input. + #[cfg(feature = "views")] + #[test] + fn test_try_from_slices() { + use crate::{arr_f64, arr_u64, ArrayV}; + + let field = Arc::new(Field::new("x", ArrowType::UInt64, false, None)); + assert!(SuperArray::try_from_slices(&[], field.clone()).is_err()); + + let ok = ArrayV::new(arr_u64![1u64, 2, 3], 0, 3); + assert!(SuperArray::try_from_slices(&[ok], field.clone()).is_ok()); + + let wrong = ArrayV::new(arr_f64![1.0, 2.0], 0, 2); + assert!(SuperArray::try_from_slices(&[wrong], field).is_err()); + } + + /// `check_type_uniformity` returns `true` for uniform chunks and + /// empty SuperArrays, and `false` when chunk types differ. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + fn test_check_type_uniformity() { + use crate::arr_f64; + + assert!(SuperArray::new().check_type_uniformity()); + + let uniform = SuperArray::from_arrays(vec![int_array(&[1, 2]), int_array(&[3])]); + assert!(uniform.check_type_uniformity()); + + let single = SuperArray::from_arrays(vec![int_array(&[1, 2])]); + assert!(single.check_type_uniformity()); + + let mixed = SuperArray::from_arrays(vec![int_array(&[1, 2]), arr_f64![3.0, 4.0]]); + assert!(!mixed.check_type_uniformity()); + } + #[test] #[should_panic(expected = "Chunk field name mismatch")] fn test_name_mismatch() { @@ -1865,6 +2198,7 @@ mod tests { assert_eq!(sa.len(), 5); } + #[cfg(not(feature = "allow_mixed_array_batches"))] #[test] #[should_panic(expected = "Chunk ArrowType mismatch")] fn push_mismatched_scale_panics() { diff --git a/src/structs/views/chunked/super_array_view.rs b/src/structs/views/chunked/super_array_view.rs index fa51174..2f75125 100644 --- a/src/structs/views/chunked/super_array_view.rs +++ b/src/structs/views/chunked/super_array_view.rs @@ -99,6 +99,19 @@ impl SuperArrayV { self.slices.iter() } + /// Reports whether every slice's `ArrowType` matches this view's field. + /// + /// The fields on this struct are public, so a view can be assembled by + /// hand. This method verifies that the `field` describes every slice, for + /// callers that require uniformity (e.g. consolidation, FFI export) + /// before proceeding. Returns `true` for an empty view. + #[cfg(feature = "allow_mixed_array_batches")] + pub fn check_type_uniformity(&self) -> bool { + self.slices + .iter() + .all(|slice| slice.array.arrow_type() == self.field.dtype) + } + /// Returns a sub-window of this chunked array view over `[offset .. offset+len)`. /// /// Produces a new `ChunkedArrayView` with updated slice metadata. @@ -334,6 +347,52 @@ mod tests { FieldArray::new(field, arr) } + /// A mixed field-free SuperArray cannot become a view: the conversion + /// synthesises a field from the first chunk and that field must describe + /// every chunk. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + #[should_panic(expected = "cannot become a view")] + fn test_mixed_super_array_to_view_panics() { + use crate::{arr_f64, arr_u64, SuperArray}; + + let sa = SuperArray::from_arrays(vec![arr_u64![1u64, 2, 3], arr_f64![4.0, 5.0]]); + let _ = SuperArrayV::from(sa); + } + + /// `check_type_uniformity` reports whether every slice matches the + /// view's field, and returns `true` for an empty view. + #[cfg(feature = "allow_mixed_array_batches")] + #[test] + fn test_view_check_type_uniformity() { + use crate::{arr_f64, arr_u64}; + + let field = Arc::new(Field::new("x", ArrowType::UInt64, false, None)); + let empty = SuperArrayV { + slices: Vec::new(), + len: 0, + field: field.clone(), + }; + assert!(empty.check_type_uniformity()); + + let uniform = SuperArrayV { + slices: vec![ArrayV::new(arr_u64![1u64, 2], 0, 2)], + len: 2, + field: field.clone(), + }; + assert!(uniform.check_type_uniformity()); + + let mixed = SuperArrayV { + slices: vec![ + ArrayV::new(arr_u64![1u64, 2], 0, 2), + ArrayV::new(arr_f64![3.0, 4.0], 0, 2), + ], + len: 4, + field, + }; + assert!(!mixed.check_type_uniformity()); + } + #[test] fn test_is_empty_and_n_pieces() { let f = Arc::new(Field::new("col", ArrowType::Int32, false, None)); @@ -1153,10 +1212,23 @@ impl From for SuperArrayV { fn from(super_array: SuperArray) -> Self { let len = super_array.len(); - // Get field from SuperArray or synthesise from first chunk + // Get field from SuperArray or synthesise from first chunk. A + // SuperArrayV field describes every chunk, so a mixed field-free + // SuperArray cannot become a view. let field = if let Some(f) = super_array.field.clone() { f } else if let Some(chunk) = super_array.chunks.first() { + #[cfg(feature = "allow_mixed_array_batches")] + for (i, sibling) in super_array.chunks.iter().enumerate().skip(1) { + assert_eq!( + sibling.arrow_type(), + chunk.arrow_type(), + "Chunk {i} ArrowType mismatch: a SuperArrayV field describes \ + every chunk, so a mixed SuperArray cannot become a view. \ + Check `SuperArray::check_type_uniformity()` and coerce to \ + one type first." + ); + } Arc::new(Field::new( "data", chunk.arrow_type(),