fix: preserve cast target metadata

datafusion/expr/src/expr.rs

@@ -34,7 +34,6 @@ use crate::{ExprSchemable, Operator, Signature, WindowFrame, WindowUDF};
 
 use arrow::datatypes::{DataType, Field, FieldRef};
 use datafusion_common::cse::{HashNode, NormalizeEq, Normalizeable};
-use datafusion_common::datatype::DataTypeExt;
 use datafusion_common::metadata::format_type_and_metadata;
 use datafusion_common::tree_node::{
     Transformed, TransformedResult, TreeNode, TreeNodeContainer, TreeNodeRecursion,
@@ -984,26 +983,76 @@ pub enum GetFieldAccess {
     },
 }
 
+/// Target of a cast expression.
+///
+/// A type-only target inherits metadata from the source expression. An explicit
+/// field supplies its own metadata, including an empty map that clears source
+/// metadata.
+#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
+pub enum CastTarget {
+    /// Cast to a data type while inheriting source metadata.
+    DataType(DataType),
+    /// Cast to an explicit field, including its metadata policy.
+    Field(FieldRef),
+}
+
+impl CastTarget {
+    /// Create a target that inherits source metadata.
+    pub fn type_only(data_type: DataType) -> Self {
+        Self::DataType(data_type)
+    }
+
+    /// Create an explicit field target.
+    pub fn explicit(field: FieldRef) -> Self {
+        Self::Field(field)
+    }
+
+    /// Return the target data type.
+    pub fn data_type(&self) -> &DataType {
+        match self {
+            Self::DataType(data_type) => data_type,
+            Self::Field(field) => field.data_type(),
+        }
+    }
+
+    /// Return the explicit target field, or `None` for a type-only target.
+    pub fn explicit_field(&self) -> Option<&FieldRef> {
+        match self {
+            Self::DataType(_) => None,
+            Self::Field(field) => Some(field),
+        }
+    }
+
+    /// Return explicit target metadata, or `None` when metadata is inherited.
+    pub fn metadata(&self) -> Option<&std::collections::HashMap<String, String>> {
+        self.explicit_field().map(|field| field.metadata())
+    }
+}
+
 /// Cast expression
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub struct Cast {
     /// The expression being cast
     pub expr: Box<Expr>,
-    /// The `DataType` the expression will yield
-    pub field: FieldRef,
+    /// The target type and metadata policy.
+    pub field: CastTarget,
 }
 
 impl Cast {
-    /// Create a new Cast expression
+    /// Create a new Cast expression with a type-only target.
     pub fn new(expr: Box<Expr>, data_type: DataType) -> Self {
         Self {
             expr,
-            field: data_type.into_nullable_field_ref(),
+            field: CastTarget::type_only(data_type),
         }
     }
 
+    /// Create a new Cast expression with explicit target metadata.
     pub fn new_from_field(expr: Box<Expr>, field: FieldRef) -> Self {
-        Self { expr, field }
+        Self {
+            expr,
+            field: CastTarget::explicit(field),
+        }
     }
 }
 
@@ -1012,21 +1061,25 @@ impl Cast {
 pub struct TryCast {
     /// The expression being cast
     pub expr: Box<Expr>,
-    /// The `DataType` the expression will yield
-    pub field: FieldRef,
+    /// The target type and metadata policy.
+    pub field: CastTarget,
 }
 
 impl TryCast {
-    /// Create a new TryCast expression
+    /// Create a new TryCast expression with a type-only target.
     pub fn new(expr: Box<Expr>, data_type: DataType) -> Self {
         Self {
             expr,
-            field: data_type.into_nullable_field_ref(),
+            field: CastTarget::type_only(data_type),
         }
     }
 
+    /// Create a new TryCast expression with explicit target metadata.
     pub fn new_from_field(expr: Box<Expr>, field: FieldRef) -> Self {
-        Self { expr, field }
+        Self {
+            expr,
+            field: CastTarget::explicit(field),
+        }
     }
 }
 
@@ -3589,12 +3642,12 @@ impl Display for Expr {
             }
             Expr::Cast(Cast { expr, field }) => {
                 let formatted =
-                    format_type_and_metadata(field.data_type(), Some(field.metadata()));
+                    format_type_and_metadata(field.data_type(), field.metadata());
                 write!(f, "CAST({expr} AS {formatted})")
             }
             Expr::TryCast(TryCast { expr, field }) => {
                 let formatted =
-                    format_type_and_metadata(field.data_type(), Some(field.metadata()));
+                    format_type_and_metadata(field.data_type(), field.metadata());
                 write!(f, "TRY_CAST({expr} AS {formatted})")
             }
             Expr::Not(expr) => write!(f, "NOT {expr}"),
@@ -4199,10 +4252,10 @@ mod test {
 
     #[test]
     fn format_cast() -> Result<()> {
-        let expr = Expr::Cast(Cast {
-            expr: Box::new(Expr::Literal(ScalarValue::Float32(Some(1.23)), None)),
-            field: DataType::Utf8.into_nullable_field_ref(),
-        });
+        let expr = Expr::Cast(Cast::new(
+            Box::new(Expr::Literal(ScalarValue::Float32(Some(1.23)), None)),
+            DataType::Utf8,
+        ));
         let expected_canonical = "CAST(Float32(1.23) AS Utf8)";
         assert_eq!(expected_canonical, format!("{expr}"));
         // Note that CAST intentionally has a name that is different from its `Display`

datafusion/expr/src/expr_schema.rs

@@ -18,9 +18,9 @@
 use super::{Between, Expr, Like, predicate_bounds};
 use crate::ValueOrLambda;
 use crate::expr::{
-    AggregateFunction, AggregateFunctionParams, Alias, BinaryExpr, Cast, InList,
-    InSubquery, Lambda, Placeholder, ScalarFunction, TryCast, Unnest, WindowFunction,
-    WindowFunctionParams,
+    AggregateFunction, AggregateFunctionParams, Alias, BinaryExpr, Cast, CastTarget,
+    InList, InSubquery, Lambda, Placeholder, ScalarFunction, TryCast, Unnest,
+    WindowFunction, WindowFunctionParams,
 };
 use crate::expr::{FieldMetadata, LambdaVariable};
 use crate::higher_order_function::HigherOrderReturnFieldArgs;
@@ -71,18 +71,23 @@ pub trait ExprSchemable {
     -> Result<(DataType, bool)>;
 }
 
-/// Derives the output field for a cast expression from the source field.
+/// Derives the output field for a cast expression from the source field, using
+/// explicit target metadata when supplied.
 /// For `TryCast`, `force_nullable` is `true` since a failed cast returns NULL.
 fn cast_output_field(
     source_field: &FieldRef,
-    target_type: &DataType,
+    target: &CastTarget,
     force_nullable: bool,
 ) -> Arc<Field> {
+    let metadata = target
+        .metadata()
+        .cloned()
+        .unwrap_or_else(|| source_field.metadata().clone());
     let mut f = source_field
         .as_ref()
         .clone()
-        .with_data_type(target_type.clone())
-        .with_metadata(source_field.metadata().clone());
+        .with_data_type(target.data_type().clone())
+        .with_metadata(metadata);
     if force_nullable {
         f = f.with_nullable(true);
     }
@@ -623,20 +628,16 @@ impl ExprSchemable for Expr {
                 func.return_field_from_args(args)
             }
             // _ => Ok((self.get_type(schema)?, self.nullable(schema)?)),
-            Expr::Cast(Cast { expr, field }) => {
-                expr.to_field(schema).map(|(_table_ref, src)| {
-                    cast_output_field(&src, field.data_type(), false)
-                })
-            }
+            Expr::Cast(Cast { expr, field }) => expr
+                .to_field(schema)
+                .map(|(_table_ref, src)| cast_output_field(&src, field, false)),
             Expr::Placeholder(Placeholder {
                 id: _,
                 field: Some(field),
             }) => Ok(Arc::clone(field).renamed(&schema_name)),
-            Expr::TryCast(TryCast { expr, field }) => {
-                expr.to_field(schema).map(|(_table_ref, src)| {
-                    cast_output_field(&src, field.data_type(), true)
-                })
-            }
+            Expr::TryCast(TryCast { expr, field }) => expr
+                .to_field(schema)
+                .map(|(_table_ref, src)| cast_output_field(&src, field, true)),
             Expr::LambdaVariable(LambdaVariable {
                 field: Some(field), ..
             }) => Ok(Arc::clone(field).renamed(&schema_name)),
@@ -1149,7 +1150,7 @@ mod tests {
     }
 
     #[test]
-    fn test_expr_metadata() {
+    fn test_expr_metadata() -> Result<()> {
         let mut meta = HashMap::new();
         meta.insert("bar".to_string(), "buzz".to_string());
         let meta = FieldMetadata::from(meta);
@@ -1179,13 +1180,53 @@ mod tests {
         // verify to_field method populates metadata
         assert_eq!(meta, expr.metadata(&schema).unwrap());
 
+        // An explicit cast target replaces source metadata. A type-only cast
+        // continues to preserve it.
+        let target_metadata = HashMap::from([(
+            "ARROW:extension:name".to_string(),
+            "arrow.uuid".to_string(),
+        )]);
+        let target_field = Arc::new(
+            Field::new("", DataType::FixedSizeBinary(16), true)
+                .with_metadata(target_metadata.clone()),
+        );
+        let cast = Expr::Cast(Cast::new_from_field(
+            Box::new(expr.clone()),
+            Arc::clone(&target_field),
+        ));
+        assert_eq!(cast.to_field(&schema)?.1.metadata(), &target_metadata);
+
+        let try_cast = Expr::TryCast(TryCast::new_from_field(
+            Box::new(expr.clone()),
+            target_field,
+        ));
+        let try_cast_field = try_cast.to_field(&schema)?.1;
+        assert_eq!(try_cast_field.metadata(), &target_metadata);
+        assert!(try_cast_field.is_nullable());
+
+        // An explicitly empty target clears source metadata even when its field
+        // has the same shape as a synthesized type-only target.
+        let explicit_empty_target =
+            Arc::new(Field::new("", DataType::FixedSizeBinary(16), true));
+        let cast = Expr::Cast(Cast::new_from_field(
+            Box::new(expr.clone()),
+            Arc::clone(&explicit_empty_target),
+        ));
+        assert!(cast.to_field(&schema)?.1.metadata().is_empty());
+        let try_cast = Expr::TryCast(TryCast::new_from_field(
+            Box::new(expr.clone()),
+            explicit_empty_target,
+        ));
+        assert!(try_cast.to_field(&schema)?.1.metadata().is_empty());
+
         // outer ref constructed by `out_ref_col_with_metadata` should be metadata-preserving
         let outer_ref = out_ref_col_with_metadata(
             DataType::Int32,
             meta.to_hashmap(),
             Column::from_name("foo"),
         );
         assert_eq!(meta, outer_ref.metadata(&schema).unwrap());
+        Ok(())
     }
 
     #[test]

datafusion/expr/src/lib.rs

@@ -112,7 +112,7 @@ pub use datafusion_expr_common::signature::{
 };
 pub use datafusion_expr_common::type_coercion::binary;
 pub use expr::{
-    Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GroupingSet, Like,
+    Between, BinaryExpr, Case, Cast, CastTarget, Expr, GetFieldAccess, GroupingSet, Like,
     Sort as SortExpr, TryCast, WindowFunctionDefinition,
 };
 pub use expr_fn::*;

datafusion/expr/src/tree_node.rs

@@ -249,10 +249,10 @@ impl TreeNode for Expr {
                 }),
             Expr::Cast(Cast { expr, field }) => expr
                 .map_elements(f)?
-                .update_data(|be| Expr::Cast(Cast::new_from_field(be, field))),
+                .update_data(|expr| Expr::Cast(Cast { expr, field })),
             Expr::TryCast(TryCast { expr, field }) => expr
                 .map_elements(f)?
-                .update_data(|be| Expr::TryCast(TryCast::new_from_field(be, field))),
+                .update_data(|expr| Expr::TryCast(TryCast { expr, field })),
             Expr::ScalarFunction(ScalarFunction { func, args }) => {
                 args.map_elements(f)?.map_data(|new_args| {
                     Ok(Expr::ScalarFunction(ScalarFunction::new_udf(

datafusion/functions/src/core/arrow_cast.rs

@@ -20,9 +20,8 @@
 use arrow::datatypes::{DataType, Field, FieldRef};
 use arrow::error::ArrowError;
 use datafusion_common::{
-    Result, ScalarValue, arrow_datafusion_err, datatype::DataTypeExt,
-    exec_datafusion_err, exec_err, internal_err, types::logical_string,
-    utils::take_function_args,
+    Result, ScalarValue, arrow_datafusion_err, exec_datafusion_err, exec_err,
+    internal_err, types::logical_string, utils::take_function_args,
 };
 
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
@@ -166,10 +165,10 @@ impl ScalarUDFImpl for ArrowCastFunc {
             source_arg
         } else {
             // Use an actual cast to get the correct type
-            Expr::Cast(datafusion_expr::Cast {
-                expr: Box::new(source_arg),
-                field: target_type.into_nullable_field_ref(),
-            })
+            Expr::Cast(datafusion_expr::Cast::new(
+                Box::new(source_arg),
+                target_type,
+            ))
         };
         // return the newly written argument to DataFusion
         Ok(ExprSimplifyResult::Simplified(new_expr))

datafusion/functions/src/core/arrow_try_cast.rs

@@ -20,8 +20,8 @@
 use arrow::datatypes::{DataType, Field, FieldRef};
 use arrow::error::ArrowError;
 use datafusion_common::{
-    Result, arrow_datafusion_err, datatype::DataTypeExt, exec_datafusion_err, exec_err,
-    internal_err, types::logical_string, utils::take_function_args,
+    Result, arrow_datafusion_err, exec_datafusion_err, exec_err, internal_err,
+    types::logical_string, utils::take_function_args,
 };
 
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
@@ -137,10 +137,10 @@ impl ScalarUDFImpl for ArrowTryCastFunc {
         let new_expr = if source_type == target_type {
             source_arg
         } else {
-            Expr::TryCast(datafusion_expr::TryCast {
-                expr: Box::new(source_arg),
-                field: target_type.into_nullable_field_ref(),
-            })
+            Expr::TryCast(datafusion_expr::TryCast::new(
+                Box::new(source_arg),
+                target_type,
+            ))
         };
         Ok(ExprSimplifyResult::Simplified(new_expr))
     }

datafusion/functions/src/core/cast_to_type.rs

@@ -130,12 +130,11 @@ impl ScalarUDFImpl for CastToTypeFunc {
             // the argument's data type is already the correct type
             source_arg
         } else {
-            let nullable = info.nullable(&source_arg)? || target_type == DataType::Null;
             // Use an actual cast to get the correct type
-            Expr::Cast(datafusion_expr::Cast {
-                expr: Box::new(source_arg),
-                field: Field::new("", target_type, nullable).into(),
-            })
+            Expr::Cast(datafusion_expr::Cast::new(
+                Box::new(source_arg),
+                target_type,
+            ))
         };
         Ok(ExprSimplifyResult::Simplified(new_expr))
     }

datafusion/functions/src/core/try_cast_to_type.rs

@@ -18,9 +18,7 @@
 //! [`TryCastToTypeFunc`]: Implementation of the `try_cast_to_type` function
 
 use arrow::datatypes::{DataType, Field, FieldRef};
-use datafusion_common::{
-    Result, datatype::DataTypeExt, internal_err, utils::take_function_args,
-};
+use datafusion_common::{Result, internal_err, utils::take_function_args};
 use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
 use datafusion_expr::{
     Coercion, ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
@@ -116,10 +114,10 @@ impl ScalarUDFImpl for TryCastToTypeFunc {
         let new_expr = if source_type == target_type {
             source_arg
         } else {
-            Expr::TryCast(datafusion_expr::TryCast {
-                expr: Box::new(source_arg),
-                field: target_type.into_nullable_field_ref(),
-            })
+            Expr::TryCast(datafusion_expr::TryCast::new(
+                Box::new(source_arg),
+                target_type,
+            ))
         };
         Ok(ExprSimplifyResult::Simplified(new_expr))
     }