com.google.guava
diff --git a/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java b/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java
index 62a3687af..105ce8b14 100644
--- a/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java
+++ b/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java
@@ -18,32 +18,16 @@
*/
package org.apache.parquet.avro;
-import org.apache.avro.LogicalType;
-import org.apache.avro.LogicalTypes;
-import org.apache.avro.Schema;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.parquet.schema.ConversionPatterns;
-import org.apache.parquet.schema.GroupType;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
-import org.apache.parquet.schema.Type;
-import org.apache.parquet.schema.Types;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
import static java.util.Optional.empty;
import static java.util.Optional.of;
-import static org.apache.avro.JsonProperties.NULL_VALUE;
+import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED;
+import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED_DEFAULT;
+import static org.apache.parquet.avro.AvroRecordConverter.getRuntimeAvroVersion;
+import static org.apache.parquet.avro.AvroWriteSupport.WRITE_FIXED_AS_INT96;
import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE;
import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT;
+import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID;
+import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID_DEFAULT;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS;
import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType;
@@ -52,29 +36,64 @@
import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType;
import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType;
-import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.*;
+import static org.apache.parquet.schema.LogicalTypeAnnotation.uuidType;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64;
import static org.apache.parquet.schema.Type.Repetition.REPEATED;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.conf.HadoopParquetConfiguration;
+import org.apache.parquet.conf.ParquetConfiguration;
+import org.apache.parquet.schema.ConversionPatterns;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
/**
*
* Converts an Avro schema into a Parquet schema, or vice versa. See package
* documentation for details of the mapping.
*
*
- * A copy of the AvroSchemaConverter, except with handling for INT96.
+ * A copy of parquet-avro 1.15.2's AvroSchemaConverter, except with handling for INT96: INT96 columns
+ * are converted to Avro bytes instead of failing when READ_INT96_AS_FIXED is not set.
*/
public class AvroSchemaConverter {
- public static final String ADD_LIST_ELEMENT_RECORDS =
- "parquet.avro.add-list-element-records";
+ public static final String ADD_LIST_ELEMENT_RECORDS = "parquet.avro.add-list-element-records";
private static final boolean ADD_LIST_ELEMENT_RECORDS_DEFAULT = true;
private final boolean assumeRepeatedIsListElement;
private final boolean writeOldListStructure;
+ private final boolean writeParquetUUID;
+ private final boolean readInt96AsFixed;
+ private final Set pathsToInt96;
public AvroSchemaConverter() {
- this.assumeRepeatedIsListElement = ADD_LIST_ELEMENT_RECORDS_DEFAULT;
- this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT;
+ this(ADD_LIST_ELEMENT_RECORDS_DEFAULT);
}
/**
@@ -86,13 +105,21 @@ public AvroSchemaConverter() {
AvroSchemaConverter(boolean assumeRepeatedIsListElement) {
this.assumeRepeatedIsListElement = assumeRepeatedIsListElement;
this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT;
+ this.writeParquetUUID = WRITE_PARQUET_UUID_DEFAULT;
+ this.readInt96AsFixed = READ_INT96_AS_FIXED_DEFAULT;
+ this.pathsToInt96 = Collections.emptySet();
}
public AvroSchemaConverter(Configuration conf) {
- this.assumeRepeatedIsListElement = conf.getBoolean(
- ADD_LIST_ELEMENT_RECORDS, ADD_LIST_ELEMENT_RECORDS_DEFAULT);
- this.writeOldListStructure = conf.getBoolean(
- WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT);
+ this(new HadoopParquetConfiguration(conf));
+ }
+
+ public AvroSchemaConverter(ParquetConfiguration conf) {
+ this.assumeRepeatedIsListElement = conf.getBoolean(ADD_LIST_ELEMENT_RECORDS, ADD_LIST_ELEMENT_RECORDS_DEFAULT);
+ this.writeOldListStructure = conf.getBoolean(WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT);
+ this.writeParquetUUID = conf.getBoolean(WRITE_PARQUET_UUID, WRITE_PARQUET_UUID_DEFAULT);
+ this.readInt96AsFixed = conf.getBoolean(READ_INT96_AS_FIXED, READ_INT96_AS_FIXED_DEFAULT);
+ this.pathsToInt96 = new HashSet<>(Arrays.asList(conf.getStrings(WRITE_FIXED_AS_INT96, new String[0])));
}
/**
@@ -125,28 +152,29 @@ public MessageType convert(Schema avroSchema) {
if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
throw new IllegalArgumentException("Avro schema must be a record.");
}
- return new MessageType(avroSchema.getFullName(), convertFields(avroSchema.getFields()));
+ return new MessageType(avroSchema.getFullName(), convertFields(avroSchema.getFields(), ""));
}
- private List convertFields(List fields) {
+ private List convertFields(List fields, String schemaPath) {
List types = new ArrayList();
for (Schema.Field field : fields) {
if (field.schema().getType().equals(Schema.Type.NULL)) {
continue; // Avro nulls are not encoded, unless they are null unions
}
- types.add(convertField(field));
+ types.add(convertField(field, appendPath(schemaPath, field.name())));
}
return types;
}
- private Type convertField(String fieldName, Schema schema) {
- return convertField(fieldName, schema, Type.Repetition.REQUIRED);
+ private Type convertField(String fieldName, Schema schema, String schemaPath) {
+ return convertField(fieldName, schema, Type.Repetition.REQUIRED, schemaPath);
}
@SuppressWarnings("deprecation")
- private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) {
+ private Type convertField(String fieldName, Schema schema, Type.Repetition repetition, String schemaPath) {
Types.PrimitiveBuilder builder;
Schema.Type type = schema.getType();
+ LogicalType logicalType = schema.getLogicalType();
if (type.equals(Schema.Type.BOOLEAN)) {
builder = Types.primitive(BOOLEAN, repetition);
} else if (type.equals(Schema.Type.INT)) {
@@ -160,35 +188,50 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet
} else if (type.equals(Schema.Type.BYTES)) {
builder = Types.primitive(BINARY, repetition);
} else if (type.equals(Schema.Type.STRING)) {
- builder = Types.primitive(BINARY, repetition).as(stringType());
+ if (logicalType != null
+ && logicalType.getName().equals(LogicalTypes.uuid().getName())
+ && writeParquetUUID) {
+ builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
+ .length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES);
+ } else {
+ builder = Types.primitive(BINARY, repetition).as(stringType());
+ }
} else if (type.equals(Schema.Type.RECORD)) {
- return new GroupType(repetition, fieldName, convertFields(schema.getFields()));
+ return new GroupType(repetition, fieldName, convertFields(schema.getFields(), schemaPath));
} else if (type.equals(Schema.Type.ENUM)) {
builder = Types.primitive(BINARY, repetition).as(enumType());
} else if (type.equals(Schema.Type.ARRAY)) {
if (writeOldListStructure) {
- return ConversionPatterns.listType(repetition, fieldName,
- convertField("array", schema.getElementType(), REPEATED));
+ return ConversionPatterns.listType(
+ repetition, fieldName, convertField("array", schema.getElementType(), REPEATED, schemaPath));
} else {
- return ConversionPatterns.listOfElements(repetition, fieldName,
- convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType()));
+ return ConversionPatterns.listOfElements(
+ repetition,
+ fieldName,
+ convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType(), schemaPath));
}
} else if (type.equals(Schema.Type.MAP)) {
- Type valType = convertField("value", schema.getValueType());
+ Type valType = convertField("value", schema.getValueType(), schemaPath);
// avro map key type is always string
return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType);
} else if (type.equals(Schema.Type.FIXED)) {
- builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition)
- .length(schema.getFixedSize());
+ if (pathsToInt96.contains(schemaPath)) {
+ if (schema.getFixedSize() != 12) {
+ throw new IllegalArgumentException("The size of the fixed type field " + schemaPath
+ + " must be 12 bytes for INT96 conversion");
+ }
+ builder = Types.primitive(PrimitiveTypeName.INT96, repetition);
+ } else {
+ builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition).length(schema.getFixedSize());
+ }
} else if (type.equals(Schema.Type.UNION)) {
- return convertUnion(fieldName, schema, repetition);
+ return convertUnion(fieldName, schema, repetition, schemaPath);
} else {
throw new UnsupportedOperationException("Cannot convert Avro type " + type);
}
// schema translation can only be done for known logical types because this
// creates an equivalence
- LogicalType logicalType = schema.getLogicalType();
if (logicalType != null) {
if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType;
@@ -204,7 +247,7 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet
return builder.named(fieldName);
}
- private Type convertUnion(String fieldName, Schema schema, Type.Repetition repetition) {
+ private Type convertUnion(String fieldName, Schema schema, Type.Repetition repetition, String schemaPath) {
List nonNullSchemas = new ArrayList(schema.getTypes().size());
// Found any schemas in the union? Required for the edge case, where the union contains only a single type.
boolean foundNullSchema = false;
@@ -225,25 +268,27 @@ private Type convertUnion(String fieldName, Schema schema, Type.Repetition repet
throw new UnsupportedOperationException("Cannot convert Avro union of only nulls");
case 1:
- return foundNullSchema ? convertField(fieldName, nonNullSchemas.get(0), repetition) :
- convertUnionToGroupType(fieldName, repetition, nonNullSchemas);
+ return foundNullSchema
+ ? convertField(fieldName, nonNullSchemas.get(0), repetition, schemaPath)
+ : convertUnionToGroupType(fieldName, repetition, nonNullSchemas, schemaPath);
default: // complex union type
- return convertUnionToGroupType(fieldName, repetition, nonNullSchemas);
+ return convertUnionToGroupType(fieldName, repetition, nonNullSchemas, schemaPath);
}
}
- private Type convertUnionToGroupType(String fieldName, Type.Repetition repetition, List nonNullSchemas) {
+ private Type convertUnionToGroupType(
+ String fieldName, Type.Repetition repetition, List nonNullSchemas, String schemaPath) {
List unionTypes = new ArrayList(nonNullSchemas.size());
int index = 0;
for (Schema childSchema : nonNullSchemas) {
- unionTypes.add( convertField("member" + index++, childSchema, Type.Repetition.OPTIONAL));
+ unionTypes.add(convertField("member" + index++, childSchema, Type.Repetition.OPTIONAL, schemaPath));
}
return new GroupType(repetition, fieldName, unionTypes);
}
- private Type convertField(Schema.Field field) {
- return convertField(field.name(), field.schema());
+ private Type convertField(Schema.Field field, String schemaPath) {
+ return convertField(field.name(), field.schema(), schemaPath);
}
public Schema convert(MessageType parquetSchema) {
@@ -255,150 +300,177 @@ Schema convert(GroupType parquetSchema) {
}
private Schema convertFields(String name, List parquetFields, Map names) {
- List fields = new ArrayList();
- Integer nameCount = names.merge(name, 1, (oldValue, value) -> oldValue + 1);
+ SchemaBuilder.FieldAssembler builder =
+ SchemaBuilder.builder(namespace(name, names)).record(name).fields();
for (Type parquetType : parquetFields) {
Schema fieldSchema = convertField(parquetType, names);
- if (parquetType.isRepetition(REPEATED)) {
- throw new UnsupportedOperationException("REPEATED not supported outside LIST or MAP. Type: " + parquetType);
+ if (parquetType.isRepetition(REPEATED)) { // If a repeated field is ungrouped, treat as REQUIRED per spec
+ builder.name(parquetType.getName())
+ .type()
+ .array()
+ .items()
+ .type(fieldSchema)
+ .arrayDefault(new ArrayList<>());
} else if (parquetType.isRepetition(Type.Repetition.OPTIONAL)) {
- fields.add(new Schema.Field(
- parquetType.getName(), optional(fieldSchema), null, NULL_VALUE));
+ builder.name(parquetType.getName()).type().optional().type(fieldSchema);
} else { // REQUIRED
- fields.add(new Schema.Field(
- parquetType.getName(), fieldSchema, null, (Object) null));
+ builder.name(parquetType.getName()).type(fieldSchema).noDefault();
}
}
- Schema schema = Schema.createRecord(name, null, nameCount > 1 ? name + nameCount : null, false);
- schema.setFields(fields);
- return schema;
+ return builder.endRecord();
}
private Schema convertField(final Type parquetType, Map names) {
if (parquetType.isPrimitive()) {
final PrimitiveType asPrimitive = parquetType.asPrimitiveType();
- final PrimitiveTypeName parquetPrimitiveTypeName =
- asPrimitive.getPrimitiveTypeName();
+ final PrimitiveTypeName parquetPrimitiveTypeName = asPrimitive.getPrimitiveTypeName();
final LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation();
Schema schema = parquetPrimitiveTypeName.convert(
- new PrimitiveType.PrimitiveTypeNameConverter() {
- @Override
- public Schema convertBOOLEAN(PrimitiveTypeName primitiveTypeName) {
- return Schema.create(Schema.Type.BOOLEAN);
- }
- @Override
- public Schema convertINT32(PrimitiveTypeName primitiveTypeName) {
- return Schema.create(Schema.Type.INT);
- }
- @Override
- public Schema convertINT64(PrimitiveTypeName primitiveTypeName) {
- return Schema.create(Schema.Type.LONG);
- }
- @Override
- public Schema convertINT96(PrimitiveTypeName primitiveTypeName) {
- // this is the modified part
- return Schema.create(Schema.Type.BYTES);
- }
- @Override
- public Schema convertFLOAT(PrimitiveTypeName primitiveTypeName) {
- return Schema.create(Schema.Type.FLOAT);
- }
- @Override
- public Schema convertDOUBLE(PrimitiveTypeName primitiveTypeName) {
- return Schema.create(Schema.Type.DOUBLE);
- }
- @Override
- public Schema convertFIXED_LEN_BYTE_ARRAY(PrimitiveTypeName primitiveTypeName) {
- int size = parquetType.asPrimitiveType().getTypeLength();
- return Schema.createFixed(parquetType.getName(), null, null, size);
- }
- @Override
- public Schema convertBINARY(PrimitiveTypeName primitiveTypeName) {
- if (annotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation ||
- annotation instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation) {
- return Schema.create(Schema.Type.STRING);
- } else {
+ new PrimitiveType.PrimitiveTypeNameConverter() {
+ @Override
+ public Schema convertBOOLEAN(PrimitiveTypeName primitiveTypeName) {
+ return Schema.create(Schema.Type.BOOLEAN);
+ }
+
+ @Override
+ public Schema convertINT32(PrimitiveTypeName primitiveTypeName) {
+ return Schema.create(Schema.Type.INT);
+ }
+
+ @Override
+ public Schema convertINT64(PrimitiveTypeName primitiveTypeName) {
+ return Schema.create(Schema.Type.LONG);
+ }
+
+ @Override
+ public Schema convertINT96(PrimitiveTypeName primitiveTypeName) {
+ if (readInt96AsFixed) {
+ return Schema.createFixed("INT96", "INT96 represented as byte[12]", null, 12);
+ }
+ // this is the modified part: CDAP reads INT96 as bytes (see AvroIndexedRecordConverter)
return Schema.create(Schema.Type.BYTES);
}
- }
- });
+
+ @Override
+ public Schema convertFLOAT(PrimitiveTypeName primitiveTypeName) {
+ return Schema.create(Schema.Type.FLOAT);
+ }
+
+ @Override
+ public Schema convertDOUBLE(PrimitiveTypeName primitiveTypeName) {
+ return Schema.create(Schema.Type.DOUBLE);
+ }
+
+ @Override
+ public Schema convertFIXED_LEN_BYTE_ARRAY(PrimitiveTypeName primitiveTypeName) {
+ if (annotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) {
+ return Schema.create(Schema.Type.STRING);
+ } else {
+ int size = parquetType.asPrimitiveType().getTypeLength();
+ String name = parquetType.getName();
+ String ns = namespace(name, names);
+ return Schema.createFixed(name, null, ns, size);
+ }
+ }
+
+ @Override
+ public Schema convertBINARY(PrimitiveTypeName primitiveTypeName) {
+ if (annotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation
+ || annotation instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation) {
+ return Schema.create(Schema.Type.STRING);
+ } else {
+ return Schema.create(Schema.Type.BYTES);
+ }
+ }
+ });
LogicalType logicalType = convertLogicalType(annotation);
- if (logicalType != null && (!(annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) ||
- parquetPrimitiveTypeName == BINARY ||
- parquetPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY)) {
+ if (logicalType != null
+ && (!(annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation)
+ || parquetPrimitiveTypeName == BINARY
+ || parquetPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY)) {
schema = logicalType.addToSchema(schema);
}
return schema;
-
} else {
GroupType parquetGroupType = parquetType.asGroupType();
LogicalTypeAnnotation logicalTypeAnnotation = parquetGroupType.getLogicalTypeAnnotation();
if (logicalTypeAnnotation != null) {
- return logicalTypeAnnotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() {
- @Override
- public Optional visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
- if (parquetGroupType.getFieldCount()!= 1) {
- throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
- }
- Type repeatedType = parquetGroupType.getType(0);
- if (!repeatedType.isRepetition(REPEATED)) {
- throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
- }
- if (isElementType(repeatedType, parquetGroupType.getName())) {
- // repeated element types are always required
- return of(Schema.createArray(convertField(repeatedType, names)));
- } else {
- Type elementType = repeatedType.asGroupType().getType(0);
- if (elementType.isRepetition(Type.Repetition.OPTIONAL)) {
- return of(Schema.createArray(optional(convertField(elementType, names))));
- } else {
- return of(Schema.createArray(convertField(elementType, names)));
+ return logicalTypeAnnotation
+ .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() {
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) {
+ if (parquetGroupType.getFieldCount() != 1) {
+ throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
+ }
+ Type repeatedType = parquetGroupType.getType(0);
+ if (!repeatedType.isRepetition(REPEATED)) {
+ throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
+ }
+ if (isElementType(repeatedType, parquetGroupType.getName())) {
+ // repeated element types are always required
+ return of(Schema.createArray(convertField(repeatedType, names)));
+ } else {
+ Type elementType =
+ repeatedType.asGroupType().getType(0);
+ if (elementType.isRepetition(Type.Repetition.OPTIONAL)) {
+ return of(Schema.createArray(optional(convertField(elementType, names))));
+ } else {
+ return of(Schema.createArray(convertField(elementType, names)));
+ }
+ }
}
- }
- }
- @Override
- // for backward-compatibility
- public Optional visit(LogicalTypeAnnotation.MapKeyValueTypeAnnotation mapKeyValueLogicalType) {
- return visitMapOrMapKeyValue();
- }
+ @Override
+ // for backward-compatibility
+ public Optional visit(
+ LogicalTypeAnnotation.MapKeyValueTypeAnnotation mapKeyValueLogicalType) {
+ return visitMapOrMapKeyValue();
+ }
- @Override
- public Optional visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
- return visitMapOrMapKeyValue();
- }
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
+ return visitMapOrMapKeyValue();
+ }
- private Optional visitMapOrMapKeyValue() {
- if (parquetGroupType.getFieldCount() != 1 || parquetGroupType.getType(0).isPrimitive()) {
- throw new UnsupportedOperationException("Invalid map type " + parquetGroupType);
- }
- GroupType mapKeyValType = parquetGroupType.getType(0).asGroupType();
- if (!mapKeyValType.isRepetition(REPEATED) ||
- mapKeyValType.getFieldCount()!=2) {
- throw new UnsupportedOperationException("Invalid map type " + parquetGroupType);
- }
- Type keyType = mapKeyValType.getType(0);
- if (!keyType.isPrimitive() ||
- !keyType.asPrimitiveType().getPrimitiveTypeName().equals(PrimitiveTypeName.BINARY) ||
- !keyType.getLogicalTypeAnnotation().equals(stringType())) {
- throw new IllegalArgumentException("Map key type must be binary (UTF8): "
- + keyType);
- }
- Type valueType = mapKeyValType.getType(1);
- if (valueType.isRepetition(Type.Repetition.OPTIONAL)) {
- return of(Schema.createMap(optional(convertField(valueType, names))));
- } else {
- return of(Schema.createMap(convertField(valueType, names)));
- }
- }
+ private Optional visitMapOrMapKeyValue() {
+ if (parquetGroupType.getFieldCount() != 1
+ || parquetGroupType.getType(0).isPrimitive()) {
+ throw new UnsupportedOperationException("Invalid map type " + parquetGroupType);
+ }
+ GroupType mapKeyValType =
+ parquetGroupType.getType(0).asGroupType();
+ if (!mapKeyValType.isRepetition(REPEATED) || mapKeyValType.getFieldCount() != 2) {
+ throw new UnsupportedOperationException("Invalid map type " + parquetGroupType);
+ }
+ Type keyType = mapKeyValType.getType(0);
+ if (!keyType.isPrimitive()
+ || !keyType.asPrimitiveType()
+ .getPrimitiveTypeName()
+ .equals(PrimitiveTypeName.BINARY)
+ || !keyType.getLogicalTypeAnnotation().equals(stringType())) {
+ throw new IllegalArgumentException(
+ "Map key type must be binary (UTF8): " + keyType);
+ }
+ Type valueType = mapKeyValType.getType(1);
+ if (valueType.isRepetition(Type.Repetition.OPTIONAL)) {
+ return of(Schema.createMap(optional(convertField(valueType, names))));
+ } else {
+ return of(Schema.createMap(convertField(valueType, names)));
+ }
+ }
- @Override
- public Optional visit(LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
- return of(Schema.create(Schema.Type.STRING));
- }
- }).orElseThrow(() -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType));
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) {
+ return of(Schema.create(Schema.Type.STRING));
+ }
+ })
+ .orElseThrow(
+ () -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType));
} else {
// if no original type then it's a record
return convertFields(parquetGroupType.getName(), parquetGroupType.getFields(), names);
@@ -422,7 +494,18 @@ private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) {
return timestampType(true, MILLIS);
} else if (logicalType instanceof LogicalTypes.TimestampMicros) {
return timestampType(true, MICROS);
+ } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) {
+ return uuidType();
}
+
+ if (avroVersionSupportsLocalTimestampTypes()) {
+ if (logicalType instanceof LogicalTypes.LocalTimestampMillis) {
+ return timestampType(false, MILLIS);
+ } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros) {
+ return timestampType(false, MICROS);
+ }
+ }
+
return null;
}
@@ -430,41 +513,65 @@ private LogicalType convertLogicalType(LogicalTypeAnnotation annotation) {
if (annotation == null) {
return null;
}
- return annotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() {
- @Override
- public Optional visit(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) {
- return of(LogicalTypes.decimal(decimalLogicalType.getPrecision(), decimalLogicalType.getScale()));
- }
+ return annotation
+ .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() {
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) {
+ return of(
+ LogicalTypes.decimal(decimalLogicalType.getPrecision(), decimalLogicalType.getScale()));
+ }
- @Override
- public Optional visit(LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
- return of(LogicalTypes.date());
- }
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) {
+ return of(LogicalTypes.date());
+ }
- @Override
- public Optional visit(LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
- LogicalTypeAnnotation.TimeUnit unit = timeLogicalType.getUnit();
- switch (unit) {
- case MILLIS:
- return of(LogicalTypes.timeMillis());
- case MICROS:
- return of(LogicalTypes.timeMicros());
- }
- return empty();
- }
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) {
+ LogicalTypeAnnotation.TimeUnit unit = timeLogicalType.getUnit();
+ switch (unit) {
+ case MILLIS:
+ return of(LogicalTypes.timeMillis());
+ case MICROS:
+ return of(LogicalTypes.timeMicros());
+ }
+ return empty();
+ }
- @Override
- public Optional visit(LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
- LogicalTypeAnnotation.TimeUnit unit = timestampLogicalType.getUnit();
- switch (unit) {
- case MILLIS:
- return of(LogicalTypes.timestampMillis());
- case MICROS:
- return of(LogicalTypes.timestampMicros());
- }
- return empty();
- }
- }).orElse(null);
+ @Override
+ public Optional visit(
+ LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
+ LogicalTypeAnnotation.TimeUnit unit = timestampLogicalType.getUnit();
+ boolean isAdjustedToUTC = timestampLogicalType.isAdjustedToUTC();
+
+ if (isAdjustedToUTC || !avroVersionSupportsLocalTimestampTypes()) {
+ switch (unit) {
+ case MILLIS:
+ return of(LogicalTypes.timestampMillis());
+ case MICROS:
+ return of(LogicalTypes.timestampMicros());
+ }
+ return empty();
+ } else {
+ switch (unit) {
+ case MILLIS:
+ return of(LogicalTypes.localTimestampMillis());
+ case MICROS:
+ return of(LogicalTypes.localTimestampMicros());
+ }
+ return empty();
+ }
+ }
+
+ @Override
+ public Optional visit(UUIDLogicalTypeAnnotation uuidLogicalType) {
+ return of(LogicalTypes.uuid());
+ }
+ })
+ .orElse(null);
}
/**
@@ -478,22 +585,43 @@ public Optional visit(LogicalTypeAnnotation.TimestampLogicalTypeAnn
*/
private boolean isElementType(Type repeatedType, String parentName) {
return (
- // can't be a synthetic layer because it would be invalid
- repeatedType.isPrimitive() ||
- repeatedType.asGroupType().getFieldCount() > 1 ||
- repeatedType.asGroupType().getType(0).isRepetition(REPEATED) ||
+ // can't be a synthetic layer because it would be invalid
+ repeatedType.isPrimitive()
+ || repeatedType.asGroupType().getFieldCount() > 1
+ || repeatedType.asGroupType().getType(0).isRepetition(REPEATED)
+ ||
// known patterns without the synthetic layer
- repeatedType.getName().equals("array") ||
- repeatedType.getName().equals(parentName + "_tuple") ||
+ repeatedType.getName().equals("array")
+ || repeatedType.getName().equals(parentName + "_tuple")
+ ||
// default assumption
- assumeRepeatedIsListElement
- );
+ assumeRepeatedIsListElement);
}
private static Schema optional(Schema original) {
// null is first in the union because Parquet's default is always null
- return Schema.createUnion(Arrays.asList(
- Schema.create(Schema.Type.NULL),
- original));
+ return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), original));
+ }
+
+ private static String appendPath(String path, String fieldName) {
+ if (path == null || path.isEmpty()) {
+ return fieldName;
+ }
+ return path + '.' + fieldName;
+ }
+
+ private static String namespace(String name, Map names) {
+ Integer nameCount = names.merge(name, 1, (oldValue, value) -> oldValue + 1);
+ return nameCount > 1 ? name + nameCount : null;
+ }
+
+ /* Avro <= 1.9 does not support conversions to LocalTimestamp{Micros, Millis} classes */
+ private static boolean avroVersionSupportsLocalTimestampTypes() {
+ final String avroVersion = getRuntimeAvroVersion();
+
+ return avroVersion == null
+ || !(avroVersion.startsWith("1.7.")
+ || avroVersion.startsWith("1.8.")
+ || avroVersion.startsWith("1.9."));
}
}
diff --git a/format-xls/pom.xml b/format-xls/pom.xml
index c3ef74b6f..8d40cf018 100644
--- a/format-xls/pom.xml
+++ b/format-xls/pom.xml
@@ -27,7 +27,6 @@
jar
5.2.5
- 2.20.0
@@ -45,7 +44,6 @@
org.apache.logging.log4j
log4j-core
compile
- ${log4j-core.version}