diff --git a/core-plugins/pom.xml b/core-plugins/pom.xml index 74e7d5c49..8ffed57ca 100644 --- a/core-plugins/pom.xml +++ b/core-plugins/pom.xml @@ -136,7 +136,7 @@ org.bouncycastle bcprov-jdk15to18 - 1.84 + 1.85 @@ -196,7 +196,6 @@ commons-io commons-io - 2.15.0 org.apache.commons diff --git a/database-plugins/pom.xml b/database-plugins/pom.xml index 90524b353..d4b9fcf89 100644 --- a/database-plugins/pom.xml +++ b/database-plugins/pom.xml @@ -35,6 +35,8 @@ org.apache.hadoop hadoop-client + + provided io.cdap.cdap diff --git a/format-delimited/pom.xml b/format-delimited/pom.xml index c17c45571..b35a556b8 100644 --- a/format-delimited/pom.xml +++ b/format-delimited/pom.xml @@ -30,7 +30,6 @@ jar - 2.5 3.1 @@ -78,7 +77,6 @@ commons-io commons-io - ${commons-io.version} diff --git a/format-orc/pom.xml b/format-orc/pom.xml index cc99170cc..dde22aa14 100644 --- a/format-orc/pom.xml +++ b/format-orc/pom.xml @@ -29,6 +29,14 @@ ORC format plugins jar + + + 1.9.8 + 3.25.5 + 2.8.1 + 3.18.0 + + io.cdap.cdap @@ -60,7 +68,7 @@ org.apache.orc orc-mapreduce - 1.1.0 + ${orc.version} org.apache.hadoop @@ -69,13 +77,30 @@ + + org.apache.hive + hive-storage-api + ${hive-storage-api.version} + + + * + * + + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + com.google.protobuf protobuf-java - 2.5.0 + ${protobuf.version} junit diff --git a/format-parquet/pom.xml b/format-parquet/pom.xml index 3a1a08484..002e42361 100644 --- a/format-parquet/pom.xml +++ b/format-parquet/pom.xml @@ -79,7 +79,14 @@ org.apache.parquet parquet-avro - 1.12.0 + + + + org.apache.parquet + parquet-jackson com.google.guava diff --git a/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java b/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java index 62a3687af..105ce8b14 100644 --- a/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java +++ b/format-parquet/src/main/java/org/apache/parquet/avro/AvroSchemaConverter.java @@ -18,32 +18,16 @@ */ package org.apache.parquet.avro; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; - -import org.apache.hadoop.conf.Configuration; -import org.apache.parquet.schema.ConversionPatterns; -import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.LogicalTypeAnnotation; -import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.PrimitiveType; -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; -import org.apache.parquet.schema.Type; -import org.apache.parquet.schema.Types; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - import static java.util.Optional.empty; import static java.util.Optional.of; -import static org.apache.avro.JsonProperties.NULL_VALUE; +import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED; +import static org.apache.parquet.avro.AvroReadSupport.READ_INT96_AS_FIXED_DEFAULT; +import static org.apache.parquet.avro.AvroRecordConverter.getRuntimeAvroVersion; +import static org.apache.parquet.avro.AvroWriteSupport.WRITE_FIXED_AS_INT96; import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE; import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT; +import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID; +import static org.apache.parquet.avro.AvroWriteSupport.WRITE_PARQUET_UUID_DEFAULT; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS; import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS; import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType; @@ -52,29 +36,64 @@ import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType; import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType; import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; -import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.*; +import static org.apache.parquet.schema.LogicalTypeAnnotation.uuidType; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.apache.parquet.schema.Type.Repetition.REPEATED; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.avro.LogicalType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.conf.HadoopParquetConfiguration; +import org.apache.parquet.conf.ParquetConfiguration; +import org.apache.parquet.schema.ConversionPatterns; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.LogicalTypeAnnotation.UUIDLogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; + /** *

* Converts an Avro schema into a Parquet schema, or vice versa. See package * documentation for details of the mapping. *

* - * A copy of the AvroSchemaConverter, except with handling for INT96. + * A copy of parquet-avro 1.15.2's AvroSchemaConverter, except with handling for INT96: INT96 columns + * are converted to Avro bytes instead of failing when READ_INT96_AS_FIXED is not set. */ public class AvroSchemaConverter { - public static final String ADD_LIST_ELEMENT_RECORDS = - "parquet.avro.add-list-element-records"; + public static final String ADD_LIST_ELEMENT_RECORDS = "parquet.avro.add-list-element-records"; private static final boolean ADD_LIST_ELEMENT_RECORDS_DEFAULT = true; private final boolean assumeRepeatedIsListElement; private final boolean writeOldListStructure; + private final boolean writeParquetUUID; + private final boolean readInt96AsFixed; + private final Set pathsToInt96; public AvroSchemaConverter() { - this.assumeRepeatedIsListElement = ADD_LIST_ELEMENT_RECORDS_DEFAULT; - this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT; + this(ADD_LIST_ELEMENT_RECORDS_DEFAULT); } /** @@ -86,13 +105,21 @@ public AvroSchemaConverter() { AvroSchemaConverter(boolean assumeRepeatedIsListElement) { this.assumeRepeatedIsListElement = assumeRepeatedIsListElement; this.writeOldListStructure = WRITE_OLD_LIST_STRUCTURE_DEFAULT; + this.writeParquetUUID = WRITE_PARQUET_UUID_DEFAULT; + this.readInt96AsFixed = READ_INT96_AS_FIXED_DEFAULT; + this.pathsToInt96 = Collections.emptySet(); } public AvroSchemaConverter(Configuration conf) { - this.assumeRepeatedIsListElement = conf.getBoolean( - ADD_LIST_ELEMENT_RECORDS, ADD_LIST_ELEMENT_RECORDS_DEFAULT); - this.writeOldListStructure = conf.getBoolean( - WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT); + this(new HadoopParquetConfiguration(conf)); + } + + public AvroSchemaConverter(ParquetConfiguration conf) { + this.assumeRepeatedIsListElement = conf.getBoolean(ADD_LIST_ELEMENT_RECORDS, ADD_LIST_ELEMENT_RECORDS_DEFAULT); + this.writeOldListStructure = conf.getBoolean(WRITE_OLD_LIST_STRUCTURE, WRITE_OLD_LIST_STRUCTURE_DEFAULT); + this.writeParquetUUID = conf.getBoolean(WRITE_PARQUET_UUID, WRITE_PARQUET_UUID_DEFAULT); + this.readInt96AsFixed = conf.getBoolean(READ_INT96_AS_FIXED, READ_INT96_AS_FIXED_DEFAULT); + this.pathsToInt96 = new HashSet<>(Arrays.asList(conf.getStrings(WRITE_FIXED_AS_INT96, new String[0]))); } /** @@ -125,28 +152,29 @@ public MessageType convert(Schema avroSchema) { if (!avroSchema.getType().equals(Schema.Type.RECORD)) { throw new IllegalArgumentException("Avro schema must be a record."); } - return new MessageType(avroSchema.getFullName(), convertFields(avroSchema.getFields())); + return new MessageType(avroSchema.getFullName(), convertFields(avroSchema.getFields(), "")); } - private List convertFields(List fields) { + private List convertFields(List fields, String schemaPath) { List types = new ArrayList(); for (Schema.Field field : fields) { if (field.schema().getType().equals(Schema.Type.NULL)) { continue; // Avro nulls are not encoded, unless they are null unions } - types.add(convertField(field)); + types.add(convertField(field, appendPath(schemaPath, field.name()))); } return types; } - private Type convertField(String fieldName, Schema schema) { - return convertField(fieldName, schema, Type.Repetition.REQUIRED); + private Type convertField(String fieldName, Schema schema, String schemaPath) { + return convertField(fieldName, schema, Type.Repetition.REQUIRED, schemaPath); } @SuppressWarnings("deprecation") - private Type convertField(String fieldName, Schema schema, Type.Repetition repetition) { + private Type convertField(String fieldName, Schema schema, Type.Repetition repetition, String schemaPath) { Types.PrimitiveBuilder builder; Schema.Type type = schema.getType(); + LogicalType logicalType = schema.getLogicalType(); if (type.equals(Schema.Type.BOOLEAN)) { builder = Types.primitive(BOOLEAN, repetition); } else if (type.equals(Schema.Type.INT)) { @@ -160,35 +188,50 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet } else if (type.equals(Schema.Type.BYTES)) { builder = Types.primitive(BINARY, repetition); } else if (type.equals(Schema.Type.STRING)) { - builder = Types.primitive(BINARY, repetition).as(stringType()); + if (logicalType != null + && logicalType.getName().equals(LogicalTypes.uuid().getName()) + && writeParquetUUID) { + builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition) + .length(LogicalTypeAnnotation.UUIDLogicalTypeAnnotation.BYTES); + } else { + builder = Types.primitive(BINARY, repetition).as(stringType()); + } } else if (type.equals(Schema.Type.RECORD)) { - return new GroupType(repetition, fieldName, convertFields(schema.getFields())); + return new GroupType(repetition, fieldName, convertFields(schema.getFields(), schemaPath)); } else if (type.equals(Schema.Type.ENUM)) { builder = Types.primitive(BINARY, repetition).as(enumType()); } else if (type.equals(Schema.Type.ARRAY)) { if (writeOldListStructure) { - return ConversionPatterns.listType(repetition, fieldName, - convertField("array", schema.getElementType(), REPEATED)); + return ConversionPatterns.listType( + repetition, fieldName, convertField("array", schema.getElementType(), REPEATED, schemaPath)); } else { - return ConversionPatterns.listOfElements(repetition, fieldName, - convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType())); + return ConversionPatterns.listOfElements( + repetition, + fieldName, + convertField(AvroWriteSupport.LIST_ELEMENT_NAME, schema.getElementType(), schemaPath)); } } else if (type.equals(Schema.Type.MAP)) { - Type valType = convertField("value", schema.getValueType()); + Type valType = convertField("value", schema.getValueType(), schemaPath); // avro map key type is always string return ConversionPatterns.stringKeyMapType(repetition, fieldName, valType); } else if (type.equals(Schema.Type.FIXED)) { - builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition) - .length(schema.getFixedSize()); + if (pathsToInt96.contains(schemaPath)) { + if (schema.getFixedSize() != 12) { + throw new IllegalArgumentException("The size of the fixed type field " + schemaPath + + " must be 12 bytes for INT96 conversion"); + } + builder = Types.primitive(PrimitiveTypeName.INT96, repetition); + } else { + builder = Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition).length(schema.getFixedSize()); + } } else if (type.equals(Schema.Type.UNION)) { - return convertUnion(fieldName, schema, repetition); + return convertUnion(fieldName, schema, repetition, schemaPath); } else { throw new UnsupportedOperationException("Cannot convert Avro type " + type); } // schema translation can only be done for known logical types because this // creates an equivalence - LogicalType logicalType = schema.getLogicalType(); if (logicalType != null) { if (logicalType instanceof LogicalTypes.Decimal) { LogicalTypes.Decimal decimal = (LogicalTypes.Decimal) logicalType; @@ -204,7 +247,7 @@ private Type convertField(String fieldName, Schema schema, Type.Repetition repet return builder.named(fieldName); } - private Type convertUnion(String fieldName, Schema schema, Type.Repetition repetition) { + private Type convertUnion(String fieldName, Schema schema, Type.Repetition repetition, String schemaPath) { List nonNullSchemas = new ArrayList(schema.getTypes().size()); // Found any schemas in the union? Required for the edge case, where the union contains only a single type. boolean foundNullSchema = false; @@ -225,25 +268,27 @@ private Type convertUnion(String fieldName, Schema schema, Type.Repetition repet throw new UnsupportedOperationException("Cannot convert Avro union of only nulls"); case 1: - return foundNullSchema ? convertField(fieldName, nonNullSchemas.get(0), repetition) : - convertUnionToGroupType(fieldName, repetition, nonNullSchemas); + return foundNullSchema + ? convertField(fieldName, nonNullSchemas.get(0), repetition, schemaPath) + : convertUnionToGroupType(fieldName, repetition, nonNullSchemas, schemaPath); default: // complex union type - return convertUnionToGroupType(fieldName, repetition, nonNullSchemas); + return convertUnionToGroupType(fieldName, repetition, nonNullSchemas, schemaPath); } } - private Type convertUnionToGroupType(String fieldName, Type.Repetition repetition, List nonNullSchemas) { + private Type convertUnionToGroupType( + String fieldName, Type.Repetition repetition, List nonNullSchemas, String schemaPath) { List unionTypes = new ArrayList(nonNullSchemas.size()); int index = 0; for (Schema childSchema : nonNullSchemas) { - unionTypes.add( convertField("member" + index++, childSchema, Type.Repetition.OPTIONAL)); + unionTypes.add(convertField("member" + index++, childSchema, Type.Repetition.OPTIONAL, schemaPath)); } return new GroupType(repetition, fieldName, unionTypes); } - private Type convertField(Schema.Field field) { - return convertField(field.name(), field.schema()); + private Type convertField(Schema.Field field, String schemaPath) { + return convertField(field.name(), field.schema(), schemaPath); } public Schema convert(MessageType parquetSchema) { @@ -255,150 +300,177 @@ Schema convert(GroupType parquetSchema) { } private Schema convertFields(String name, List parquetFields, Map names) { - List fields = new ArrayList(); - Integer nameCount = names.merge(name, 1, (oldValue, value) -> oldValue + 1); + SchemaBuilder.FieldAssembler builder = + SchemaBuilder.builder(namespace(name, names)).record(name).fields(); for (Type parquetType : parquetFields) { Schema fieldSchema = convertField(parquetType, names); - if (parquetType.isRepetition(REPEATED)) { - throw new UnsupportedOperationException("REPEATED not supported outside LIST or MAP. Type: " + parquetType); + if (parquetType.isRepetition(REPEATED)) { // If a repeated field is ungrouped, treat as REQUIRED per spec + builder.name(parquetType.getName()) + .type() + .array() + .items() + .type(fieldSchema) + .arrayDefault(new ArrayList<>()); } else if (parquetType.isRepetition(Type.Repetition.OPTIONAL)) { - fields.add(new Schema.Field( - parquetType.getName(), optional(fieldSchema), null, NULL_VALUE)); + builder.name(parquetType.getName()).type().optional().type(fieldSchema); } else { // REQUIRED - fields.add(new Schema.Field( - parquetType.getName(), fieldSchema, null, (Object) null)); + builder.name(parquetType.getName()).type(fieldSchema).noDefault(); } } - Schema schema = Schema.createRecord(name, null, nameCount > 1 ? name + nameCount : null, false); - schema.setFields(fields); - return schema; + return builder.endRecord(); } private Schema convertField(final Type parquetType, Map names) { if (parquetType.isPrimitive()) { final PrimitiveType asPrimitive = parquetType.asPrimitiveType(); - final PrimitiveTypeName parquetPrimitiveTypeName = - asPrimitive.getPrimitiveTypeName(); + final PrimitiveTypeName parquetPrimitiveTypeName = asPrimitive.getPrimitiveTypeName(); final LogicalTypeAnnotation annotation = parquetType.getLogicalTypeAnnotation(); Schema schema = parquetPrimitiveTypeName.convert( - new PrimitiveType.PrimitiveTypeNameConverter() { - @Override - public Schema convertBOOLEAN(PrimitiveTypeName primitiveTypeName) { - return Schema.create(Schema.Type.BOOLEAN); - } - @Override - public Schema convertINT32(PrimitiveTypeName primitiveTypeName) { - return Schema.create(Schema.Type.INT); - } - @Override - public Schema convertINT64(PrimitiveTypeName primitiveTypeName) { - return Schema.create(Schema.Type.LONG); - } - @Override - public Schema convertINT96(PrimitiveTypeName primitiveTypeName) { - // this is the modified part - return Schema.create(Schema.Type.BYTES); - } - @Override - public Schema convertFLOAT(PrimitiveTypeName primitiveTypeName) { - return Schema.create(Schema.Type.FLOAT); - } - @Override - public Schema convertDOUBLE(PrimitiveTypeName primitiveTypeName) { - return Schema.create(Schema.Type.DOUBLE); - } - @Override - public Schema convertFIXED_LEN_BYTE_ARRAY(PrimitiveTypeName primitiveTypeName) { - int size = parquetType.asPrimitiveType().getTypeLength(); - return Schema.createFixed(parquetType.getName(), null, null, size); - } - @Override - public Schema convertBINARY(PrimitiveTypeName primitiveTypeName) { - if (annotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation || - annotation instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation) { - return Schema.create(Schema.Type.STRING); - } else { + new PrimitiveType.PrimitiveTypeNameConverter() { + @Override + public Schema convertBOOLEAN(PrimitiveTypeName primitiveTypeName) { + return Schema.create(Schema.Type.BOOLEAN); + } + + @Override + public Schema convertINT32(PrimitiveTypeName primitiveTypeName) { + return Schema.create(Schema.Type.INT); + } + + @Override + public Schema convertINT64(PrimitiveTypeName primitiveTypeName) { + return Schema.create(Schema.Type.LONG); + } + + @Override + public Schema convertINT96(PrimitiveTypeName primitiveTypeName) { + if (readInt96AsFixed) { + return Schema.createFixed("INT96", "INT96 represented as byte[12]", null, 12); + } + // this is the modified part: CDAP reads INT96 as bytes (see AvroIndexedRecordConverter) return Schema.create(Schema.Type.BYTES); } - } - }); + + @Override + public Schema convertFLOAT(PrimitiveTypeName primitiveTypeName) { + return Schema.create(Schema.Type.FLOAT); + } + + @Override + public Schema convertDOUBLE(PrimitiveTypeName primitiveTypeName) { + return Schema.create(Schema.Type.DOUBLE); + } + + @Override + public Schema convertFIXED_LEN_BYTE_ARRAY(PrimitiveTypeName primitiveTypeName) { + if (annotation instanceof LogicalTypeAnnotation.UUIDLogicalTypeAnnotation) { + return Schema.create(Schema.Type.STRING); + } else { + int size = parquetType.asPrimitiveType().getTypeLength(); + String name = parquetType.getName(); + String ns = namespace(name, names); + return Schema.createFixed(name, null, ns, size); + } + } + + @Override + public Schema convertBINARY(PrimitiveTypeName primitiveTypeName) { + if (annotation instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation + || annotation instanceof LogicalTypeAnnotation.EnumLogicalTypeAnnotation) { + return Schema.create(Schema.Type.STRING); + } else { + return Schema.create(Schema.Type.BYTES); + } + } + }); LogicalType logicalType = convertLogicalType(annotation); - if (logicalType != null && (!(annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) || - parquetPrimitiveTypeName == BINARY || - parquetPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY)) { + if (logicalType != null + && (!(annotation instanceof LogicalTypeAnnotation.DecimalLogicalTypeAnnotation) + || parquetPrimitiveTypeName == BINARY + || parquetPrimitiveTypeName == FIXED_LEN_BYTE_ARRAY)) { schema = logicalType.addToSchema(schema); } return schema; - } else { GroupType parquetGroupType = parquetType.asGroupType(); LogicalTypeAnnotation logicalTypeAnnotation = parquetGroupType.getLogicalTypeAnnotation(); if (logicalTypeAnnotation != null) { - return logicalTypeAnnotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { - @Override - public Optional visit(LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) { - if (parquetGroupType.getFieldCount()!= 1) { - throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); - } - Type repeatedType = parquetGroupType.getType(0); - if (!repeatedType.isRepetition(REPEATED)) { - throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); - } - if (isElementType(repeatedType, parquetGroupType.getName())) { - // repeated element types are always required - return of(Schema.createArray(convertField(repeatedType, names))); - } else { - Type elementType = repeatedType.asGroupType().getType(0); - if (elementType.isRepetition(Type.Repetition.OPTIONAL)) { - return of(Schema.createArray(optional(convertField(elementType, names)))); - } else { - return of(Schema.createArray(convertField(elementType, names))); + return logicalTypeAnnotation + .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { + @Override + public Optional visit( + LogicalTypeAnnotation.ListLogicalTypeAnnotation listLogicalType) { + if (parquetGroupType.getFieldCount() != 1) { + throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); + } + Type repeatedType = parquetGroupType.getType(0); + if (!repeatedType.isRepetition(REPEATED)) { + throw new UnsupportedOperationException("Invalid list type " + parquetGroupType); + } + if (isElementType(repeatedType, parquetGroupType.getName())) { + // repeated element types are always required + return of(Schema.createArray(convertField(repeatedType, names))); + } else { + Type elementType = + repeatedType.asGroupType().getType(0); + if (elementType.isRepetition(Type.Repetition.OPTIONAL)) { + return of(Schema.createArray(optional(convertField(elementType, names)))); + } else { + return of(Schema.createArray(convertField(elementType, names))); + } + } } - } - } - @Override - // for backward-compatibility - public Optional visit(LogicalTypeAnnotation.MapKeyValueTypeAnnotation mapKeyValueLogicalType) { - return visitMapOrMapKeyValue(); - } + @Override + // for backward-compatibility + public Optional visit( + LogicalTypeAnnotation.MapKeyValueTypeAnnotation mapKeyValueLogicalType) { + return visitMapOrMapKeyValue(); + } - @Override - public Optional visit(LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) { - return visitMapOrMapKeyValue(); - } + @Override + public Optional visit( + LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) { + return visitMapOrMapKeyValue(); + } - private Optional visitMapOrMapKeyValue() { - if (parquetGroupType.getFieldCount() != 1 || parquetGroupType.getType(0).isPrimitive()) { - throw new UnsupportedOperationException("Invalid map type " + parquetGroupType); - } - GroupType mapKeyValType = parquetGroupType.getType(0).asGroupType(); - if (!mapKeyValType.isRepetition(REPEATED) || - mapKeyValType.getFieldCount()!=2) { - throw new UnsupportedOperationException("Invalid map type " + parquetGroupType); - } - Type keyType = mapKeyValType.getType(0); - if (!keyType.isPrimitive() || - !keyType.asPrimitiveType().getPrimitiveTypeName().equals(PrimitiveTypeName.BINARY) || - !keyType.getLogicalTypeAnnotation().equals(stringType())) { - throw new IllegalArgumentException("Map key type must be binary (UTF8): " - + keyType); - } - Type valueType = mapKeyValType.getType(1); - if (valueType.isRepetition(Type.Repetition.OPTIONAL)) { - return of(Schema.createMap(optional(convertField(valueType, names)))); - } else { - return of(Schema.createMap(convertField(valueType, names))); - } - } + private Optional visitMapOrMapKeyValue() { + if (parquetGroupType.getFieldCount() != 1 + || parquetGroupType.getType(0).isPrimitive()) { + throw new UnsupportedOperationException("Invalid map type " + parquetGroupType); + } + GroupType mapKeyValType = + parquetGroupType.getType(0).asGroupType(); + if (!mapKeyValType.isRepetition(REPEATED) || mapKeyValType.getFieldCount() != 2) { + throw new UnsupportedOperationException("Invalid map type " + parquetGroupType); + } + Type keyType = mapKeyValType.getType(0); + if (!keyType.isPrimitive() + || !keyType.asPrimitiveType() + .getPrimitiveTypeName() + .equals(PrimitiveTypeName.BINARY) + || !keyType.getLogicalTypeAnnotation().equals(stringType())) { + throw new IllegalArgumentException( + "Map key type must be binary (UTF8): " + keyType); + } + Type valueType = mapKeyValType.getType(1); + if (valueType.isRepetition(Type.Repetition.OPTIONAL)) { + return of(Schema.createMap(optional(convertField(valueType, names)))); + } else { + return of(Schema.createMap(convertField(valueType, names))); + } + } - @Override - public Optional visit(LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) { - return of(Schema.create(Schema.Type.STRING)); - } - }).orElseThrow(() -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType)); + @Override + public Optional visit( + LogicalTypeAnnotation.EnumLogicalTypeAnnotation enumLogicalType) { + return of(Schema.create(Schema.Type.STRING)); + } + }) + .orElseThrow( + () -> new UnsupportedOperationException("Cannot convert Parquet type " + parquetType)); } else { // if no original type then it's a record return convertFields(parquetGroupType.getName(), parquetGroupType.getFields(), names); @@ -422,7 +494,18 @@ private LogicalTypeAnnotation convertLogicalType(LogicalType logicalType) { return timestampType(true, MILLIS); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { return timestampType(true, MICROS); + } else if (logicalType.getName().equals(LogicalTypes.uuid().getName()) && writeParquetUUID) { + return uuidType(); } + + if (avroVersionSupportsLocalTimestampTypes()) { + if (logicalType instanceof LogicalTypes.LocalTimestampMillis) { + return timestampType(false, MILLIS); + } else if (logicalType instanceof LogicalTypes.LocalTimestampMicros) { + return timestampType(false, MICROS); + } + } + return null; } @@ -430,41 +513,65 @@ private LogicalType convertLogicalType(LogicalTypeAnnotation annotation) { if (annotation == null) { return null; } - return annotation.accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { - @Override - public Optional visit(LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) { - return of(LogicalTypes.decimal(decimalLogicalType.getPrecision(), decimalLogicalType.getScale())); - } + return annotation + .accept(new LogicalTypeAnnotation.LogicalTypeAnnotationVisitor() { + @Override + public Optional visit( + LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalType) { + return of( + LogicalTypes.decimal(decimalLogicalType.getPrecision(), decimalLogicalType.getScale())); + } - @Override - public Optional visit(LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) { - return of(LogicalTypes.date()); - } + @Override + public Optional visit( + LogicalTypeAnnotation.DateLogicalTypeAnnotation dateLogicalType) { + return of(LogicalTypes.date()); + } - @Override - public Optional visit(LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) { - LogicalTypeAnnotation.TimeUnit unit = timeLogicalType.getUnit(); - switch (unit) { - case MILLIS: - return of(LogicalTypes.timeMillis()); - case MICROS: - return of(LogicalTypes.timeMicros()); - } - return empty(); - } + @Override + public Optional visit( + LogicalTypeAnnotation.TimeLogicalTypeAnnotation timeLogicalType) { + LogicalTypeAnnotation.TimeUnit unit = timeLogicalType.getUnit(); + switch (unit) { + case MILLIS: + return of(LogicalTypes.timeMillis()); + case MICROS: + return of(LogicalTypes.timeMicros()); + } + return empty(); + } - @Override - public Optional visit(LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) { - LogicalTypeAnnotation.TimeUnit unit = timestampLogicalType.getUnit(); - switch (unit) { - case MILLIS: - return of(LogicalTypes.timestampMillis()); - case MICROS: - return of(LogicalTypes.timestampMicros()); - } - return empty(); - } - }).orElse(null); + @Override + public Optional visit( + LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) { + LogicalTypeAnnotation.TimeUnit unit = timestampLogicalType.getUnit(); + boolean isAdjustedToUTC = timestampLogicalType.isAdjustedToUTC(); + + if (isAdjustedToUTC || !avroVersionSupportsLocalTimestampTypes()) { + switch (unit) { + case MILLIS: + return of(LogicalTypes.timestampMillis()); + case MICROS: + return of(LogicalTypes.timestampMicros()); + } + return empty(); + } else { + switch (unit) { + case MILLIS: + return of(LogicalTypes.localTimestampMillis()); + case MICROS: + return of(LogicalTypes.localTimestampMicros()); + } + return empty(); + } + } + + @Override + public Optional visit(UUIDLogicalTypeAnnotation uuidLogicalType) { + return of(LogicalTypes.uuid()); + } + }) + .orElse(null); } /** @@ -478,22 +585,43 @@ public Optional visit(LogicalTypeAnnotation.TimestampLogicalTypeAnn */ private boolean isElementType(Type repeatedType, String parentName) { return ( - // can't be a synthetic layer because it would be invalid - repeatedType.isPrimitive() || - repeatedType.asGroupType().getFieldCount() > 1 || - repeatedType.asGroupType().getType(0).isRepetition(REPEATED) || + // can't be a synthetic layer because it would be invalid + repeatedType.isPrimitive() + || repeatedType.asGroupType().getFieldCount() > 1 + || repeatedType.asGroupType().getType(0).isRepetition(REPEATED) + || // known patterns without the synthetic layer - repeatedType.getName().equals("array") || - repeatedType.getName().equals(parentName + "_tuple") || + repeatedType.getName().equals("array") + || repeatedType.getName().equals(parentName + "_tuple") + || // default assumption - assumeRepeatedIsListElement - ); + assumeRepeatedIsListElement); } private static Schema optional(Schema original) { // null is first in the union because Parquet's default is always null - return Schema.createUnion(Arrays.asList( - Schema.create(Schema.Type.NULL), - original)); + return Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), original)); + } + + private static String appendPath(String path, String fieldName) { + if (path == null || path.isEmpty()) { + return fieldName; + } + return path + '.' + fieldName; + } + + private static String namespace(String name, Map names) { + Integer nameCount = names.merge(name, 1, (oldValue, value) -> oldValue + 1); + return nameCount > 1 ? name + nameCount : null; + } + + /* Avro <= 1.9 does not support conversions to LocalTimestamp{Micros, Millis} classes */ + private static boolean avroVersionSupportsLocalTimestampTypes() { + final String avroVersion = getRuntimeAvroVersion(); + + return avroVersion == null + || !(avroVersion.startsWith("1.7.") + || avroVersion.startsWith("1.8.") + || avroVersion.startsWith("1.9.")); } } diff --git a/format-xls/pom.xml b/format-xls/pom.xml index c3ef74b6f..8d40cf018 100644 --- a/format-xls/pom.xml +++ b/format-xls/pom.xml @@ -27,7 +27,6 @@ jar 5.2.5 - 2.20.0 @@ -45,7 +44,6 @@ org.apache.logging.log4j log4j-core compile - ${log4j-core.version}
io.cdap.cdap @@ -71,7 +69,6 @@ commons-io commons-io - 2.15.0 org.apache.commons diff --git a/pom.xml b/pom.xml index 594766c6c..29d2bfd65 100644 --- a/pom.xml +++ b/pom.xml @@ -82,12 +82,14 @@ UTF-8 1.2 + 2.0.3 7.1 1.11.4 3.11.12 2.0.2.2 1.10 3.1 + 2.16.1 2.6 3.5 3.2.2 @@ -101,20 +103,27 @@ 13.0.1 3.3.6 2.2.4 + 0.5 1.4.1 4.13.1 2.24.0 2.6 - 1.1.2 + 1.1.10.12 1.7.15 4.0.3 3.4.5 2.2.0 - 20160212 + 20260719 + 2.4.10 + 2.25.5 4.1.75.Final 1.3.0 + 1.15.2 + + 1.17.1 3.3.2 - 2.13.4.2 + 2.18.12 + ${jackson.version} ${project.basedir}/src/test/java/ @@ -133,6 +142,37 @@ + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + org.apache.logging.log4j + log4j-bom + ${log4j2.version} + pom + import + + + commons-io + commons-io + ${commons-io.version} + + + + net.minidev + json-smart + ${json-smart.version} + + + + org.iq80.snappy + snappy + ${iq80-snappy.version} + org.slf4j slf4j-api @@ -495,6 +535,46 @@ snappy-java ${snappy.version} + + org.apache.parquet + parquet-avro + ${parquet.version} + + + org.apache.parquet + parquet-hadoop + ${parquet.version} + + + org.apache.parquet + parquet-column + ${parquet.version} + + + org.apache.parquet + parquet-common + ${parquet.version} + + + org.apache.parquet + parquet-encoding + ${parquet.version} + + + org.apache.parquet + parquet-format-structures + ${parquet.version} + + + org.apache.parquet + parquet-jackson + ${parquet-jackson.version} + + + io.airlift + aircompressor + ${aircompressor.version} + org.cassandraunit cassandra-unit @@ -624,6 +704,15 @@ avro-mapred ${avro.version} + + + org.apache.avro + avro-ipc + + + org.apache.avro + avro-ipc-jetty + org.mortbay.jetty jetty