diff --git a/format-parquet/pom.xml b/format-parquet/pom.xml index 3a1a08484..3a4aae5ff 100644 --- a/format-parquet/pom.xml +++ b/format-parquet/pom.xml @@ -81,6 +81,10 @@ parquet-avro 1.12.0 + + org.xerial.snappy + snappy-java + com.google.guava guava diff --git a/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java b/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java index 8195d47c9..0fe0bde8e 100644 --- a/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java +++ b/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java @@ -26,13 +26,18 @@ import org.apache.avro.generic.GenericRecordBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetReader; import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.xerial.snappy.Snappy; import java.io.File; import java.io.IOException; @@ -154,4 +159,36 @@ public void testPathFieldConflict() throws IOException { expectedCauses.add(expectedCause); Assert.assertEquals(expectedCauses, actualCauses); } + + @Test + public void testSnappyParquetWriteAndRead() throws IOException { + Assert.assertNotNull(Snappy.getNativeLibraryVersion()); + Configuration hConf = new Configuration(); + File parquetFile = new File(TMP_FOLDER.newFolder(), "snappy.parquet"); + Path parquetPath = new Path(parquetFile.toURI()); + Schema expectedSchema = Schema.recordOf("x", + Schema.Field.of("id", Schema.of(Schema.Type.INT)), + Schema.Field.of("name", Schema.of(Schema.Type.STRING))); + org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(expectedSchema.toString()); + GenericRecord record = new GenericRecordBuilder(avroSchema) + .set("id", 42) + .set("name", "snappy-row") + .build(); + + try (ParquetWriter parquetWriter = + AvroParquetWriter.builder(HadoopOutputFile.fromPath(parquetPath, hConf)) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withSchema(avroSchema) + .build()) { + parquetWriter.write(record); + } + + try (ParquetReader parquetReader = + AvroParquetReader.builder(HadoopInputFile.fromPath(parquetPath, hConf)).build()) { + GenericRecord actual = parquetReader.read(); + Assert.assertNotNull(actual); + Assert.assertEquals(42, actual.get("id")); + Assert.assertEquals("snappy-row", actual.get("name").toString()); + } + } } diff --git a/pom.xml b/pom.xml index b7cea865f..963866d95 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ 4.13.1 2.24.0 2.6 - 1.1.2 + 1.1.10.5 1.7.15 4.0.3 3.4.5