From afb725e8a4eea2758ecda83bfb9e44edc646a69f Mon Sep 17 00:00:00 2001
From: sahusanket <86019177+sahusanket@users.noreply.github.com>
Date: Thu, 19 Nov 2026 08:45:56 +0000
Subject: [PATCH] Bundle snappy-java 1.1.10.5 in format-parquet plugin
---
format-parquet/pom.xml | 4 ++
.../input/ParquetInputFormatProviderTest.java | 37 +++++++++++++++++++
pom.xml | 2 +-
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/format-parquet/pom.xml b/format-parquet/pom.xml
index 3a1a08484..3a4aae5ff 100644
--- a/format-parquet/pom.xml
+++ b/format-parquet/pom.xml
@@ -81,6 +81,10 @@
parquet-avro
1.12.0
+
+ org.xerial.snappy
+ snappy-java
+
com.google.guava
guava
diff --git a/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java b/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java
index 8195d47c9..0fe0bde8e 100644
--- a/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java
+++ b/format-parquet/src/test/java/io/cdap/plugin/format/parquet/input/ParquetInputFormatProviderTest.java
@@ -26,13 +26,18 @@
import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.parquet.avro.AvroParquetReader;
import org.apache.parquet.avro.AvroParquetWriter;
+import org.apache.parquet.hadoop.ParquetReader;
import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.xerial.snappy.Snappy;
import java.io.File;
import java.io.IOException;
@@ -154,4 +159,36 @@ public void testPathFieldConflict() throws IOException {
expectedCauses.add(expectedCause);
Assert.assertEquals(expectedCauses, actualCauses);
}
+
+ @Test
+ public void testSnappyParquetWriteAndRead() throws IOException {
+ Assert.assertNotNull(Snappy.getNativeLibraryVersion());
+ Configuration hConf = new Configuration();
+ File parquetFile = new File(TMP_FOLDER.newFolder(), "snappy.parquet");
+ Path parquetPath = new Path(parquetFile.toURI());
+ Schema expectedSchema = Schema.recordOf("x",
+ Schema.Field.of("id", Schema.of(Schema.Type.INT)),
+ Schema.Field.of("name", Schema.of(Schema.Type.STRING)));
+ org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(expectedSchema.toString());
+ GenericRecord record = new GenericRecordBuilder(avroSchema)
+ .set("id", 42)
+ .set("name", "snappy-row")
+ .build();
+
+ try (ParquetWriter parquetWriter =
+ AvroParquetWriter.builder(HadoopOutputFile.fromPath(parquetPath, hConf))
+ .withCompressionCodec(CompressionCodecName.SNAPPY)
+ .withSchema(avroSchema)
+ .build()) {
+ parquetWriter.write(record);
+ }
+
+ try (ParquetReader parquetReader =
+ AvroParquetReader.builder(HadoopInputFile.fromPath(parquetPath, hConf)).build()) {
+ GenericRecord actual = parquetReader.read();
+ Assert.assertNotNull(actual);
+ Assert.assertEquals(42, actual.get("id"));
+ Assert.assertEquals("snappy-row", actual.get("name").toString());
+ }
+ }
}
diff --git a/pom.xml b/pom.xml
index b7cea865f..963866d95 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,7 +106,7 @@
4.13.1
2.24.0
2.6
- 1.1.2
+ 1.1.10.5
1.7.15
4.0.3
3.4.5