diff --git a/adls-plugins/pom.xml b/adls-plugins/pom.xml index 84ba8d2..a0a591b 100644 --- a/adls-plugins/pom.xml +++ b/adls-plugins/pom.xml @@ -28,6 +28,35 @@ adls-plugins + + + org.apache.avro + avro + ${avro.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.commons + commons-compress + + + commons-io + commons-io + + + org.apache.commons + commons-lang3 + + + org.apache.avro avro-mapred @@ -41,12 +70,21 @@ com.fasterxml.jackson.core jackson-databind + + + org.apache.avro + avro-ipc + + + org.apache.avro + avro-ipc-jetty + com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} junit @@ -62,15 +100,32 @@ org.apache.orc orc-mapreduce - 1.1.0 + 1.9.8 + org.apache.hadoop - hadoop-hdfs + * + + + + + com.google.protobuf + protobuf-java + 3.25.5 + + + org.apache.hive + hive-storage-api + 2.8.1 + - org.apache.hadoop - hadoop-common + * + * diff --git a/adls-plugins/src/test/java/io/cdap/plugin/sink/StructuredToOrcTransformerTest.java b/adls-plugins/src/test/java/io/cdap/plugin/sink/StructuredToOrcTransformerTest.java new file mode 100644 index 0000000..e89eb35 --- /dev/null +++ b/adls-plugins/src/test/java/io/cdap/plugin/sink/StructuredToOrcTransformerTest.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sink; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.NullWritable; +import org.apache.orc.OrcFile; +import org.apache.orc.Reader; +import org.apache.orc.Writer; +import org.apache.orc.mapred.OrcStruct; +import org.apache.orc.mapreduce.OrcMapreduceRecordReader; +import org.apache.orc.mapreduce.OrcMapreduceRecordWriter; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +/** + * Writes records produced by {@link StructuredToOrcTransformer} to an ORC file and reads them back. + */ +public class StructuredToOrcTransformerTest { + + private static final Schema SCHEMA = + Schema.recordOf("record", + Schema.Field.of("bool_field", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))), + Schema.Field.of("int_field", Schema.of(Schema.Type.INT)), + Schema.Field.of("long_field", Schema.of(Schema.Type.LONG)), + Schema.Field.of("double_field", Schema.of(Schema.Type.DOUBLE)), + Schema.Field.of("bytes_field", Schema.of(Schema.Type.BYTES)), + Schema.Field.of("string_field", Schema.of(Schema.Type.STRING))); + + @Rule + public TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Test + public void testWriteAndReadOrcFile() throws Exception { + StructuredToOrcTransformer transformer = new StructuredToOrcTransformer(SCHEMA); + Configuration conf = new Configuration(); + Path path = new Path(new File(tmpFolder.getRoot(), "test.orc").toURI()); + + OrcStruct first = transformer.transform(record(true, 1, "a")); + Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf).setSchema(first.getSchema())); + OrcMapreduceRecordWriter recordWriter = new OrcMapreduceRecordWriter<>(writer); + recordWriter.write(NullWritable.get(), first); + recordWriter.write(NullWritable.get(), transformer.transform(record(null, 2, "b"))); + recordWriter.close(null); + + Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf)); + Assert.assertEquals(2, reader.getNumberOfRows()); + Assert.assertEquals("struct", reader.getSchema().toString()); + + OrcMapreduceRecordReader recordReader = new OrcMapreduceRecordReader<>(reader, reader.options()); + Assert.assertTrue(recordReader.nextKeyValue()); + OrcStruct row = recordReader.getCurrentValue(); + Assert.assertEquals("true", row.getFieldValue("bool_field").toString()); + Assert.assertEquals("1", row.getFieldValue("int_field").toString()); + Assert.assertEquals("a", row.getFieldValue("string_field").toString()); + Assert.assertTrue(recordReader.nextKeyValue()); + row = recordReader.getCurrentValue(); + Assert.assertNull(row.getFieldValue("bool_field")); + Assert.assertEquals("2", row.getFieldValue("int_field").toString()); + Assert.assertEquals("b", row.getFieldValue("string_field").toString()); + Assert.assertFalse(recordReader.nextKeyValue()); + recordReader.close(); + } + + private static StructuredRecord record(Boolean bool, int i, String s) { + return StructuredRecord.builder(SCHEMA) + .set("bool_field", bool) + .set("int_field", i) + .set("long_field", (long) i) + .set("double_field", (double) i) + .set("bytes_field", s.getBytes()) + .set("string_field", s) + .build(); + } +} diff --git a/azure-blob-store/pom.xml b/azure-blob-store/pom.xml index b35f2d1..053c5e3 100644 --- a/azure-blob-store/pom.xml +++ b/azure-blob-store/pom.xml @@ -42,7 +42,6 @@ com.fasterxml.jackson.core jackson-databind - ${jackson.databind.version} org.apache.commons diff --git a/pom.xml b/pom.xml index 1e5b084..d15560d 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ 6.11.0 3.3.6 2.13.0 - 2.13.4.2 + 2.18.12 32.0.0-jre 1.11.4 widgets @@ -141,6 +141,30 @@ + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + + org.codehaus.jettison + jettison + 1.5.4 + + + org.iq80.snappy + snappy + 0.5 + + + + org.apache.commons + commons-lang3 + 3.18.0 + io.cdap.cdap cdap-formats