Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 60 additions & 5 deletions adls-plugins/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,35 @@
<artifactId>adls-plugins</artifactId>

<dependencies>
<!-- Avro used to come in only through avro-ipc; declare it so the plugin jar still packages it. Its commons-*
dependencies were not packaged before either (Hadoop, provided, supplies them), so keep them out. -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
Expand All @@ -41,12 +70,21 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<!-- avro-ipc / avro-ipc-jetty are only used by Avro RPC (tether), not by AvroKeyOutputFormat. They bring
Jetty 9.4.54 (jetty-server, -http, -io, -servlet, -security, -util, -util-ajax) into the plugin jar. -->
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc-jetty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand All @@ -62,15 +100,32 @@
<dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-mapreduce</artifactId>
<version>1.1.0</version>
<version>1.9.8</version>
<exclusions>
<!-- Hadoop is provided by CDAP (hadoop-client 3.3.6, provided, in the parent pom). Without this, orc-mapreduce
packages hadoop-mapreduce-client-core / hadoop-yarn-common 2.6.0 and with them Netty 3.6.2, log4j 1.2.17,
slf4j-log4j12, jersey 1.9 and the codehaus jackson 1.9.13 jars. -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- orc-core 1.9.x needs protobuf-java 3.x and hive-storage-api at runtime, but its published (dependency-reduced)
pom does not list them. Without protobuf-java here, Hadoop's provided protobuf 2.5.0 would win. -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.25.5</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>2.8.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright © 2026 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

package io.cdap.plugin.sink;

import io.cdap.cdap.api.data.format.StructuredRecord;
import io.cdap.cdap.api.data.schema.Schema;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.orc.OrcFile;
import org.apache.orc.Reader;
import org.apache.orc.Writer;
import org.apache.orc.mapred.OrcStruct;
import org.apache.orc.mapreduce.OrcMapreduceRecordReader;
import org.apache.orc.mapreduce.OrcMapreduceRecordWriter;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import java.io.File;

/**
* Writes records produced by {@link StructuredToOrcTransformer} to an ORC file and reads them back.
*/
public class StructuredToOrcTransformerTest {

private static final Schema SCHEMA =
Schema.recordOf("record",
Schema.Field.of("bool_field", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))),
Schema.Field.of("int_field", Schema.of(Schema.Type.INT)),
Schema.Field.of("long_field", Schema.of(Schema.Type.LONG)),
Schema.Field.of("double_field", Schema.of(Schema.Type.DOUBLE)),
Schema.Field.of("bytes_field", Schema.of(Schema.Type.BYTES)),
Schema.Field.of("string_field", Schema.of(Schema.Type.STRING)));

@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();

@Test
public void testWriteAndReadOrcFile() throws Exception {
StructuredToOrcTransformer transformer = new StructuredToOrcTransformer(SCHEMA);
Configuration conf = new Configuration();
Path path = new Path(new File(tmpFolder.getRoot(), "test.orc").toURI());

OrcStruct first = transformer.transform(record(true, 1, "a"));
Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf).setSchema(first.getSchema()));
OrcMapreduceRecordWriter<OrcStruct> recordWriter = new OrcMapreduceRecordWriter<>(writer);
recordWriter.write(NullWritable.get(), first);
recordWriter.write(NullWritable.get(), transformer.transform(record(null, 2, "b")));
recordWriter.close(null);

Comment on lines +62 to +66

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The ORC Writer implements Closeable. If an exception is thrown during recordWriter.write(...) before recordWriter.close(null) is reached, the writer will not be closed, leading to a resource leak. Wrapping the Writer in a try-with-resources block ensures it is always closed properly.

Suggested change
OrcMapreduceRecordWriter<OrcStruct> recordWriter = new OrcMapreduceRecordWriter<>(writer);
recordWriter.write(NullWritable.get(), first);
recordWriter.write(NullWritable.get(), transformer.transform(record(null, 2, "b")));
recordWriter.close(null);
try (Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf).setSchema(first.getSchema()))) {
OrcMapreduceRecordWriter<OrcStruct> recordWriter = new OrcMapreduceRecordWriter<>(writer);
recordWriter.write(NullWritable.get(), first);
recordWriter.write(NullWritable.get(), transformer.transform(record(null, 2, "b")));
recordWriter.close(null);
}

Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(conf));
Assert.assertEquals(2, reader.getNumberOfRows());
Assert.assertEquals("struct<bool_field:boolean,int_field:int,long_field:bigint,double_field:double,"
+ "bytes_field:binary,string_field:string>", reader.getSchema().toString());

OrcMapreduceRecordReader<OrcStruct> recordReader = new OrcMapreduceRecordReader<>(reader, reader.options());
Assert.assertTrue(recordReader.nextKeyValue());
OrcStruct row = recordReader.getCurrentValue();
Assert.assertEquals("true", row.getFieldValue("bool_field").toString());
Assert.assertEquals("1", row.getFieldValue("int_field").toString());
Assert.assertEquals("a", row.getFieldValue("string_field").toString());
Assert.assertTrue(recordReader.nextKeyValue());
row = recordReader.getCurrentValue();
Assert.assertNull(row.getFieldValue("bool_field"));
Assert.assertEquals("2", row.getFieldValue("int_field").toString());
Assert.assertEquals("b", row.getFieldValue("string_field").toString());
Assert.assertFalse(recordReader.nextKeyValue());
recordReader.close();
Comment on lines +72 to +84

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The OrcMapreduceRecordReader implements Closeable. If any of the assertions fail, recordReader.close() will not be called, resulting in a resource leak. Using a try-with-resources block ensures that the reader is closed under all circumstances.

Suggested change
OrcMapreduceRecordReader<OrcStruct> recordReader = new OrcMapreduceRecordReader<>(reader, reader.options());
Assert.assertTrue(recordReader.nextKeyValue());
OrcStruct row = recordReader.getCurrentValue();
Assert.assertEquals("true", row.getFieldValue("bool_field").toString());
Assert.assertEquals("1", row.getFieldValue("int_field").toString());
Assert.assertEquals("a", row.getFieldValue("string_field").toString());
Assert.assertTrue(recordReader.nextKeyValue());
row = recordReader.getCurrentValue();
Assert.assertNull(row.getFieldValue("bool_field"));
Assert.assertEquals("2", row.getFieldValue("int_field").toString());
Assert.assertEquals("b", row.getFieldValue("string_field").toString());
Assert.assertFalse(recordReader.nextKeyValue());
recordReader.close();
try (OrcMapreduceRecordReader<OrcStruct> recordReader = new OrcMapreduceRecordReader<>(reader, reader.options())) {
Assert.assertTrue(recordReader.nextKeyValue());
OrcStruct row = recordReader.getCurrentValue();
Assert.assertEquals("true", row.getFieldValue("bool_field").toString());
Assert.assertEquals("1", row.getFieldValue("int_field").toString());
Assert.assertEquals("a", row.getFieldValue("string_field").toString());
Assert.assertTrue(recordReader.nextKeyValue());
row = recordReader.getCurrentValue();
Assert.assertNull(row.getFieldValue("bool_field"));
Assert.assertEquals("2", row.getFieldValue("int_field").toString());
Assert.assertEquals("b", row.getFieldValue("string_field").toString());
Assert.assertFalse(recordReader.nextKeyValue());
}

}

private static StructuredRecord record(Boolean bool, int i, String s) {

Check warning on line 87 in adls-plugins/src/test/java/io/cdap/plugin/sink/StructuredToOrcTransformerTest.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Rename this method to not match a restricted identifier.

See more on https://sonarcloud.io/project/issues?id=data-integrations_azure&issues=AaHomq-4V4C1TST6l33e&open=AaHomq-4V4C1TST6l33e&pullRequest=171
return StructuredRecord.builder(SCHEMA)
.set("bool_field", bool)
.set("int_field", i)
.set("long_field", (long) i)
.set("double_field", (double) i)
.set("bytes_field", s.getBytes())

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Using String.getBytes() without specifying an explicit charset relies on the platform's default encoding, which can cause flaky tests or unexpected behavior across different environments. It is safer to specify StandardCharsets.UTF_8 explicitly.

Suggested change
.set("bytes_field", s.getBytes())
.set("bytes_field", s.getBytes(java.nio.charset.StandardCharsets.UTF_8))

.set("string_field", s)
.build();
}
}
1 change: 0 additions & 1 deletion azure-blob-store/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
26 changes: 25 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
<cdap.version>6.11.0</cdap.version>
<hadoop.version>3.3.6</hadoop.version>
<cdap.plugin.version>2.13.0</cdap.plugin.version>
<jackson.databind.version>2.13.4.2</jackson.databind.version>
<jackson.version>2.18.12</jackson.version>
<guava.version>32.0.0-jre</guava.version>
<avro.version>1.11.4</avro.version>
<widgets.dir>widgets</widgets.dir>
Expand Down Expand Up @@ -141,6 +141,30 @@

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
<version>${jackson.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- orc-core 1.1.0 depends on jettison 1.1 and iq80 snappy 0.2 (CVE fixes in 1.5.4 / 0.5). -->
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
<dependency>
<groupId>org.iq80.snappy</groupId>
<artifactId>snappy</artifactId>
<version>0.5</version>
</dependency>
<!-- orc-core / orc-mapreduce 1.9.8 bring commons-lang3 3.12.0. -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.18.0</version>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-formats</artifactId>
Expand Down
Loading