Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
158 changes: 89 additions & 69 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -64,24 +64,27 @@
<cdap.version>6.11.0</cdap.version>
<cdap.plugin.version>2.13.0</cdap.plugin.version>
<dropwizard.metrics-core.version>3.2.6</dropwizard.metrics-core.version>
<google.http.client.version>1.41.8</google.http.client.version>
<google.oauth.client.version>1.33.3</google.oauth.client.version>
<flogger.system.backend.version>0.7.1</flogger.system.backend.version>
<gcs.connector.version>hadoop3-2.2.21</gcs.connector.version>
<google.cloud.bigtable.version>1.17.1</google.cloud.bigtable.version>
<google.cloud.bigtable.version>2.21.0</google.cloud.bigtable.version>
<google.cloud.bigquery.version>1.137.1</google.cloud.bigquery.version>
<google.cloud.kms.version>2.0.2</google.cloud.kms.version>
<google.cloud.pubsub.version>1.108.1</google.cloud.pubsub.version>
<google.cloud.spanner.version>6.10.1</google.cloud.spanner.version>
<google.cloud.speech.version>1.24.7</google.cloud.speech.version>
<google.cloud.storage.version>2.3.0</google.cloud.storage.version>
<google.cloud.datastore.version>1.105.1</google.cloud.datastore.version>
<google.protobuf.java.version>3.19.4</google.protobuf.java.version>
<google.protobuf.java.version>3.25.5</google.protobuf.java.version>
<google.tink.version>1.3.0-rc3</google.tink.version>
<org.json.version>20260719</org.json.version>
<guava.version>27.0.1-jre</guava.version>
<hadoop.version>3.3.6</hadoop.version>
<hbase-shaded-client.version>1.4.13</hbase-shaded-client.version>
<hbase-shaded-server.version>1.4.13</hbase-shaded-server.version>
<httpclient.version>4.5.13</httpclient.version>
<jackson.core.version>2.13.4.2</jackson.core.version>
<hbase-shaded-mapreduce.version>2.5.16-hadoop3</hbase-shaded-mapreduce.version>
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
<jackson.core.version>2.18.12</jackson.core.version>
<junit.version>4.13.1</junit.version>
<powermock.version>2.0.2</powermock.version>
<slf4j.version>1.7.5</slf4j.version>
Expand All @@ -93,6 +96,41 @@

<dependencyManagement>
<dependencies>
<!-- Aligns every Jackson module (jackson-core, -databind, -dataformat-cbor, ...) on one version. -->
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
<version>${jackson.core.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- Aligns protobuf-java and protobuf-java-util. -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-bom</artifactId>
<version>${google.protobuf.java.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- google-oauth-client 1.33.3 is built against google-http-client 1.41.x. -->
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-bom</artifactId>
<version>${google.http.client.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.google.oauth-client</groupId>
<artifactId>google-oauth-client</artifactId>
<version>${google.oauth.client.version}</version>
</dependency>
<!-- org.json comes in through tink and is used by GCSTinkTool / TinkDecryptor. -->
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>${org.json.version}</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-bigquery</artifactId>
Expand All @@ -103,6 +141,16 @@
<artifactId>google-api-services-storage</artifactId>
<version>v1-rev20211201-1.32.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down Expand Up @@ -350,33 +398,18 @@
</exclusions>
</dependency>
<dependency>
<!--
Shaded Bigtable HBase 2.x client (it bundles and relocates grpc, gax, protobuf and guava).
Its unshaded hbase-client is replaced by hbase-shaded-mapreduce below.
-->
<groupId>com.google.cloud.bigtable</groupId>
<artifactId>bigtable-hbase-1.x-mapreduce</artifactId>
<artifactId>bigtable-hbase-2.x-hadoop</artifactId>
<version>${google.cloud.bigtable.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
</exclusion>
<!-- bigtable-hbase-1.x-mapreduce version 1.17.1 has a dependency on hbase-common:1.4.12
which masks the shaded dependencies below . Exclude this dependency.
All the classes should be present in the shaded jars-->
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -461,53 +494,29 @@
</dependency>
<dependency>
<!--
Required by bigtable-hbase-1.x-mapreduce instead of excluded non-shaded version.
Shaded library is used to avoid dependency conflicts with Datastore module on profobuf-java dependency.
Bigtable requires version 2.x and Datastore module requires 3.x protocol.
HBase client and TableInputFormat / TableOutputFormat used by the Bigtable source and sink, with
HBase's own third-party dependencies (protobuf, netty, guava, ...) relocated. Hadoop is provided by CDAP.
-->
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client</artifactId>
<version>${hbase-shaded-client.version}</version>
<artifactId>hbase-shaded-mapreduce</artifactId>
<version>${hbase-shaded-mapreduce.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
<groupId>org.apache.hadoop</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<!--
Required by bigtable-hbase-1.x-mapreduce instead of excluded non-shaded version.
Shaded library is used to avoid dependency conflicts with Datastore module on profobuf-java dependency.
Bigtable requires version 2.x and Datastore module requires 3.x protocol.
-->
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-server</artifactId>
<version>${hbase-shaded-server.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>*</artifactId>
</exclusion>
<!-- Keep javax.annotation-api 1.3.2 from the Google Cloud clients instead of 1.2. -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down Expand Up @@ -653,6 +662,16 @@
<artifactId>avro-mapred</artifactId>
<version>${avro.version}</version>
<exclusions>
<!-- avro-ipc is only needed for Avro RPC / tether jobs, which the plugins do not use.
It brings in velocity-engine-core and, through avro-ipc-jetty, Jetty. -->
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc-jetty</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
Expand Down Expand Up @@ -806,7 +825,7 @@
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.5</version>
<version>1.1.10.12</version>
</dependency>
<!-- Start: dependency for Google PubSub Streaming Source -->
<dependency>
Expand Down Expand Up @@ -890,12 +909,6 @@
<version>0.2.0</version>
</dependency>
<!-- End: dependency used by the Dataplex connector -->
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-bigtable -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>1.17.1</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-common-protos</artifactId>
Expand Down Expand Up @@ -1282,6 +1295,13 @@
<version>0.5.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<!-- Bigtable admin client used by the e2e test setup only. -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>1.17.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
Expand Down
16 changes: 8 additions & 8 deletions src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@
import io.cdap.plugin.gcp.bigtable.common.HBaseColumn;
import io.cdap.plugin.gcp.common.SourceOutputFormatProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.util.Bytes;
Expand Down Expand Up @@ -222,15 +222,15 @@ private void validateInputSchema(Schema inputSchema, FailureCollector collector)

private void createTable(Connection connection, TableName tableName, FailureCollector collector) {
try (Admin admin = connection.getAdmin()) {
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(tableName);
config.getColumnMappings(collector)
.values()
.stream()
.map(HBaseColumn::getFamily)
.distinct()
.map(HColumnDescriptor::new)
.forEach(tableDescriptor::addFamily);
admin.createTable(tableDescriptor);
.map(family -> ColumnFamilyDescriptorBuilder.of(family))
.forEach(tableDescriptor::setColumnFamily);
admin.createTable(tableDescriptor.build());
} catch (IOException e) {
collector.addFailure(
String.format("Failed to create table '%s' in Bigtable : %s", tableName, e.getMessage()), null)
Expand All @@ -242,8 +242,8 @@ private void createTable(Connection connection, TableName tableName, FailureColl
private void validateExistingTable(Connection connection, TableName tableName, FailureCollector collector)
throws IOException {
try (Table table = connection.getTable(tableName)) {
Set<String> existingFamilies = table.getTableDescriptor()
.getFamiliesKeys()
Set<String> existingFamilies = table.getDescriptor()
.getColumnFamilyNames()
.stream()
.map(Bytes::toString)
.collect(Collectors.toSet());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,8 @@ private void validateOutputSchema(Configuration configuration, FailureCollector
TableName tableName = TableName.valueOf(config.table);
try (Connection connection = BigtableConfiguration.connect(configuration);
Table table = connection.getTable(tableName)) {
Set<String> existingFamilies = table.getTableDescriptor()
.getFamiliesKeys()
Set<String> existingFamilies = table.getDescriptor()
.getColumnFamilyNames()
.stream()
.map(Bytes::toString)
.collect(Collectors.toSet());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import io.cdap.cdap.api.data.format.UnexpectedFormatException;
import io.cdap.cdap.api.data.schema.Schema;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;

Expand Down Expand Up @@ -49,10 +50,11 @@ public StructuredRecord transform(Result result) {
recordBuilder.set(keyAlias, value);
}
for (Cell cell : result.rawCells()) {
String columnName = Bytes.toString(cell.getFamilyArray()) + ':' + Bytes.toString(cell.getQualifierArray());
String columnName = Bytes.toString(CellUtil.cloneFamily(cell)) + ':' +
Bytes.toString(CellUtil.cloneQualifier(cell));
Comment on lines +53 to +54

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Using CellUtil.cloneFamily(cell) and CellUtil.cloneQualifier(cell) allocates a new byte[] for every cell in every row. In high-throughput data pipelines, this creates significant garbage collection pressure.

Instead, you can use the Bytes.toString(byte[], int, int) overload with the cell's array, offset, and length to decode the string directly from the backing array without allocating intermediate byte arrays.

Suggested change
String columnName = Bytes.toString(CellUtil.cloneFamily(cell)) + ':' +
Bytes.toString(CellUtil.cloneQualifier(cell));
String columnName = Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()) + ':' +
Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());

String fieldName = columnMappings.get(columnName);
if (fieldName != null) {
Object value = convertBytesToFieldValue(cell.getValueArray(), fieldName);
Object value = convertBytesToFieldValue(CellUtil.cloneValue(cell), fieldName);
recordBuilder.set(fieldName, value);
}
}
Expand Down
Loading
Loading