diff --git a/pom.xml b/pom.xml
index e09f83e83d..afa1f68432 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,9 +64,11 @@
6.11.0
2.13.0
3.2.6
+ 1.41.8
+ 1.33.3
0.7.1
hadoop3-2.2.21
- 1.17.1
+ 2.21.0
1.137.1
2.0.2
1.108.1
@@ -74,14 +76,15 @@
1.24.7
2.3.0
1.105.1
- 3.19.4
+ 3.25.5
1.3.0-rc3
+ 20260719
27.0.1-jre
3.3.6
- 1.4.13
- 1.4.13
- 4.5.13
- 2.13.4.2
+ 2.5.16-hadoop3
+ 4.5.14
+ 4.4.16
+ 2.18.12
4.13.1
2.0.2
1.7.5
@@ -93,6 +96,41 @@
+
+
+ com.fasterxml.jackson
+ jackson-bom
+ ${jackson.core.version}
+ pom
+ import
+
+
+
+ com.google.protobuf
+ protobuf-bom
+ ${google.protobuf.java.version}
+ pom
+ import
+
+
+
+ com.google.http-client
+ google-http-client-bom
+ ${google.http.client.version}
+ pom
+ import
+
+
+ com.google.oauth-client
+ google-oauth-client
+ ${google.oauth.client.version}
+
+
+
+ org.json
+ json
+ ${org.json.version}
+
com.google.apis
google-api-services-bigquery
@@ -103,6 +141,16 @@
google-api-services-storage
v1-rev20211201-1.32.1
+
+ org.apache.httpcomponents
+ httpclient
+ ${httpclient.version}
+
+
+ org.apache.httpcomponents
+ httpcore
+ ${httpcore.version}
+
@@ -350,33 +398,18 @@
+
com.google.cloud.bigtable
- bigtable-hbase-1.x-mapreduce
+ bigtable-hbase-2.x-hadoop
${google.cloud.bigtable.version}
org.apache.hbase
hbase-client
-
- org.apache.hbase
- hbase-server
-
-
- org.apache.hbase
- hbase-annotations
-
-
-
- org.apache.hbase
- hbase-common
-
-
- org.apache.hadoop
- hadoop-common
-
@@ -461,53 +494,29 @@
org.apache.hbase
- hbase-shaded-client
- ${hbase-shaded-client.version}
+ hbase-shaded-mapreduce
+ ${hbase-shaded-mapreduce.version}
- org.slf4j
- slf4j-log4j12
-
-
- log4j
- log4j
-
-
- org.apache.htrace
- htrace-core
+ org.apache.hadoop
+ *
-
-
-
-
- org.apache.hbase
- hbase-shaded-server
- ${hbase-shaded-server.version}
-
org.slf4j
- slf4j-log4j12
+ *
- log4j
- log4j
-
-
- org.apache.htrace
- htrace-core
+ org.apache.logging.log4j
+ *
+
- org.apache.hadoop
- hadoop-common
+ javax.annotation
+ javax.annotation-api
@@ -653,6 +662,16 @@
avro-mapred
${avro.version}
+
+
+ org.apache.avro
+ avro-ipc
+
+
+ org.apache.avro
+ avro-ipc-jetty
+
com.fasterxml.jackson.core
jackson-core
@@ -806,7 +825,7 @@
org.xerial.snappy
snappy-java
- 1.1.10.5
+ 1.1.10.12
@@ -890,12 +909,6 @@
0.2.0
-
-
- com.google.cloud
- google-cloud-bigtable
- 1.17.1
-
com.google.api.grpc
proto-google-common-protos
@@ -1282,6 +1295,13 @@
0.5.0-SNAPSHOT
test
+
+
+ com.google.cloud
+ google-cloud-bigtable
+ 1.17.1
+ test
+
ch.qos.logback
logback-classic
diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java b/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java
index 1301f0cc3d..e6196c26f2 100644
--- a/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java
+++ b/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java
@@ -41,14 +41,14 @@
import io.cdap.plugin.gcp.bigtable.common.HBaseColumn;
import io.cdap.plugin.gcp.common.SourceOutputFormatProvider;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
import org.apache.hadoop.hbase.util.Bytes;
@@ -222,15 +222,15 @@ private void validateInputSchema(Schema inputSchema, FailureCollector collector)
private void createTable(Connection connection, TableName tableName, FailureCollector collector) {
try (Admin admin = connection.getAdmin()) {
- HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
+ TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(tableName);
config.getColumnMappings(collector)
.values()
.stream()
.map(HBaseColumn::getFamily)
.distinct()
- .map(HColumnDescriptor::new)
- .forEach(tableDescriptor::addFamily);
- admin.createTable(tableDescriptor);
+ .map(family -> ColumnFamilyDescriptorBuilder.of(family))
+ .forEach(tableDescriptor::setColumnFamily);
+ admin.createTable(tableDescriptor.build());
} catch (IOException e) {
collector.addFailure(
String.format("Failed to create table '%s' in Bigtable : %s", tableName, e.getMessage()), null)
@@ -242,8 +242,8 @@ private void createTable(Connection connection, TableName tableName, FailureColl
private void validateExistingTable(Connection connection, TableName tableName, FailureCollector collector)
throws IOException {
try (Table table = connection.getTable(tableName)) {
- Set existingFamilies = table.getTableDescriptor()
- .getFamiliesKeys()
+ Set existingFamilies = table.getDescriptor()
+ .getColumnFamilyNames()
.stream()
.map(Bytes::toString)
.collect(Collectors.toSet());
diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java b/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java
index d1d05ee822..294081310c 100644
--- a/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java
+++ b/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java
@@ -192,8 +192,8 @@ private void validateOutputSchema(Configuration configuration, FailureCollector
TableName tableName = TableName.valueOf(config.table);
try (Connection connection = BigtableConfiguration.connect(configuration);
Table table = connection.getTable(tableName)) {
- Set existingFamilies = table.getTableDescriptor()
- .getFamiliesKeys()
+ Set existingFamilies = table.getDescriptor()
+ .getColumnFamilyNames()
.stream()
.map(Bytes::toString)
.collect(Collectors.toSet());
diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java b/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java
index 7caba7beec..045c33e94c 100644
--- a/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java
+++ b/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java
@@ -19,6 +19,7 @@
import io.cdap.cdap.api.data.format.UnexpectedFormatException;
import io.cdap.cdap.api.data.schema.Schema;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
@@ -49,10 +50,11 @@ public StructuredRecord transform(Result result) {
recordBuilder.set(keyAlias, value);
}
for (Cell cell : result.rawCells()) {
- String columnName = Bytes.toString(cell.getFamilyArray()) + ':' + Bytes.toString(cell.getQualifierArray());
+ String columnName = Bytes.toString(CellUtil.cloneFamily(cell)) + ':' +
+ Bytes.toString(CellUtil.cloneQualifier(cell));
String fieldName = columnMappings.get(columnName);
if (fieldName != null) {
- Object value = convertBytesToFieldValue(cell.getValueArray(), fieldName);
+ Object value = convertBytesToFieldValue(CellUtil.cloneValue(cell), fieldName);
recordBuilder.set(fieldName, value);
}
}
diff --git a/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java b/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java
new file mode 100644
index 0000000000..8231b745e4
--- /dev/null
+++ b/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright © 2026 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.gcp.gcs.source;
+
+import com.google.common.io.ByteStreams;
+import com.google.crypto.tink.Aead;
+import com.google.crypto.tink.JsonKeysetReader;
+import com.google.crypto.tink.JsonKeysetWriter;
+import com.google.crypto.tink.KeysetHandle;
+import com.google.crypto.tink.StreamingAead;
+import com.google.crypto.tink.aead.AeadConfig;
+import com.google.crypto.tink.aead.AeadKeyTemplates;
+import com.google.crypto.tink.streamingaead.StreamingAeadConfig;
+import com.google.crypto.tink.streamingaead.StreamingAeadKeyTemplates;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Base64;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Checks that the encryption metadata written by {@link GCSTinkTool} can be parsed with org.json and read back by
+ * Tink the same way {@link TinkDecryptor} does. A local AEAD key stands in for the Cloud KMS key.
+ */
+public class TinkKeysetJsonTest {
+
+ private static final int ITERATIONS = 10;
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ AeadConfig.register();
+ StreamingAeadConfig.register();
+ }
+
+ @Test
+ public void testMetadataRoundTrip() throws Exception {
+ Aead keysetAead = KeysetHandle.generateNew(AeadKeyTemplates.AES128_GCM).getPrimitive(Aead.class);
+ byte[] plaintext = "some plaintext to encrypt".getBytes(UTF_8);
+ byte[] associatedData = "gs://bucket/path/file.csv".getBytes(UTF_8);
+
+ // Each iteration uses a new keyset, so different random key ids go through the JSON round trip.
+ for (int i = 0; i < ITERATIONS; i++) {
+ KeysetHandle handle = KeysetHandle.generateNew(StreamingAeadKeyTemplates.AES256_GCM_HKDF_4KB);
+
+ // Encrypt and write the metadata the same way as GCSTinkTool.
+ ByteArrayOutputStream ciphertext = new ByteArrayOutputStream();
+ try (OutputStream os = handle.getPrimitive(StreamingAead.class).newEncryptingStream(ciphertext,
+ associatedData)) {
+ os.write(plaintext);
+ }
+ ByteArrayOutputStream keyset = new ByteArrayOutputStream();
+ handle.write(JsonKeysetWriter.withOutputStream(keyset), keysetAead);
+ JSONObject metadataObj = new JSONObject();
+ metadataObj.put("kms", "gcp-kms://projects/p/locations/global/keyRings/r/cryptoKeys/k");
+ metadataObj.put("aad", Base64.getEncoder().encodeToString(associatedData));
+ metadataObj.put("keyset", new JSONObject(new String(keyset.toByteArray(), UTF_8)));
+ String metadataJson = metadataObj.toString(4);
+
+ // Parse and read it back the same way as TinkDecryptor.
+ JSONObject metadata = new JSONObject(metadataJson);
+ KeysetHandle readHandle = KeysetHandle.read(JsonKeysetReader.withJsonObject(metadata.getJSONObject("keyset")),
+ keysetAead);
+ byte[] aad = Base64.getDecoder().decode(metadata.getString("aad"));
+ Assert.assertEquals(handle.getKeysetInfo().getPrimaryKeyId(), readHandle.getKeysetInfo().getPrimaryKeyId());
+
+ try (InputStream is = readHandle.getPrimitive(StreamingAead.class)
+ .newDecryptingStream(new ByteArrayInputStream(ciphertext.toByteArray()), aad)) {
+ Assert.assertArrayEquals(plaintext, ByteStreams.toByteArray(is));
+ }
+ }
+ }
+}