diff --git a/pom.xml b/pom.xml index e09f83e83d..afa1f68432 100644 --- a/pom.xml +++ b/pom.xml @@ -64,9 +64,11 @@ 6.11.0 2.13.0 3.2.6 + 1.41.8 + 1.33.3 0.7.1 hadoop3-2.2.21 - 1.17.1 + 2.21.0 1.137.1 2.0.2 1.108.1 @@ -74,14 +76,15 @@ 1.24.7 2.3.0 1.105.1 - 3.19.4 + 3.25.5 1.3.0-rc3 + 20260719 27.0.1-jre 3.3.6 - 1.4.13 - 1.4.13 - 4.5.13 - 2.13.4.2 + 2.5.16-hadoop3 + 4.5.14 + 4.4.16 + 2.18.12 4.13.1 2.0.2 1.7.5 @@ -93,6 +96,41 @@ + + + com.fasterxml.jackson + jackson-bom + ${jackson.core.version} + pom + import + + + + com.google.protobuf + protobuf-bom + ${google.protobuf.java.version} + pom + import + + + + com.google.http-client + google-http-client-bom + ${google.http.client.version} + pom + import + + + com.google.oauth-client + google-oauth-client + ${google.oauth.client.version} + + + + org.json + json + ${org.json.version} + com.google.apis google-api-services-bigquery @@ -103,6 +141,16 @@ google-api-services-storage v1-rev20211201-1.32.1 + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpcore + ${httpcore.version} + @@ -350,33 +398,18 @@ + com.google.cloud.bigtable - bigtable-hbase-1.x-mapreduce + bigtable-hbase-2.x-hadoop ${google.cloud.bigtable.version} org.apache.hbase hbase-client - - org.apache.hbase - hbase-server - - - org.apache.hbase - hbase-annotations - - - - org.apache.hbase - hbase-common - - - org.apache.hadoop - hadoop-common - @@ -461,53 +494,29 @@ org.apache.hbase - hbase-shaded-client - ${hbase-shaded-client.version} + hbase-shaded-mapreduce + ${hbase-shaded-mapreduce.version} - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.apache.htrace - htrace-core + org.apache.hadoop + * - - - - - org.apache.hbase - hbase-shaded-server - ${hbase-shaded-server.version} - org.slf4j - slf4j-log4j12 + * - log4j - log4j - - - org.apache.htrace - htrace-core + org.apache.logging.log4j + * + - org.apache.hadoop - hadoop-common + javax.annotation + javax.annotation-api @@ -653,6 +662,16 @@ avro-mapred ${avro.version} + + + org.apache.avro + avro-ipc + + + org.apache.avro + avro-ipc-jetty + com.fasterxml.jackson.core jackson-core @@ -806,7 +825,7 @@ org.xerial.snappy snappy-java - 1.1.10.5 + 1.1.10.12 @@ -890,12 +909,6 @@ 0.2.0 - - - com.google.cloud - google-cloud-bigtable - 1.17.1 - com.google.api.grpc proto-google-common-protos @@ -1282,6 +1295,13 @@ 0.5.0-SNAPSHOT test + + + com.google.cloud + google-cloud-bigtable + 1.17.1 + test + ch.qos.logback logback-classic diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java b/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java index 1301f0cc3d..e6196c26f2 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java +++ b/src/main/java/io/cdap/plugin/gcp/bigtable/sink/BigtableSink.java @@ -41,14 +41,14 @@ import io.cdap.plugin.gcp.bigtable.common.HBaseColumn; import io.cdap.plugin.gcp.common.SourceOutputFormatProvider; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.hbase.util.Bytes; @@ -222,15 +222,15 @@ private void validateInputSchema(Schema inputSchema, FailureCollector collector) private void createTable(Connection connection, TableName tableName, FailureCollector collector) { try (Admin admin = connection.getAdmin()) { - HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); + TableDescriptorBuilder tableDescriptor = TableDescriptorBuilder.newBuilder(tableName); config.getColumnMappings(collector) .values() .stream() .map(HBaseColumn::getFamily) .distinct() - .map(HColumnDescriptor::new) - .forEach(tableDescriptor::addFamily); - admin.createTable(tableDescriptor); + .map(family -> ColumnFamilyDescriptorBuilder.of(family)) + .forEach(tableDescriptor::setColumnFamily); + admin.createTable(tableDescriptor.build()); } catch (IOException e) { collector.addFailure( String.format("Failed to create table '%s' in Bigtable : %s", tableName, e.getMessage()), null) @@ -242,8 +242,8 @@ private void createTable(Connection connection, TableName tableName, FailureColl private void validateExistingTable(Connection connection, TableName tableName, FailureCollector collector) throws IOException { try (Table table = connection.getTable(tableName)) { - Set existingFamilies = table.getTableDescriptor() - .getFamiliesKeys() + Set existingFamilies = table.getDescriptor() + .getColumnFamilyNames() .stream() .map(Bytes::toString) .collect(Collectors.toSet()); diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java b/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java index d1d05ee822..294081310c 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java +++ b/src/main/java/io/cdap/plugin/gcp/bigtable/source/BigtableSource.java @@ -192,8 +192,8 @@ private void validateOutputSchema(Configuration configuration, FailureCollector TableName tableName = TableName.valueOf(config.table); try (Connection connection = BigtableConfiguration.connect(configuration); Table table = connection.getTable(tableName)) { - Set existingFamilies = table.getTableDescriptor() - .getFamiliesKeys() + Set existingFamilies = table.getDescriptor() + .getColumnFamilyNames() .stream() .map(Bytes::toString) .collect(Collectors.toSet()); diff --git a/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java b/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java index 7caba7beec..045c33e94c 100644 --- a/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java +++ b/src/main/java/io/cdap/plugin/gcp/bigtable/source/HBaseResultToRecordTransformer.java @@ -19,6 +19,7 @@ import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; @@ -49,10 +50,11 @@ public StructuredRecord transform(Result result) { recordBuilder.set(keyAlias, value); } for (Cell cell : result.rawCells()) { - String columnName = Bytes.toString(cell.getFamilyArray()) + ':' + Bytes.toString(cell.getQualifierArray()); + String columnName = Bytes.toString(CellUtil.cloneFamily(cell)) + ':' + + Bytes.toString(CellUtil.cloneQualifier(cell)); String fieldName = columnMappings.get(columnName); if (fieldName != null) { - Object value = convertBytesToFieldValue(cell.getValueArray(), fieldName); + Object value = convertBytesToFieldValue(CellUtil.cloneValue(cell), fieldName); recordBuilder.set(fieldName, value); } } diff --git a/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java b/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java new file mode 100644 index 0000000000..8231b745e4 --- /dev/null +++ b/src/test/java/io/cdap/plugin/gcp/gcs/source/TinkKeysetJsonTest.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcp.gcs.source; + +import com.google.common.io.ByteStreams; +import com.google.crypto.tink.Aead; +import com.google.crypto.tink.JsonKeysetReader; +import com.google.crypto.tink.JsonKeysetWriter; +import com.google.crypto.tink.KeysetHandle; +import com.google.crypto.tink.StreamingAead; +import com.google.crypto.tink.aead.AeadConfig; +import com.google.crypto.tink.aead.AeadKeyTemplates; +import com.google.crypto.tink.streamingaead.StreamingAeadConfig; +import com.google.crypto.tink.streamingaead.StreamingAeadKeyTemplates; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Base64; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Checks that the encryption metadata written by {@link GCSTinkTool} can be parsed with org.json and read back by + * Tink the same way {@link TinkDecryptor} does. A local AEAD key stands in for the Cloud KMS key. + */ +public class TinkKeysetJsonTest { + + private static final int ITERATIONS = 10; + + @BeforeClass + public static void setupClass() throws Exception { + AeadConfig.register(); + StreamingAeadConfig.register(); + } + + @Test + public void testMetadataRoundTrip() throws Exception { + Aead keysetAead = KeysetHandle.generateNew(AeadKeyTemplates.AES128_GCM).getPrimitive(Aead.class); + byte[] plaintext = "some plaintext to encrypt".getBytes(UTF_8); + byte[] associatedData = "gs://bucket/path/file.csv".getBytes(UTF_8); + + // Each iteration uses a new keyset, so different random key ids go through the JSON round trip. + for (int i = 0; i < ITERATIONS; i++) { + KeysetHandle handle = KeysetHandle.generateNew(StreamingAeadKeyTemplates.AES256_GCM_HKDF_4KB); + + // Encrypt and write the metadata the same way as GCSTinkTool. + ByteArrayOutputStream ciphertext = new ByteArrayOutputStream(); + try (OutputStream os = handle.getPrimitive(StreamingAead.class).newEncryptingStream(ciphertext, + associatedData)) { + os.write(plaintext); + } + ByteArrayOutputStream keyset = new ByteArrayOutputStream(); + handle.write(JsonKeysetWriter.withOutputStream(keyset), keysetAead); + JSONObject metadataObj = new JSONObject(); + metadataObj.put("kms", "gcp-kms://projects/p/locations/global/keyRings/r/cryptoKeys/k"); + metadataObj.put("aad", Base64.getEncoder().encodeToString(associatedData)); + metadataObj.put("keyset", new JSONObject(new String(keyset.toByteArray(), UTF_8))); + String metadataJson = metadataObj.toString(4); + + // Parse and read it back the same way as TinkDecryptor. + JSONObject metadata = new JSONObject(metadataJson); + KeysetHandle readHandle = KeysetHandle.read(JsonKeysetReader.withJsonObject(metadata.getJSONObject("keyset")), + keysetAead); + byte[] aad = Base64.getDecoder().decode(metadata.getString("aad")); + Assert.assertEquals(handle.getKeysetInfo().getPrimaryKeyId(), readHandle.getKeysetInfo().getPrimaryKeyId()); + + try (InputStream is = readHandle.getPrimitive(StreamingAead.class) + .newDecryptingStream(new ByteArrayInputStream(ciphertext.toByteArray()), aad)) { + Assert.assertArrayEquals(plaintext, ByteStreams.toByteArray(is)); + } + } + } +}