diff --git a/kafka-plugins-client/pom.xml b/kafka-plugins-client/pom.xml index 650bfc9..c80fe0a 100644 --- a/kafka-plugins-client/pom.xml +++ b/kafka-plugins-client/pom.xml @@ -61,10 +61,22 @@ + + + commons-io + commons-io + 2.16.1 + org.apache.kafka kafka_2.12 ${kafka.version} + + test org.slf4j @@ -203,11 +215,13 @@ io.netty netty-all ${netty.version} + test io.cdap.http netty-http ${netty-http.version} + test com.fasterxml.jackson.core @@ -253,7 +267,7 @@ <_exportcontents>io.cdap.plugin.kafka.*;org.apache.spark.streaming.kafka010.*; org.apache.kafka.*;com.google.common.base.*; - *;inline=false;scope=compile + *;inline=false;scope=compile|runtime true lib diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java index 49d9362..ed1e13d 100644 --- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java +++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java @@ -78,6 +78,9 @@ public void initialize(AlertPublisherContext context) throws Exception { props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); props.put("producer.type", "sync"); + // Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the + // cluster. Keep the 2.x behavior; users can still turn it on through producerProperties. + props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false"); // Override any property set above with user specified producer properties for (Map.Entry producerProperty : config.getProducerProperties().entrySet()) { diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java index 4ad98e9..cb4028b 100644 --- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java +++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java @@ -30,6 +30,7 @@ import java.util.NoSuchElementException; import java.util.Properties; import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32; /** @@ -92,11 +93,24 @@ public KafkaMessage getNext(KafkaKey kafkaKey) { ByteBuffer payload = value == null ? ByteBuffer.wrap(EMPTY_BYTE_ARRAY) : ByteBuffer.wrap(value); kafkaKey.set(currentOffset, consumerRecord.offset() + 1, - consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), consumerRecord.checksum()); + consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), checksum(consumerRecord)); currentOffset = consumerRecord.offset() + 1; // increase offset return new KafkaMessage(payload, key); } + /** + * Returns a checksum of the record's timestamp and serialized key and value sizes. This is the same partial + * checksum that {@code ConsumerRecord.checksum()} returned for v2 records before it was removed in Kafka 3.0 + * (computed with CRC32 instead of CRC32C). It is only used to order and identify {@link KafkaKey}s. + */ + private static long checksum(ConsumerRecord record) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + 2 * Integer.BYTES); + buffer.putLong(record.timestamp()).putInt(record.serializedKeySize()).putInt(record.serializedValueSize()); + CRC32 crc = new CRC32(); + crc.update(buffer.array()); + return crc.getValue(); + } + /** * Fetch messages from Kafka. * diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java index 1ad1ad6..6e41f5b 100644 --- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java +++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java @@ -251,6 +251,9 @@ private static class KafkaOutputFormatProvider implements OutputFormatProvider { conf.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName()); KafkaHelpers.setupKerberosLogin(conf, kafkaSinkConfig.principal, kafkaSinkConfig.keytabLocation); + // Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the + // cluster. Keep the 2.x behavior; users can still turn it on through kafkaProperties. + conf.put("additional." + ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false"); addKafkaProperties(kafkaSinkConfig.kafkaProperties); conf.put(ASYNC, kafkaSinkConfig.async); diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java index 780e495..2db0856 100644 --- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java +++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java @@ -35,7 +35,7 @@ import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.requests.ListOffsetRequest; +import org.apache.kafka.common.requests.ListOffsetsRequest; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.Function; @@ -118,9 +118,9 @@ static JavaInputDStream> getConsumerRecordJavaDSt for (Map.Entry entry : offsets.entrySet()) { TopicPartition topicAndPartition = entry.getKey(); Long offset = entry.getValue(); - if (offset == ListOffsetRequest.EARLIEST_TIMESTAMP) { + if (offset == ListOffsetsRequest.EARLIEST_TIMESTAMP) { earliestOffsetRequest.add(topicAndPartition); - } else if (offset == ListOffsetRequest.LATEST_TIMESTAMP) { + } else if (offset == ListOffsetsRequest.LATEST_TIMESTAMP) { latestOffsetRequest.add(topicAndPartition); } } diff --git a/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java b/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java index 6d29a0a..dd3bff6 100644 --- a/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java +++ b/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java @@ -19,26 +19,20 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.util.concurrent.AbstractIdleService; -import kafka.metrics.KafkaMetricsReporter; import kafka.server.KafkaConfig; import kafka.server.KafkaServer; import kafka.zookeeper.ZooKeeperClientExpiredException; import kafka.zookeeper.ZooKeeperClientTimeoutException; -import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.utils.Time; import org.apache.twill.internal.utils.Networks; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; -import scala.collection.JavaConverters; -import scala.collection.Seq; import java.net.BindException; -import java.util.Collections; import java.util.Properties; import java.util.Random; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; /** * A {@link com.google.common.util.concurrent.Service} implementation for running an instance of Kafka server in @@ -84,7 +78,7 @@ protected void startUp() throws Exception { LOG.warn("Session expired when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause); } else if (rootCause instanceof BindException) { - LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause); + LOG.warn("Kafka failed to bind to {}. Attempt number {}.", kafkaConfig.listeners(), tries, rootCause); } else { LOG.error("Exception when connecting to Zookeeper", e); throw e; @@ -109,46 +103,7 @@ protected void shutDown() throws Exception { } private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) { - Seq metricsReporters = - JavaConverters.collectionAsScalaIterableConverter( - Collections.emptyList()).asScala().toSeq(); - return new KafkaServer(kafkaConfig, new Time() { - - @Override - public long milliseconds() { - return System.currentTimeMillis(); - } - - @Override - public long nanoseconds() { - return System.nanoTime(); - } - - @Override - public void sleep(long ms) { - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - Thread.interrupted(); - } - } - - @Override - public void waitObject(Object o, Supplier condition, long deadlineMs) throws InterruptedException { - while (!condition.get()) { - long currentTimeMs = this.milliseconds(); - if (currentTimeMs >= deadlineMs) { - throw new TimeoutException("Condition not satisfied before deadline"); - } - o.wait(deadlineMs - currentTimeMs); - } - } - - @Override - public long hiResClockMs() { - return System.currentTimeMillis(); - } - }, Option.apply("embedded-server"), metricsReporters); + return new KafkaServer(kafkaConfig, Time.SYSTEM, Option.apply("embedded-server"), false); } /** @@ -163,8 +118,11 @@ private KafkaConfig createKafkaConfig(Properties properties) { if (port == null || "0".equals(port)) { int randomPort = Networks.getRandomPort(); Preconditions.checkState(randomPort > 0, "Failed to get random port."); - prop.setProperty("port", Integer.toString(randomPort)); + port = Integer.toString(randomPort); } + // Kafka 3.0 removed the "port" broker property; translate it to a PLAINTEXT listener. + prop.remove("port"); + prop.setProperty("listeners", "PLAINTEXT://localhost:" + port); return new KafkaConfig(prop); } diff --git a/pom.xml b/pom.xml index c7016f1..2cbfe60 100644 --- a/pom.xml +++ b/pom.xml @@ -80,17 +80,42 @@ 3.1.2 widgets docs - 2.6.0 + 3.9.2 2.6.0 4.1.75.Final 1.3.0 - 1.8.2 + 1.11.4 ${project.basedir} 1.0.9 1.7.25 - 2.10.5.1 + 2.18.12 + + + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + + org.apache.commons + commons-compress + 1.26.2 + + + + org.xerial.snappy + snappy-java + 1.1.10.12 + + + + io.cdap.cdap @@ -262,7 +287,7 @@ org.apache.commons commons-lang3 - 3.9 + 3.18.0 com.google.guava