diff --git a/kafka-plugins-client/pom.xml b/kafka-plugins-client/pom.xml
index 650bfc9..c80fe0a 100644
--- a/kafka-plugins-client/pom.xml
+++ b/kafka-plugins-client/pom.xml
@@ -61,10 +61,22 @@
+
+
+ commons-io
+ commons-io
+ 2.16.1
+
org.apache.kafka
kafka_2.12
${kafka.version}
+
+ test
org.slf4j
@@ -203,11 +215,13 @@
io.netty
netty-all
${netty.version}
+ test
io.cdap.http
netty-http
${netty-http.version}
+ test
com.fasterxml.jackson.core
@@ -253,7 +267,7 @@
<_exportcontents>io.cdap.plugin.kafka.*;org.apache.spark.streaming.kafka010.*;
org.apache.kafka.*;com.google.common.base.*;
- *;inline=false;scope=compile
+ *;inline=false;scope=compile|runtime
true
lib
diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java
index 49d9362..ed1e13d 100644
--- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java
+++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/alertpublisher/KafkaAlertPublisher.java
@@ -78,6 +78,9 @@ public void initialize(AlertPublisherContext context) throws Exception {
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put("producer.type", "sync");
+ // Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the
+ // cluster. Keep the 2.x behavior; users can still turn it on through producerProperties.
+ props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false");
// Override any property set above with user specified producer properties
for (Map.Entry producerProperty : config.getProducerProperties().entrySet()) {
diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java
index 4ad98e9..cb4028b 100644
--- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java
+++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/batch/source/Kafka10Reader.java
@@ -30,6 +30,7 @@
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import java.util.zip.CRC32;
/**
@@ -92,11 +93,24 @@ public KafkaMessage getNext(KafkaKey kafkaKey) {
ByteBuffer payload = value == null ? ByteBuffer.wrap(EMPTY_BYTE_ARRAY) : ByteBuffer.wrap(value);
kafkaKey.set(currentOffset, consumerRecord.offset() + 1,
- consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), consumerRecord.checksum());
+ consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), checksum(consumerRecord));
currentOffset = consumerRecord.offset() + 1; // increase offset
return new KafkaMessage(payload, key);
}
+ /**
+ * Returns a checksum of the record's timestamp and serialized key and value sizes. This is the same partial
+ * checksum that {@code ConsumerRecord.checksum()} returned for v2 records before it was removed in Kafka 3.0
+ * (computed with CRC32 instead of CRC32C). It is only used to order and identify {@link KafkaKey}s.
+ */
+ private static long checksum(ConsumerRecord record) {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + 2 * Integer.BYTES);
+ buffer.putLong(record.timestamp()).putInt(record.serializedKeySize()).putInt(record.serializedValueSize());
+ CRC32 crc = new CRC32();
+ crc.update(buffer.array());
+ return crc.getValue();
+ }
+
/**
* Fetch messages from Kafka.
*
diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java
index 1ad1ad6..6e41f5b 100644
--- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java
+++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/sink/KafkaBatchSink.java
@@ -251,6 +251,9 @@ private static class KafkaOutputFormatProvider implements OutputFormatProvider {
conf.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
KafkaHelpers.setupKerberosLogin(conf, kafkaSinkConfig.principal, kafkaSinkConfig.keytabLocation);
+ // Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the
+ // cluster. Keep the 2.x behavior; users can still turn it on through kafkaProperties.
+ conf.put("additional." + ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false");
addKafkaProperties(kafkaSinkConfig.kafkaProperties);
conf.put(ASYNC, kafkaSinkConfig.async);
diff --git a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java
index 780e495..2db0856 100644
--- a/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java
+++ b/kafka-plugins-client/src/main/java/io/cdap/plugin/kafka/source/KafkaStreamingSourceUtil.java
@@ -35,7 +35,7 @@
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.requests.ListOffsetRequest;
+import org.apache.kafka.common.requests.ListOffsetsRequest;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
@@ -118,9 +118,9 @@ static JavaInputDStream> getConsumerRecordJavaDSt
for (Map.Entry entry : offsets.entrySet()) {
TopicPartition topicAndPartition = entry.getKey();
Long offset = entry.getValue();
- if (offset == ListOffsetRequest.EARLIEST_TIMESTAMP) {
+ if (offset == ListOffsetsRequest.EARLIEST_TIMESTAMP) {
earliestOffsetRequest.add(topicAndPartition);
- } else if (offset == ListOffsetRequest.LATEST_TIMESTAMP) {
+ } else if (offset == ListOffsetsRequest.LATEST_TIMESTAMP) {
latestOffsetRequest.add(topicAndPartition);
}
}
diff --git a/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java b/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java
index 6d29a0a..dd3bff6 100644
--- a/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java
+++ b/kafka-plugins-client/src/test/java/io/cdap/plugin/kafka/EmbeddedKafkaServer.java
@@ -19,26 +19,20 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.AbstractIdleService;
-import kafka.metrics.KafkaMetricsReporter;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
import kafka.zookeeper.ZooKeeperClientExpiredException;
import kafka.zookeeper.ZooKeeperClientTimeoutException;
-import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.utils.Time;
import org.apache.twill.internal.utils.Networks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
-import scala.collection.JavaConverters;
-import scala.collection.Seq;
import java.net.BindException;
-import java.util.Collections;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
/**
* A {@link com.google.common.util.concurrent.Service} implementation for running an instance of Kafka server in
@@ -84,7 +78,7 @@ protected void startUp() throws Exception {
LOG.warn("Session expired when connecting to ZooKeeper from KafkaServer. Attempt number {}.",
tries, rootCause);
} else if (rootCause instanceof BindException) {
- LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause);
+ LOG.warn("Kafka failed to bind to {}. Attempt number {}.", kafkaConfig.listeners(), tries, rootCause);
} else {
LOG.error("Exception when connecting to Zookeeper", e);
throw e;
@@ -109,46 +103,7 @@ protected void shutDown() throws Exception {
}
private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
- Seq metricsReporters =
- JavaConverters.collectionAsScalaIterableConverter(
- Collections.emptyList()).asScala().toSeq();
- return new KafkaServer(kafkaConfig, new Time() {
-
- @Override
- public long milliseconds() {
- return System.currentTimeMillis();
- }
-
- @Override
- public long nanoseconds() {
- return System.nanoTime();
- }
-
- @Override
- public void sleep(long ms) {
- try {
- Thread.sleep(ms);
- } catch (InterruptedException e) {
- Thread.interrupted();
- }
- }
-
- @Override
- public void waitObject(Object o, Supplier condition, long deadlineMs) throws InterruptedException {
- while (!condition.get()) {
- long currentTimeMs = this.milliseconds();
- if (currentTimeMs >= deadlineMs) {
- throw new TimeoutException("Condition not satisfied before deadline");
- }
- o.wait(deadlineMs - currentTimeMs);
- }
- }
-
- @Override
- public long hiResClockMs() {
- return System.currentTimeMillis();
- }
- }, Option.apply("embedded-server"), metricsReporters);
+ return new KafkaServer(kafkaConfig, Time.SYSTEM, Option.apply("embedded-server"), false);
}
/**
@@ -163,8 +118,11 @@ private KafkaConfig createKafkaConfig(Properties properties) {
if (port == null || "0".equals(port)) {
int randomPort = Networks.getRandomPort();
Preconditions.checkState(randomPort > 0, "Failed to get random port.");
- prop.setProperty("port", Integer.toString(randomPort));
+ port = Integer.toString(randomPort);
}
+ // Kafka 3.0 removed the "port" broker property; translate it to a PLAINTEXT listener.
+ prop.remove("port");
+ prop.setProperty("listeners", "PLAINTEXT://localhost:" + port);
return new KafkaConfig(prop);
}
diff --git a/pom.xml b/pom.xml
index c7016f1..2cbfe60 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,17 +80,42 @@
3.1.2
widgets
docs
- 2.6.0
+ 3.9.2
2.6.0
4.1.75.Final
1.3.0
- 1.8.2
+ 1.11.4
${project.basedir}
1.0.9
1.7.25
- 2.10.5.1
+ 2.18.12
+
+
+
+
+ com.fasterxml.jackson
+ jackson-bom
+ ${jackson.version}
+ pom
+ import
+
+
+
+ org.apache.commons
+ commons-compress
+ 1.26.2
+
+
+
+ org.xerial.snappy
+ snappy-java
+ 1.1.10.12
+
+
+
+
io.cdap.cdap
@@ -262,7 +287,7 @@
org.apache.commons
commons-lang3
- 3.9
+ 3.18.0
com.google.guava