Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion kafka-plugins-client/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,22 @@
</exclusion>
</exclusions>
</dependency>
<!--
commons-compress 1.26.x (via avro) needs commons-io 2.12+ (e.g. CloseShieldInputStream.wrap in the
bzip2 codec). Without a direct dependency Maven mediates to the commons-io 2.4 of the provided
hadoop-common, which is not packaged in the plugin jar.
-->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.16.1</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<version>${kafka.version}</version>
<!-- Embedded Kafka broker for unit tests only; plugin code uses kafka-clients. -->
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
Expand Down Expand Up @@ -203,11 +215,13 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.http</groupId>
<artifactId>netty-http</artifactId>
<version>${netty-http.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down Expand Up @@ -253,7 +267,7 @@
<instructions>
<_exportcontents>io.cdap.plugin.kafka.*;org.apache.spark.streaming.kafka010.*;
org.apache.kafka.*;com.google.common.base.*;</_exportcontents>
<Embed-Dependency>*;inline=false;scope=compile</Embed-Dependency>
<Embed-Dependency>*;inline=false;scope=compile|runtime</Embed-Dependency>
<Embed-Transitive>true</Embed-Transitive>
<Embed-Directory>lib</Embed-Directory>
</instructions>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ public void initialize(AlertPublisherContext context) throws Exception {
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
props.put("producer.type", "sync");
// Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the
// cluster. Keep the 2.x behavior; users can still turn it on through producerProperties.
props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false");

// Override any property set above with user specified producer properties
for (Map.Entry<String, String> producerProperty : config.getProducerProperties().entrySet()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;


/**
Expand Down Expand Up @@ -92,11 +93,24 @@ public KafkaMessage getNext(KafkaKey kafkaKey) {
ByteBuffer payload = value == null ? ByteBuffer.wrap(EMPTY_BYTE_ARRAY) : ByteBuffer.wrap(value);

kafkaKey.set(currentOffset, consumerRecord.offset() + 1,
consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), consumerRecord.checksum());
consumerRecord.serializedKeySize() + consumerRecord.serializedValueSize(), checksum(consumerRecord));
currentOffset = consumerRecord.offset() + 1; // increase offset
return new KafkaMessage(payload, key);
}

/**
* Returns a checksum of the record's timestamp and serialized key and value sizes. This is the same partial
* checksum that {@code ConsumerRecord.checksum()} returned for v2 records before it was removed in Kafka 3.0
* (computed with CRC32 instead of CRC32C). It is only used to order and identify {@link KafkaKey}s.
*/
private static long checksum(ConsumerRecord<byte[], byte[]> record) {
ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + 2 * Integer.BYTES);
buffer.putLong(record.timestamp()).putInt(record.serializedKeySize()).putInt(record.serializedValueSize());
CRC32 crc = new CRC32();
crc.update(buffer.array());
return crc.getValue();
}
Comment on lines +106 to +112

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Allocating a ByteBuffer and a CRC32 instance for every single record can create significant garbage collection pressure in high-throughput environments. We can optimize this by updating the CRC32 instance directly using bit-shifting, which avoids allocating the ByteBuffer entirely.

  private static long checksum(ConsumerRecord<byte[], byte[]> record) {
    CRC32 crc = new CRC32();
    long t = record.timestamp();
    crc.update((int) (t >>> 56));
    crc.update((int) (t >>> 48));
    crc.update((int) (t >>> 40));
    crc.update((int) (t >>> 32));
    crc.update((int) (t >>> 24));
    crc.update((int) (t >>> 16));
    crc.update((int) (t >>> 8));
    crc.update((int) t);

    int keySize = record.serializedKeySize();
    crc.update(keySize >>> 24);
    crc.update(keySize >>> 16);
    crc.update(keySize >>> 8);
    crc.update(keySize);

    int valSize = record.serializedValueSize();
    crc.update(valSize >>> 24);
    crc.update(valSize >>> 16);
    crc.update(valSize >>> 8);
    crc.update(valSize);

    return crc.getValue();
  }


/**
* Fetch messages from Kafka.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,9 @@ private static class KafkaOutputFormatProvider implements OutputFormatProvider {
conf.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());

KafkaHelpers.setupKerberosLogin(conf, kafkaSinkConfig.principal, kafkaSinkConfig.keytabLocation);
// Kafka clients 3.0+ enable idempotence by default, which needs acks=all and IDEMPOTENT_WRITE on the
// cluster. Keep the 2.x behavior; users can still turn it on through kafkaProperties.
conf.put("additional." + ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false");
addKafkaProperties(kafkaSinkConfig.kafkaProperties);

conf.put(ASYNC, kafkaSinkConfig.async);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.requests.ListOffsetRequest;
import org.apache.kafka.common.requests.ListOffsetsRequest;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
Expand Down Expand Up @@ -118,9 +118,9 @@ static JavaInputDStream<ConsumerRecord<byte[], byte[]>> getConsumerRecordJavaDSt
for (Map.Entry<TopicPartition, Long> entry : offsets.entrySet()) {
TopicPartition topicAndPartition = entry.getKey();
Long offset = entry.getValue();
if (offset == ListOffsetRequest.EARLIEST_TIMESTAMP) {
if (offset == ListOffsetsRequest.EARLIEST_TIMESTAMP) {
earliestOffsetRequest.add(topicAndPartition);
} else if (offset == ListOffsetRequest.LATEST_TIMESTAMP) {
} else if (offset == ListOffsetsRequest.LATEST_TIMESTAMP) {
latestOffsetRequest.add(topicAndPartition);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,20 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.util.concurrent.AbstractIdleService;
import kafka.metrics.KafkaMetricsReporter;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
import kafka.zookeeper.ZooKeeperClientExpiredException;
import kafka.zookeeper.ZooKeeperClientTimeoutException;
import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.utils.Time;
import org.apache.twill.internal.utils.Networks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Option;
import scala.collection.JavaConverters;
import scala.collection.Seq;

import java.net.BindException;
import java.util.Collections;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;

/**
* A {@link com.google.common.util.concurrent.Service} implementation for running an instance of Kafka server in
Expand Down Expand Up @@ -84,7 +78,7 @@ protected void startUp() throws Exception {
LOG.warn("Session expired when connecting to ZooKeeper from KafkaServer. Attempt number {}.",
tries, rootCause);
} else if (rootCause instanceof BindException) {
LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause);
LOG.warn("Kafka failed to bind to {}. Attempt number {}.", kafkaConfig.listeners(), tries, rootCause);
} else {
LOG.error("Exception when connecting to Zookeeper", e);
throw e;
Expand All @@ -109,46 +103,7 @@ protected void shutDown() throws Exception {
}

private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
Seq<KafkaMetricsReporter> metricsReporters =
JavaConverters.collectionAsScalaIterableConverter(
Collections.<KafkaMetricsReporter>emptyList()).asScala().toSeq();
return new KafkaServer(kafkaConfig, new Time() {

@Override
public long milliseconds() {
return System.currentTimeMillis();
}

@Override
public long nanoseconds() {
return System.nanoTime();
}

@Override
public void sleep(long ms) {
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
Thread.interrupted();
}
}

@Override
public void waitObject(Object o, Supplier<Boolean> condition, long deadlineMs) throws InterruptedException {
while (!condition.get()) {
long currentTimeMs = this.milliseconds();
if (currentTimeMs >= deadlineMs) {
throw new TimeoutException("Condition not satisfied before deadline");
}
o.wait(deadlineMs - currentTimeMs);
}
}

@Override
public long hiResClockMs() {
return System.currentTimeMillis();
}
}, Option.apply("embedded-server"), metricsReporters);
return new KafkaServer(kafkaConfig, Time.SYSTEM, Option.apply("embedded-server"), false);
}

/**
Expand All @@ -163,8 +118,11 @@ private KafkaConfig createKafkaConfig(Properties properties) {
if (port == null || "0".equals(port)) {
int randomPort = Networks.getRandomPort();
Preconditions.checkState(randomPort > 0, "Failed to get random port.");
prop.setProperty("port", Integer.toString(randomPort));
port = Integer.toString(randomPort);
}
// Kafka 3.0 removed the "port" broker property; translate it to a PLAINTEXT listener.
prop.remove("port");
prop.setProperty("listeners", "PLAINTEXT://localhost:" + port);

return new KafkaConfig(prop);
}
Expand Down
33 changes: 29 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,42 @@
<spark3.version>3.1.2</spark3.version>
<widgets.dir>widgets</widgets.dir>
<docs.dir>docs</docs.dir>
<kafka.version>2.6.0</kafka.version>
<kafka.version>3.9.2</kafka.version>
<hadoop.version>2.6.0</hadoop.version>
<netty.version>4.1.75.Final</netty.version>
<netty-http.version>1.3.0</netty-http.version>
<avro.version>1.8.2</avro.version>
<avro.version>1.11.4</avro.version>
<main.basedir>${project.basedir}</main.basedir>
<logback.version>1.0.9</logback.version>
<slf4j.version>1.7.25</slf4j.version>
<jackson.version>2.10.5.1</jackson.version>
<jackson.version>2.18.12</jackson.version>

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The Jackson version 2.18.12 does not exist on Maven Central. The latest stable version in the 2.18.x line is 2.18.2. Please correct this version to prevent dependency resolution and build failures.

Suggested change
<jackson.version>2.18.12</jackson.version>
<jackson.version>2.18.2</jackson.version>

</properties>

<dependencyManagement>
<dependencies>
<!-- Align every Jackson 2 module (core, databind, annotations, dataformats, modules). -->
<dependency>
<groupId>com.fasterxml.jackson</groupId>
<artifactId>jackson-bom</artifactId>
<version>${jackson.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- avro 1.11.4 brings commons-compress 1.22; 1.26.2 fixes CVE-2024-25710 / CVE-2024-26308. -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.2</version>
</dependency>
<!-- kafka-clients 3.9.2 brings snappy-java 1.1.10.5; align with the cross-track target. -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.10.12</version>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<groupId>io.cdap.cdap</groupId>
Expand Down Expand Up @@ -262,7 +287,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.9</version>
<version>3.18.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down
Loading