From 5615be088db05a724fbb8891ba6b7c4f03f62298 Mon Sep 17 00:00:00 2001 From: Deepika Lalwani Date: Tue, 17 Nov 2026 05:37:02 +0000 Subject: [PATCH] Only offer gzip and deflate in the HTTP response compressor NettyHttpService added HttpContentCompressor with the no-arg constructor. In newer Netty 4.1.x releases that constructor also turns on zstd (and brotli) whenever those libraries are on the classpath. When an older zstd-jni comes first on the classpath (for example zstd-jni 1.4.9-1 from a Hadoop 3.3.x runtime), creating the compressor fails with NoSuchMethodError: Zstd.defaultCompressionLevel(), and every channel initialization in the service fails. CDAP hits this on Dataproc after moving to Netty 4.1.137.Final. Build the compressor with explicit gzip and deflate options instead, so the available encodings no longer depend on what else is on the classpath. gzip and deflate are what the service offered before. Testing: HttpServerTest and HttpsServerTest (102 tests) pass, including the new testCompressResponseOnlyGzipOrDeflate. A standalone check with Netty 4.1.137.Final plus zstd-jni 1.4.9-1 on the classpath reproduces the NoSuchMethodError with the old constructor and passes with the new one. --- .../java/io/cdap/http/NettyHttpService.java | 8 +++++++- .../java/io/cdap/http/HttpServerTest.java | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cdap/http/NettyHttpService.java b/src/main/java/io/cdap/http/NettyHttpService.java index 6b58ad7..6b3faf6 100644 --- a/src/main/java/io/cdap/http/NettyHttpService.java +++ b/src/main/java/io/cdap/http/NettyHttpService.java @@ -34,6 +34,7 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.compression.StandardCompressionOptions; import io.netty.handler.codec.http.HttpContentCompressor; import io.netty.handler.codec.http.HttpServerCodec; import io.netty.handler.codec.http.HttpServerKeepAliveHandler; @@ -416,7 +417,12 @@ protected void initChannel(SocketChannel ch) throws Exception { pipeline.addLast("ssl", sslHandlerFactory.create(ch.alloc())); } pipeline.addLast("codec", new HttpServerCodec()); - pipeline.addLast("compressor", new HttpContentCompressor()); + // Only offer gzip and deflate. The no-arg constructor in newer Netty versions also turns on + // zstd (and brotli) whenever those libraries are on the classpath, and fails with a + // NoSuchMethodError when an older zstd-jni is loaded first (for example from a Hadoop + // or Spark runtime classpath). + pipeline.addLast("compressor", new HttpContentCompressor(StandardCompressionOptions.gzip(), + StandardCompressionOptions.deflate())); pipeline.addLast("chunkedWriter", new ChunkedWriteHandler()); pipeline.addLast("keepAlive", new HttpServerKeepAliveHandler()); pipeline.addLast("router", new RequestRouter(resourceHandler, httpChunkLimit, sslHandlerFactory != null)); diff --git a/src/test/java/io/cdap/http/HttpServerTest.java b/src/test/java/io/cdap/http/HttpServerTest.java index 8d33345..f03b61c 100644 --- a/src/test/java/io/cdap/http/HttpServerTest.java +++ b/src/test/java/io/cdap/http/HttpServerTest.java @@ -884,6 +884,25 @@ public void testCompressResponse() throws Exception { Assert.assertEquals("Testing message chunk", getContent(urlConn)); } + @Test + public void testCompressResponseOnlyGzipOrDeflate() throws Exception { + // Encodings other than gzip and deflate are not offered, so the response is not compressed. + HttpURLConnection urlConn = request("/test/v1/compressResponse?message=Testing+message", HttpMethod.GET); + urlConn.setRequestProperty(HttpHeaderNames.ACCEPT_ENCODING.toString(), "zstd, br, snappy"); + + Assert.assertEquals(HttpResponseStatus.OK.code(), urlConn.getResponseCode()); + Assert.assertNull(urlConn.getHeaderField(HttpHeaderNames.CONTENT_ENCODING.toString())); + Assert.assertEquals("Testing message", getContent(urlConn)); + + // When gzip is also accepted, gzip is used. + urlConn = request("/test/v1/compressResponse?message=Testing+message", HttpMethod.GET); + urlConn.setRequestProperty(HttpHeaderNames.ACCEPT_ENCODING.toString(), "zstd, br, gzip"); + + Assert.assertEquals(HttpResponseStatus.OK.code(), urlConn.getResponseCode()); + Assert.assertEquals("gzip", urlConn.getHeaderField(HttpHeaderNames.CONTENT_ENCODING.toString())); + Assert.assertEquals("Testing message", getContent(urlConn)); + } + @Test public void testContinueHandler() throws Exception { // Send two request with BodyConsumer handling with keep-alive and expect continue header