From 7959981819018f55799969da1dca95e397fb1339 Mon Sep 17 00:00:00 2001 From: Sanket Sahu Date: Tue, 17 Nov 2026 14:49:52 +0000 Subject: [PATCH] Compile CDAP with --release 11 Move CDAP's compile target from Java 8 to Java 11 (--release 11) on develop (aligned with release/6.11): - Build & toolchain: set 11 in root pom.xml, the data-pipeline plugins archetype (6.12.0-SNAPSHOT), and user-guide plugin sample; bump maven-bundle-plugin to 5.1.9, unify maven-javadoc-plugin at 3.6.3 (doclint=none, release=11), replace findbugs-maven-plugin 3.0.5 with spotbugs-maven-plugin 4.8.6.4, add Temurin 11 setup in .github/workflows/build.yml, and disable the dist maven-jar-plugin jar execution on pom-packaging aggregator modules for Maven 3.9 install. - Bytecode rewriters: keep asm.version at 7.1 (managing asm, asm-commons, asm-tree, asm-analysis, and asm-util) and update all 37 Opcodes.ASM5 visitor sites to Opcodes.ASM7 so Java 11 NestHost and NestMembers attributes are handled cleanly without bumping ASM; add Java 11 nestmate coverage in ClassesTest and SparkRunnerClassLoaderTest. - DNS cache TTL (CDAP-20781): replace the direct sun.net.InetAddressCachePolicy import in AbstractProgramTwillRunnable with initializeDnsCachePolicy(), which sets the public networkaddress.cache.ttl security property to 30 before installing SecurityManager unless the user already set a TTL. - Guice 4.0:no_aop: switch CDAP modules to com.google.inject:guice:4.0:no_aop and exclude plain com.google.inject:guice on transitive dependencies so Guice error formatting on Java 11 (v55) classes does not fail inside Guice 4.0's shaded ASM 5 (without keeping an orphaned unclassified guice dependency entry). TODO(Guava-32): Upgrade guice.version to 5.1.0 and remove no_aop classifier and exclusions once Guava 32 lands. - Java 11+ launch checks: check for Java 11+ (including JEP 322 + delimiter strings and filtering "Picked up ..." lines) in RemoteExecutionTwillPreparer, functions.sh (fixing missing leading space before -Xlog:gc*), and cdap.bat (stopping cli and tx-debugger when the check fails). - EMR Java 11 JAXB & extension TCCL: package jaxb-api 2.3.1 and jaxb-runtime 2.3.8 in cdap-runtime-ext-emr and set the EMR extension classloader as thread context classloader in ElasticMapReduceProvisioner through one helper. - Java 11 platform classloader, AppClassLoader, & test compatibility: replace null parent classloaders in LogPipelineCheck, AbstractArtifactManager, and PackageFilterClassLoader with ClassLoader.getPlatformClassLoader(); fall back to java.class.path (shared ClassLoaders.parseJavaClassPath()) in ClassPath and ClassLoaders when encountering ClassLoader.getSystemClassLoader(); use per-call SimpleDateFormat via parseDateTime() in TimePartitionedFileSetTest and MapReduceWithPartitionedTest for JEP 252 CLDR compatibility; and add @PowerMockIgnore for java.xml in AutoInstallTest. --- .github/workflows/build.yml | 5 + .../artifact/AbstractArtifactManager.java | 4 +- .../AbstractProgramTwillRunnable.java | 32 +++-- .../remote/RemoteExecutionTwillPreparer.java | 20 ++- .../service/http/HttpHandlerGenerator.java | 10 +- .../artifact/AbstractArtifactManagerTest.java | 87 +++++++++++++ .../batch/MapReduceWithPartitionedTest.java | 9 +- .../AbstractProgramTwillRunnableTest.java | 100 ++++++++++++++ .../internal/capability/AutoInstallTest.java | 2 + .../cdap-etl/cdap-data-pipeline3_2.12/pom.xml | 2 +- .../cdap-etl/cdap-data-streams3_2.12/pom.xml | 4 +- .../resources/archetype-resources/pom.xml | 7 +- .../cdap-etl/cdap-etl-archetypes/pom.xml | 13 ++ .../cdap-etl/cdap-etl-batch/pom.xml | 4 +- cdap-app-templates/cdap-etl/pom.xml | 16 +++ .../cdap-program-report/pom.xml | 4 +- cdap-app-templates/pom.xml | 16 +++ cdap-cli/pom.xml | 1 + cdap-common/bin/functions.sh | 12 +- cdap-common/pom.xml | 1 + .../common/dataset/DatasetClassRewriter.java | 4 +- .../cdap/common/internal/guava/ClassPath.java | 35 +++-- .../cdap/cdap/common/lang/ClassLoaders.java | 64 +++++++-- .../common/lang/PackageFilterClassLoader.java | 5 +- .../common/security/AuthEnforceRewriter.java | 8 +- .../io/cdap/cdap/internal/asm/Classes.java | 4 +- .../common/guice/GuiceErrorMessageTest.java | 77 +++++++++++ .../cdap/common/lang/ClassLoaderTest.java | 55 ++++++++ .../cdap/cdap/internal/asm/ClassesTest.java | 82 ++++++++++++ .../TimePartitionedFileSetTest.java | 29 +++-- cdap-docs-gen/pom.xml | 1 + .../source/pipelines/plugins/pom.xml | 3 +- cdap-kms/pom.xml | 1 + .../cdap/master/startup/LogPipelineCheck.java | 3 +- .../master/startup/LogPipelineCheckTest.java | 35 +++++ cdap-runtime-ext-emr/pom.xml | 17 +++ .../emr/ElasticMapReduceProvisioner.java | 94 +++++++++----- .../emr/ElasticMapReduceProvisionerTest.java | 75 +++++++++++ .../AuthEnforceRewriterTest.java | 1 + .../spark/classloader/SparkClassRewriter.java | 48 +++---- .../SparkRunnerClassLoaderTest.java | 40 +++++- cdap-standalone/bin/cdap.bat | 46 +++++-- pom.xml | 122 +++++++++++++++--- 43 files changed, 1020 insertions(+), 178 deletions(-) create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManagerTest.java create mode 100644 cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnableTest.java create mode 100644 cdap-common/src/test/java/io/cdap/cdap/common/guice/GuiceErrorMessageTest.java create mode 100644 cdap-common/src/test/java/io/cdap/cdap/internal/asm/ClassesTest.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index beb08ed60978..f7fb1da2d5bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,6 +48,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' - name: Cache uses: actions/cache@v4 with: diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java index 4bc0ff98aa64..eb365f1e13db 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManager.java @@ -30,8 +30,6 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; import javax.annotation.Nullable; import org.apache.twill.filesystem.Location; @@ -48,7 +46,7 @@ protected AbstractArtifactManager(CConfiguration cConf) { File tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile(); this.tmpDir = DirUtils.createTempDir(tmpDir); - this.bootstrapClassLoader = new URLClassLoader(new URL[0], null); + this.bootstrapClassLoader = ClassLoader.getPlatformClassLoader(); } /** diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java index 6c48872f7224..257ae50f33db 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnable.java @@ -16,7 +16,9 @@ package io.cdap.cdap.internal.app.runtime.distributed; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.io.Closeables; import com.google.common.reflect.TypeToken; @@ -70,6 +72,7 @@ import java.net.ProxySelector; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.security.Security; import java.util.Arrays; import java.util.Collection; import java.util.Deque; @@ -96,7 +99,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; -import sun.net.InetAddressCachePolicy; /** * A {@link TwillRunnable} for running a program through a {@link ProgramRunner}. @@ -106,6 +108,10 @@ public abstract class AbstractProgramTwillRunnable implements TwillRunnable { + static final String NETWORKADDRESS_CACHE_TTL = "networkaddress.cache.ttl"; + static final String SUN_NET_INETADDR_TTL = "sun.net.inetaddr.ttl"; + static final String DEFAULT_DNS_CACHE_TTL_SECONDS = "30"; + private static final Logger LOG = LoggerFactory.getLogger(AbstractProgramTwillRunnable.class); private static final Gson GSON = ApplicationSpecificationAdapter.addTypeAdapters( new GsonBuilder()) @@ -163,6 +169,19 @@ public final void initialize(TwillContext context) { } } + /** + * Sets a default DNS cache TTL unless the user already set one. Without an explicit TTL, + * {@code InetAddressCachePolicy} caches DNS lookups forever once a {@link SecurityManager} is + * installed, so this must run before {@link System#setSecurityManager}. See CDAP-20781. + */ + @VisibleForTesting + static void initializeDnsCachePolicy() { + if (Strings.nullToEmpty(Security.getProperty(NETWORKADDRESS_CACHE_TTL)).trim().isEmpty() + && Strings.nullToEmpty(System.getProperty(SUN_NET_INETADDR_TTL)).trim().isEmpty()) { + Security.setProperty(NETWORKADDRESS_CACHE_TTL, DEFAULT_DNS_CACHE_TTL_SECONDS); + } + } + /** * Prepares this instance to execute a program. * @@ -173,15 +192,8 @@ private void doInitialize(File programOptionFile) throws Exception { controllerFuture = new CompletableFuture<>(); programCompletion = new CompletableFuture<>(); - // Make sure InetAddressCachePolicy is not set to FOREVER. - // This is needed because if InetAddressCachePolicy is loaded after System#setSecurityManager() call, - // caching policy is set to FOREVER. With FOREVER, caching policy, this twill runnable will not be able to - // reach out to other services within the cluster as IP is cached forever. - // Look at CDAP-20781 for more information. - if (InetAddressCachePolicy.get() == InetAddressCachePolicy.FOREVER) { - LOG.warn("InetAddress cache policy is set to FOREVER. This will ips to be cached forever. " - + "Ensure cache policy is not set to FOREVER."); - } + // Must run before installing the SecurityManager below (CDAP-20781). + initializeDnsCachePolicy(); // Setup process wide settings Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler()); diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java index 73612175c011..e70a912fe18b 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/remote/RemoteExecutionTwillPreparer.java @@ -465,9 +465,25 @@ private String generateLaunchScript( args.put(ClusterProperties.KERBEROS_KEYTAB, kerberosKeytab); } + scriptWriter.println("if [ -n \"${JAVA_HOME}\" ] && [ -x \"${JAVA_HOME}/bin/java\" ]; then"); + scriptWriter.println(" JAVA_CMD=\"${JAVA_HOME}/bin/java\""); + scriptWriter.println("else"); + scriptWriter.println(" JAVA_CMD=\"java\""); + scriptWriter.println("fi"); + scriptWriter.println("JAVA_VER_RAW=$(\"${JAVA_CMD}\" -version 2>&1" + + " | awk -F '\"' '!/^Picked up / && /version \"/ {print $2; exit}')"); + scriptWriter.println("JAVA_MAJOR=$(echo \"${JAVA_VER_RAW}\"" + + " | awk -F '[._+-]' '{if ($1 == \"1\") print $2; else print $1}')"); + scriptWriter.println("case \"${JAVA_MAJOR}\" in ''|*[!0-9]*) JAVA_MAJOR=0 ;; esac"); + scriptWriter.println("if [ \"${JAVA_MAJOR}\" -lt 11 ]; then"); + scriptWriter.println(" echo \"Java 11+ required on the cluster" + + " (found: ${JAVA_VER_RAW:-unknown})\" >&2"); + scriptWriter.println(" exit 1"); + scriptWriter.println("fi"); + scriptWriter.printf( - "nohup java -Djava.io.tmpdir=tmp -Dcdap.runid=%s -cp %s/%s -Xmx%dm %s %s '%s' true %s " - + ">%s/stdout 2>%s/stderr &\n", + "nohup \"${JAVA_CMD}\" -Djava.io.tmpdir=tmp -Dcdap.runid=%s -cp %s/%s -Xmx%dm %s %s '%s'" + + " true %s >%s/stdout 2>%s/stderr &\n", getProgramRunId().getRun(), targetPath, Constants.Files.LAUNCHER_JAR, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java index 614e2c29faf9..d7fb0c3bcbbc 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/service/http/HttpHandlerGenerator.java @@ -196,7 +196,7 @@ private void inspectHandler(final TypeToken delegateType, final TypeToken .getResourceAsStream(Type.getInternalName(rawType) + ".class") ) { ClassReader classReader = new ClassReader(sourceBytes); - classReader.accept(new ClassVisitor(Opcodes.ASM5) { + classReader.accept(new ClassVisitor(Opcodes.ASM7) { // Only need to visit @Path at the class level if we are inspecting the user handler class private final boolean inspectDelegate = delegateType.equals(inspectType); @@ -215,7 +215,7 @@ public AnnotationVisitor visitAnnotation(String desc, boolean visible) { if (inspectDelegate && type.equals(Type.getType(Path.class))) { visitedPath = true; AnnotationVisitor annotationVisitor = classWriter.visitAnnotation(desc, visible); - return new AnnotationVisitor(Opcodes.ASM5, annotationVisitor) { + return new AnnotationVisitor(Opcodes.ASM7, annotationVisitor) { @Override public void visit(String name, Object value) { // "value" is the key for the Path annotation string. @@ -374,7 +374,7 @@ private class HandlerMethodVisitor extends MethodVisitor { HandlerMethodVisitor(TypeToken delegateType, MethodVisitor mv, String desc, String signature, int access, String name, String[] exceptions, Type classType, ClassWriter classWriter, List> preservedClasses) { - super(Opcodes.ASM5, mv); + super(Opcodes.ASM7, mv); this.delegateType = delegateType; this.desc = desc; this.signature = signature; @@ -392,7 +392,7 @@ private class HandlerMethodVisitor extends MethodVisitor { public AnnotationVisitor visitAnnotation(String desc, boolean visible) { // Memorize all visible annotations if (visible) { - AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc); + AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM7, desc); annotations.add(annotationNode); return annotationNode; } @@ -404,7 +404,7 @@ public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, bo // Memorize all visible annotations for each parameter. // It needs to store in a Multimap because there can be multiple annotations per parameter. if (visible) { - AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc); + AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM7, desc); paramAnnotations.put(parameter, annotationNode); return annotationNode; } diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManagerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManagerTest.java new file mode 100644 index 000000000000..2f6ed76e3cd9 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/artifact/AbstractArtifactManagerTest.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime.artifact; + +import io.cdap.cdap.api.artifact.ArtifactClasses; +import io.cdap.cdap.api.artifact.ArtifactInfo; +import io.cdap.cdap.api.artifact.ArtifactScope; +import io.cdap.cdap.api.artifact.CloseableClassLoader; +import io.cdap.cdap.common.conf.CConfiguration; +import io.cdap.cdap.common.conf.Constants; +import io.cdap.cdap.common.test.AppJarHelper; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import org.apache.twill.filesystem.LocalLocationFactory; +import org.apache.twill.filesystem.Location; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Tests {@link AbstractArtifactManager} classloader behavior on Java 11. + */ +public class AbstractArtifactManagerTest { + + @ClassRule + public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder(); + + /** + * Standalone class packaged into the test artifact jar with no parent CDAP API dependencies. + */ + public static class SampleArtifactClass { + } + + @Test + public void testNullParentLoadsPlatformModules() throws Exception { + CConfiguration cConf = CConfiguration.create(); + cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath()); + + Location jarLocation = AppJarHelper.createDeploymentJar( + new LocalLocationFactory(TMP_FOLDER.newFolder()), SampleArtifactClass.class); + ArtifactInfo artifactInfo = new ArtifactInfo( + "sample-artifact", "1.0.0", ArtifactScope.USER, ArtifactClasses.builder().build(), + Collections.emptyMap()); + + AbstractArtifactManager manager = new AbstractArtifactManager(cConf) { + @Override + public List listArtifacts() { + return Collections.singletonList(artifactInfo); + } + + @Override + public List listArtifacts(String namespace) { + return listArtifacts(); + } + + @Override + protected Location getArtifactLocation(ArtifactInfo info, @Nullable String namespace) + throws IOException { + return jarLocation; + } + }; + + try (CloseableClassLoader classLoader = manager.createClassLoader(artifactInfo, null)) { + Assert.assertNotNull(classLoader.loadClass(SampleArtifactClass.class.getName())); + Assert.assertSame(Class.forName("java.sql.Driver"), classLoader.loadClass("java.sql.Driver")); + Assert.assertSame(Class.forName("org.ietf.jgss.GSSManager"), + classLoader.loadClass("org.ietf.jgss.GSSManager")); + } + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceWithPartitionedTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceWithPartitionedTest.java index e35189284c98..15fb4cab526f 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceWithPartitionedTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/batch/MapReduceWithPartitionedTest.java @@ -38,7 +38,8 @@ import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms; import io.cdap.cdap.internal.app.runtime.BasicArguments; import io.cdap.cdap.test.XSlowTests; -import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -57,7 +58,9 @@ @Category(XSlowTests.class) public class MapReduceWithPartitionedTest extends MapReduceRunnerTestBase { - static final DateFormat DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT, Locale.US); + private static long parseDateTime(String value) throws ParseException { + return new SimpleDateFormat("M/d/yy h:mm a", Locale.US).parse(value).getTime(); + } @Test @@ -75,7 +78,7 @@ public void apply() { } }); - final long time = DATE_FORMAT.parse("1/15/15 11:15 am").getTime(); + final long time = parseDateTime("1/15/15 11:15 am"); final long time5 = time + TimeUnit.MINUTES.toMillis(5); // run the partition writer m/r with this output partition time diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnableTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnableTest.java new file mode 100644 index 000000000000..73f4e1a34117 --- /dev/null +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/AbstractProgramTwillRunnableTest.java @@ -0,0 +1,100 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.app.runtime.distributed; + +import java.security.Security; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Unit tests for {@link AbstractProgramTwillRunnable#initializeDnsCachePolicy()}. + */ +public class AbstractProgramTwillRunnableTest { + + private String originalSecurityTtl; + private String originalSystemTtl; + + @Before + public void setUp() { + originalSecurityTtl = Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL); + originalSystemTtl = System.getProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + } + + @After + public void tearDown() { + Security.setProperty( + AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, + originalSecurityTtl == null ? "" : originalSecurityTtl); + if (originalSystemTtl == null) { + System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + } else { + System.setProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL, originalSystemTtl); + } + } + + @Test + public void testDefaultsToThirtySecondsWhenUnset() { + Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, ""); + System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + + AbstractProgramTwillRunnable.initializeDnsCachePolicy(); + Assert.assertEquals("30", + Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL)); + } + + @Test + public void testDefaultsWhenSecurityPropertyIsBlank() { + Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, " "); + System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + + AbstractProgramTwillRunnable.initializeDnsCachePolicy(); + Assert.assertEquals("30", + Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL)); + } + + @Test + public void testPreservesPreSetSecurityProperty() { + Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "60"); + System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + + AbstractProgramTwillRunnable.initializeDnsCachePolicy(); + Assert.assertEquals("60", + Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL)); + } + + @Test + public void testPreservesPreSetSystemProperty() { + Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, ""); + System.setProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL, "45"); + + AbstractProgramTwillRunnable.initializeDnsCachePolicy(); + Assert.assertEquals("", + Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL)); + } + + @Test + public void testPreservesExplicitForeverPolicy() { + Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "-1"); + System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL); + + AbstractProgramTwillRunnable.initializeDnsCachePolicy(); + Assert.assertEquals("-1", + Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL)); + } +} diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/AutoInstallTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/AutoInstallTest.java index 3fe91b0a9276..1c37b411672b 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/AutoInstallTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/capability/AutoInstallTest.java @@ -53,6 +53,7 @@ import org.junit.runners.BlockJUnit4ClassRunner; import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; @@ -62,6 +63,7 @@ */ @RunWith(PowerMockRunner.class) @PowerMockRunnerDelegate(BlockJUnit4ClassRunner.class) +@PowerMockIgnore({"javax.xml.*", "org.xml.*", "org.w3c.*", "com.sun.org.apache.xerces.*", "jdk.internal.reflect.*"}) @PrepareForTest({Files.class, HubPackage.class, HttpClients.class}) public class AutoInstallTest { diff --git a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml index 651f2c4c1e59..ad2ff6a4cec9 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-pipeline3_2.12/pom.xml @@ -170,7 +170,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 true diff --git a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml index 02d163cbd72e..552a5cb750b8 100644 --- a/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-data-streams3_2.12/pom.xml @@ -125,7 +125,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 true @@ -225,7 +225,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 org.apache.maven.plugins diff --git a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml index b0e32cf2e8a3..26176b52b84c 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/cdap-data-pipeline-plugins-archetype/src/main/resources/archetype-resources/pom.xml @@ -28,7 +28,7 @@ UTF-8 3.1.1 - 6.11.0-SNAPSHOT + 6.12.0-SNAPSHOT 2.3.0 widgets @@ -256,8 +256,7 @@ maven-compiler-plugin 3.7.0 - 1.8 - 1.8 + 11 @@ -288,7 +287,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 true diff --git a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml index 58697c87df9f..16ace5999282 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-archetypes/pom.xml @@ -61,6 +61,19 @@ + + + org.apache.maven.plugins + maven-jar-plugin + + + jar + false + none + + + + diff --git a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml index e9c300b786be..0cd6243045e3 100644 --- a/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml +++ b/cdap-app-templates/cdap-etl/cdap-etl-batch/pom.xml @@ -114,7 +114,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 true @@ -150,7 +150,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 diff --git a/cdap-app-templates/cdap-etl/pom.xml b/cdap-app-templates/cdap-etl/pom.xml index 8efc41db4b34..e4fa045b33c1 100644 --- a/cdap-app-templates/cdap-etl/pom.xml +++ b/cdap-app-templates/cdap-etl/pom.xml @@ -275,4 +275,20 @@ + + + + org.apache.maven.plugins + maven-jar-plugin + + + jar + false + none + + + + + + diff --git a/cdap-app-templates/cdap-program-report/pom.xml b/cdap-app-templates/cdap-program-report/pom.xml index 0208c5eb42dd..e4cf7b638a3c 100644 --- a/cdap-app-templates/cdap-program-report/pom.xml +++ b/cdap-app-templates/cdap-program-report/pom.xml @@ -282,7 +282,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 true @@ -316,7 +316,7 @@ org.apache.felix maven-bundle-plugin - 3.5.0 + 5.1.9 diff --git a/cdap-app-templates/pom.xml b/cdap-app-templates/pom.xml index 5adce66a6776..178a006a4070 100644 --- a/cdap-app-templates/pom.xml +++ b/cdap-app-templates/pom.xml @@ -35,4 +35,20 @@ cdap-program-report + + + + org.apache.maven.plugins + maven-jar-plugin + + + jar + false + none + + + + + + diff --git a/cdap-cli/pom.xml b/cdap-cli/pom.xml index 5a96df00d81d..98aad976731e 100644 --- a/cdap-cli/pom.xml +++ b/cdap-cli/pom.xml @@ -62,6 +62,7 @@ com.google.inject guice + no_aop jline diff --git a/cdap-common/bin/functions.sh b/cdap-common/bin/functions.sh index 1182829c879f..3b7f20082870 100644 --- a/cdap-common/bin/functions.sh +++ b/cdap-common/bin/functions.sh @@ -335,6 +335,7 @@ cdap_kinit() { # cdap_set_java () { local readonly __java __java_version + local __java_version_raw # Check JAVA_HOME, first if [[ -n ${JAVA_HOME} ]] && [[ -d ${JAVA_HOME} ]]; then __java="${JAVA_HOME}"/bin/java @@ -345,13 +346,16 @@ cdap_set_java () { die "JAVA_HOME is not set and 'java' was not found in your PATH. Please set JAVA_HOME to the location of your Java install" fi fi - __java_version_raw=$("${__java}" -version 2>&1 | awk -F '"' '/version/ {print $2}') - __java_version=$(echo "$__java_version_raw" | awk -F. '{if ($1 == "1") print $2; else print $1}') - if [[ -z ${__java_version} ]]; then + __java_version_raw=$("${__java}" -version 2>&1 | awk -F '"' '!/^Picked up / && /version "/ {print $2; exit}') + __java_version=$(echo "$__java_version_raw" | awk -F '[._+-]' '{if ($1 == "1") print $2; else print $1}') + if [[ -z ${__java_version} ]] || [[ ! "${__java_version}" =~ ^[0-9]+$ ]]; then die "Could not detect Java version. Aborting..." fi export JAVA_VERSION=${__java_version} export JAVA=${__java} + if [[ "${JAVA_VERSION}" -lt 11 ]]; then + die "Java 11+ is required (found Java ${JAVA_VERSION})" + fi return 0 } @@ -661,7 +665,7 @@ cdap_start_java() { if [ "$JAVA_VERSION" -le 8 ]; then __defines+=" -verbose:gc -Xloggc:${__gc_log_and_heapdump_dir}/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M" else - __defines+="-Xlog:gc*:file=${__gc_log_and_heapdump_dir}/gc.log:time,uptime,level,tags:filecount=10,filesize=1M" + __defines+=" -Xlog:gc*:file=${__gc_log_and_heapdump_dir}/gc.log:time,uptime,level,tags:filecount=10,filesize=1M" fi logecho "$(date) Starting CDAP ${__name} service on ${HOSTNAME}" echo diff --git a/cdap-common/pom.xml b/cdap-common/pom.xml index cc81058e854b..fea80b0c4e94 100644 --- a/cdap-common/pom.xml +++ b/cdap-common/pom.xml @@ -112,6 +112,7 @@ com.google.inject guice + no_aop com.google.inject.extensions diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java index a760a008cfd8..fbf93a5ae399 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/dataset/DatasetClassRewriter.java @@ -100,7 +100,7 @@ private final class DatasetClassVisitor extends ClassVisitor { private boolean interfaceClass; private DatasetClassVisitor(String className, ClassWriter cw) { - super(Opcodes.ASM5, cw); + super(Opcodes.ASM7, cw); this.datasetType = Type.getObjectType(className.replace('.', '/')); this.fields = new HashSet<>(); } @@ -137,7 +137,7 @@ public MethodVisitor visitMethod(int access, final String name, DATASET_RUNTIME_CONTEXT_TYPE.getDescriptor(), null, null); } - return new FinallyAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new FinallyAdapter(Opcodes.ASM7, mv, access, name, desc) { private boolean hasRead; private boolean hasWrite; diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java b/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java index 2f7d0e9c1fd9..89684badeff1 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/internal/guava/ClassPath.java @@ -21,6 +21,7 @@ import com.google.common.annotations.Beta; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.CharMatcher; +import com.google.common.base.Objects; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Splitter; @@ -32,6 +33,7 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.common.reflect.Reflection; +import io.cdap.cdap.common.lang.ClassLoaders; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; @@ -382,23 +384,32 @@ static ImmutableMap getClassPathEntries(ClassLoader classloade if (classloader instanceof URLClassLoader) { URLClassLoader urlClassLoader = (URLClassLoader) classloader; for (URL entry : urlClassLoader.getURLs()) { - URI uri; - try { - uri = entry.toURI(); - if (!predicate.apply(uri)) { - continue; - } - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } - if (!entries.containsKey(uri)) { - entries.put(uri, classloader); - } + addClassPathEntry(entries, entry, classloader, predicate); + } + } else if (Objects.equal(classloader, ClassLoader.getSystemClassLoader())) { + for (URL entry : ClassLoaders.parseJavaClassPath()) { + addClassPathEntry(entries, entry, classloader, predicate); } } return ImmutableMap.copyOf(entries); } + private static void addClassPathEntry(Map entries, URL entry, + ClassLoader classloader, Predicate predicate) { + URI uri; + try { + uri = entry.toURI(); + if (!predicate.apply(uri)) { + return; + } + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + if (!entries.containsKey(uri)) { + entries.put(uri, classloader); + } + } + @VisibleForTesting static final class Scanner { diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java index dfc46ccabd46..70d99f6005a2 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/ClassLoaders.java @@ -19,6 +19,7 @@ import com.google.common.base.Objects; import com.google.common.base.Splitter; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -27,6 +28,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; +import java.util.Arrays; import java.util.Collection; import java.util.Deque; import java.util.Iterator; @@ -51,6 +53,8 @@ public final class ClassLoaders { // Class-Path attributed in JAR are " " separated private static final Splitter CLASS_PATH_ATTR_SPLITTER = Splitter.on(" ").omitEmptyStrings(); + private static final Splitter JAVA_CLASS_PATH_SPLITTER = + Splitter.on(File.pathSeparator).omitEmptyStrings(); private ClassLoaders() { } @@ -173,13 +177,12 @@ public static > T getClassLoaderURLs(ClassLoad */ public static > T getClassLoaderURLs(ClassLoader classLoader, boolean childFirst, T urls) { - Deque classLoaders = collectURLClassLoaders(classLoader, new LinkedList<>()); + Deque> classLoaderURLs = collectClassLoaderURLs(classLoader, new LinkedList<>()); - Iterator iterator = - childFirst ? classLoaders.iterator() : classLoaders.descendingIterator(); + Iterator> iterator = + childFirst ? classLoaderURLs.iterator() : classLoaderURLs.descendingIterator(); while (iterator.hasNext()) { - URLClassLoader cl = iterator.next(); - for (URL url : cl.getURLs()) { + for (URL url : iterator.next()) { if (urls.add(url) && (url.getProtocol().equals("file"))) { addClassPathFromJar(url, urls); } @@ -190,16 +193,17 @@ public static > T getClassLoaderURLs(ClassLoad } /** - * Collects {@link URLClassLoader} along the {@link ClassLoader} chain. This method recognizes - * both {@link Delegator} and {@link CombineClassLoader} and will unfold the delegating - * classloaders inside it. The order of insertion + * Collects URL lists along the {@link ClassLoader} chain. This method recognizes both + * {@link Delegator} and {@link CombineClassLoader} and will unfold the delegating classloaders + * inside it. On Java 9+, {@link ClassLoader#getSystemClassLoader()} is not a + * {@link URLClassLoader}, so its URLs are resolved from {@code java.class.path}. * * @param classLoader the classloader to start the search from * @param result a collection for storing the result * @param type of the collection * @return the result collection */ - private static > T collectURLClassLoaders( + private static >> T collectClassLoaderURLs( ClassLoader classLoader, T result) { // Do BFS from the bottom of the ClassLoader chain @@ -227,7 +231,9 @@ private static > T collectURLClassL queue.addFirst((ClassLoader) delegate); } } else if (cl instanceof URLClassLoader) { - result.add((URLClassLoader) cl); + result.add(Arrays.asList(((URLClassLoader) cl).getURLs())); + } else if (Objects.equal(cl, ClassLoader.getSystemClassLoader())) { + result.add(parseJavaClassPath()); } if (cl.getParent() != null) { @@ -238,6 +244,44 @@ private static > T collectURLClassL return result; } + /** + * Returns the URLs listed in the {@code java.class.path} system property, in order. On Java 9+ + * this is how the URLs of {@link ClassLoader#getSystemClassLoader()} are found, since it is no + * longer a {@link URLClassLoader}. + */ + public static List parseJavaClassPath() { + ImmutableList.Builder urls = ImmutableList.builder(); + for (String entry : JAVA_CLASS_PATH_SPLITTER.split(System.getProperty("java.class.path", ""))) { + URL url = toClassPathUrl(entry); + if (url != null) { + urls.add(url); + } + } + return urls.build(); + } + + @Nullable + private static URL toClassPathUrl(String entry) { + File file = new File(entry); + try { + return file.toURI().toURL(); + } catch (SecurityException e) { + try { + String path = file.getAbsolutePath(); + if (file.isDirectory() && !path.endsWith("/")) { + path = path + "/"; + } + return new URL("file", null, path); + } catch (MalformedURLException | SecurityException malformed) { + LOG.warn("Malformed classpath entry: {}", entry, malformed); + return null; + } + } catch (MalformedURLException e) { + LOG.warn("Malformed classpath entry: {}", entry, e); + return null; + } + } + /** * Creates a {@link Function} that perform class name to class resource lookup. * diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/lang/PackageFilterClassLoader.java b/cdap-common/src/main/java/io/cdap/cdap/common/lang/PackageFilterClassLoader.java index b407d19b12de..2083b8781e20 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/lang/PackageFilterClassLoader.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/lang/PackageFilterClassLoader.java @@ -43,9 +43,8 @@ public class PackageFilterClassLoader extends ClassLoader implements Closeable { public PackageFilterClassLoader(ClassLoader parent, Predicate predicate) { super(parent); this.predicate = predicate; - // There is no reliable way to get bootstrap ClassLoader from Java (System.class.getClassLoader() may return null). - // A URLClassLoader with no URLs and with a null parent will load class from bootstrap ClassLoader only. - this.bootstrapClassLoader = new URLClassLoader(new URL[0], null); + // Delegate to the platform classloader so Java 11 platform modules (e.g. java.sql) stay visible. + this.bootstrapClassLoader = new URLClassLoader(new URL[0], ClassLoader.getPlatformClassLoader()); } @Override diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java b/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java index bf29bada05ed..4ae9e6d540b0 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/security/AuthEnforceRewriter.java @@ -182,7 +182,7 @@ private static final class AuthEnforceAnnotationVisitor extends ClassVisitor { private boolean interfaceClass; AuthEnforceAnnotationVisitor(String className) { - super(Opcodes.ASM5); + super(Opcodes.ASM7); this.className = className; this.methodAnnotations = new HashMap<>(); this.fieldDetails = new HashMap<>(); @@ -221,7 +221,7 @@ public MethodVisitor visitMethod(final int access, final String methodName, // AuthEnforce annotation details and also sets a boolean flag hasEnforce which is used later in // visitParameterAnnotation to visit the annotations on parameters of this method only if it had AuthEnforce // annotation. - return new MethodVisitor(Opcodes.ASM5) { + return new MethodVisitor(Opcodes.ASM7) { final Map parameterDetails = new HashMap<>(); AnnotationNode authEnforceAnnotationNode; @@ -388,7 +388,7 @@ private final class AuthEnforceAnnotationRewriter extends ClassVisitor { AuthEnforceAnnotationRewriter(String className, ClassWriter cw, Map fieldDetails, Map methodAnnotations) { - super(Opcodes.ASM5, cw); + super(Opcodes.ASM7, cw); this.className = className; this.classType = Type.getObjectType(className.replace(".", "/")); this.methodAnnotations = methodAnnotations; @@ -409,7 +409,7 @@ public MethodVisitor visitMethod(int access, final String methodName, final Stri if (annotationDetail == null) { return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, methodName, methodDesc) { + return new AdviceAdapter(Opcodes.ASM7, mv, access, methodName, methodDesc) { @Override protected void onMethodEnter() { LOG.trace( diff --git a/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java b/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java index 009fdaba1af5..5805e6ce63a2 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java +++ b/cdap-common/src/main/java/io/cdap/cdap/internal/asm/Classes.java @@ -118,7 +118,7 @@ public static byte[] rewriteMethodToNoop(final String className, InputStream byteCodeStream, final Set methods) throws IOException { ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(final int access, final String name, final String desc, String signature, String[] exceptions) { @@ -141,7 +141,7 @@ public MethodVisitor visitMethod(final int access, final String name, final Stri // VisitMaxs with 0 so that COMPUTE_MAXS from ClassWriter will compute the right values. adapter.visitMaxs(0, 0); - return new MethodVisitor(Opcodes.ASM5) { + return new MethodVisitor(Opcodes.ASM7) { }; } }, 0); diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/guice/GuiceErrorMessageTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/guice/GuiceErrorMessageTest.java new file mode 100644 index 000000000000..41858ecec5fd --- /dev/null +++ b/cdap-common/src/test/java/io/cdap/cdap/common/guice/GuiceErrorMessageTest.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.common.guice; + +import com.google.inject.ConfigurationException; +import com.google.inject.Guice; +import com.google.inject.Inject; +import com.google.inject.Injector; +import java.io.DataInputStream; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import org.junit.Assert; +import org.junit.Test; + +/** + * Verifies that Guice error formatting and stack traces succeed on Java 11 (class file version 55) + * CDAP classes without failing in Guice 4.0's shaded ASM5 {@code LineNumbers}. + */ +public class GuiceErrorMessageTest { + + private interface UnboundDependency { + } + + private static final class UnboundConsumer { + private final UnboundDependency dependency; + + @Inject + UnboundConsumer(UnboundDependency dependency) { + this.dependency = dependency; + } + } + + @Test + public void testConfigurationExceptionFormattingOnJava11Class() throws Exception { + String resourcePath = UnboundConsumer.class.getName().replace('.', '/') + ".class"; + try (InputStream in = UnboundConsumer.class.getClassLoader().getResourceAsStream(resourcePath); + DataInputStream dataIn = new DataInputStream(in)) { + Assert.assertNotNull("Expected class resource for " + resourcePath, in); + Assert.assertEquals(0xCAFEBABE, dataIn.readInt()); + dataIn.readUnsignedShort(); // minor version + int majorVersion = dataIn.readUnsignedShort(); + Assert.assertEquals("Expected Java 11 (v55) bytecode", 55, majorVersion); + } + + Injector injector = Guice.createInjector(); + try { + injector.getInstance(UnboundConsumer.class); + Assert.fail("Expected ConfigurationException for unbound dependency"); + } catch (ConfigurationException expected) { + String message = expected.getMessage(); + Assert.assertNotNull(message); + Assert.assertTrue("Unexpected Guice message: " + message, + message.contains("No implementation for " + UnboundDependency.class.getName())); + + StringWriter sw = new StringWriter(); + try (PrintWriter pw = new PrintWriter(sw)) { + expected.printStackTrace(pw); + } + Assert.assertTrue(sw.toString().contains(ConfigurationException.class.getName())); + } + } +} diff --git a/cdap-common/src/test/java/io/cdap/cdap/common/lang/ClassLoaderTest.java b/cdap-common/src/test/java/io/cdap/cdap/common/lang/ClassLoaderTest.java index 1d81117214f1..cf485d72fcc4 100644 --- a/cdap-common/src/test/java/io/cdap/cdap/common/lang/ClassLoaderTest.java +++ b/cdap-common/src/test/java/io/cdap/cdap/common/lang/ClassLoaderTest.java @@ -30,6 +30,7 @@ import io.cdap.cdap.api.app.ApplicationConfigurer; import io.cdap.cdap.api.common.Bytes; import io.cdap.cdap.common.app.MainClassLoader; +import io.cdap.cdap.common.internal.guava.ClassPath; import io.cdap.cdap.common.io.Locations; import io.cdap.cdap.common.lang.jar.BundleJarUtil; import io.cdap.cdap.common.test.AppJarHelper; @@ -39,7 +40,9 @@ import java.io.InputStream; import java.lang.reflect.Method; import java.net.URL; +import java.net.URLClassLoader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.Executor; @@ -90,6 +93,12 @@ public void testPackageFilter() throws ClassNotFoundException { // Load class resource that is in allowed package. Should return null. resourceName = SchemaGenerator.class.getName().replace('.', '/') + ".class"; Assert.assertNull(classLoader.getResource(resourceName)); + + // Java 11 platform modules moved out of bootstrap must still be loadable. + Assert.assertSame(Class.forName("java.sql.Driver"), classLoader.loadClass("java.sql.Driver")); + Assert.assertSame(Class.forName("org.ietf.jgss.GSSManager"), + classLoader.loadClass("org.ietf.jgss.GSSManager")); + Assert.assertNotNull(classLoader.getResource("java/sql/Driver.class")); } @Test @@ -255,4 +264,50 @@ public void testMoreExecutorsRewrite() throws Exception { Assert.assertEquals(currentThreadName, executorThreadName.get()); } + + @Test + public void testParseJavaClassPath() throws Exception { + File dir = TMP_FOLDER.newFolder(); + File jar = TMP_FOLDER.newFile("parse-java-class-path.jar"); + String originalClassPath = System.getProperty("java.class.path"); + try { + // Empty segments are skipped and the order is kept + System.setProperty("java.class.path", File.pathSeparator + dir.getAbsolutePath() + + File.pathSeparator + File.pathSeparator + jar.getAbsolutePath() + File.pathSeparator); + Assert.assertEquals(Arrays.asList(dir.toURI().toURL(), jar.toURI().toURL()), + ClassLoaders.parseJavaClassPath()); + + System.setProperty("java.class.path", ""); + Assert.assertEquals(Collections.emptyList(), ClassLoaders.parseJavaClassPath()); + } finally { + System.setProperty("java.class.path", originalClassPath); + } + } + + @Test + public void testGetClassLoaderURLsIncludesSystemClassPath() throws Exception { + List systemUrls = ClassLoaders.parseJavaClassPath(); + Assert.assertFalse(systemUrls.isEmpty()); + URL systemUrl = systemUrls.get(0); + URL childUrl = TMP_FOLDER.newFolder().toURI().toURL(); + + try (URLClassLoader child = + new URLClassLoader(new URL[]{childUrl}, ClassLoader.getSystemClassLoader())) { + List childFirst = ClassLoaders.getClassLoaderURLs(child, true, new ArrayList<>()); + Assert.assertEquals(childUrl, childFirst.get(0)); + Assert.assertTrue(childFirst.indexOf(systemUrl) > 0); + + List parentFirst = ClassLoaders.getClassLoaderURLs(child, false, new ArrayList<>()); + Assert.assertTrue(parentFirst.indexOf(systemUrl) >= 0); + Assert.assertTrue(parentFirst.indexOf(systemUrl) < parentFirst.indexOf(childUrl)); + } + } + + @Test + public void testClassPathIncludesSystemClassPathOnJava11() throws IOException { + ClassPath classPath = ClassPath.from(ClassLoader.getSystemClassLoader()); + boolean foundSelf = classPath.getAllClasses().stream() + .anyMatch(info -> ClassLoaderTest.class.getName().equals(info.getName())); + Assert.assertTrue(foundSelf); + } } diff --git a/cdap-common/src/test/java/io/cdap/cdap/internal/asm/ClassesTest.java b/cdap-common/src/test/java/io/cdap/cdap/internal/asm/ClassesTest.java new file mode 100644 index 000000000000..0181a18de1df --- /dev/null +++ b/cdap-common/src/test/java/io/cdap/cdap/internal/asm/ClassesTest.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2026 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.cdap.internal.asm; + +import java.io.InputStream; +import java.util.Collections; +import java.util.concurrent.Callable; +import org.junit.Assert; +import org.junit.Test; +import org.objectweb.asm.Type; + +/** + * Tests {@link Classes} bytecode helpers on Java 11 classes with nestmate attributes. + */ +public class ClassesTest { + + /** + * Sample Java 11 nest host with nested and anonymous classes for ASM7 rewrite tests. + */ + public static class SampleNestHost { + + private final String secret = "nest-secret"; + + public void shouldBeNoop() { + throw new IllegalStateException("Method should have been rewritten to no-op"); + } + + public String callNestedPrivateAccess() throws Exception { + return new NestedMember().readSecret(); + } + + /** + * Nested member accessing private field of {@link SampleNestHost} via JEP 181 nestmates. + */ + public class NestedMember implements Callable { + + @Override + public String call() { + return secret; + } + + private String readSecret() { + return call(); + } + } + } + + @Test + public void testRewriteMethodToNoopOnJava11Nestmates() throws Exception { + ByteCodeClassLoader classLoader = new ByteCodeClassLoader(getClass().getClassLoader()); + for (Class cls : new Class[]{ + ClassesTest.class, SampleNestHost.class, SampleNestHost.NestedMember.class}) { + String resource = cls.getName().replace('.', '/') + ".class"; + try (InputStream is = getClass().getClassLoader().getResourceAsStream(resource)) { + Assert.assertNotNull(is); + byte[] rewritten = Classes.rewriteMethodToNoop( + cls.getName(), is, Collections.singleton("shouldBeNoop")); + classLoader.addClass(new ClassDefinition(rewritten, Type.getInternalName(cls))); + } + } + + Class hostClass = classLoader.loadClass(SampleNestHost.class.getName()); + Object host = hostClass.getDeclaredConstructor().newInstance(); + hostClass.getMethod("shouldBeNoop").invoke(host); + Assert.assertEquals("nest-secret", + hostClass.getMethod("callNestedPrivateAccess").invoke(host)); + } +} diff --git a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/partitioned/TimePartitionedFileSetTest.java b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/partitioned/TimePartitionedFileSetTest.java index 984c8eb06995..1016eeb2805f 100644 --- a/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/partitioned/TimePartitionedFileSetTest.java +++ b/cdap-data-fabric/src/test/java/io/cdap/cdap/data2/dataset2/lib/partitioned/TimePartitionedFileSetTest.java @@ -33,11 +33,11 @@ import io.cdap.cdap.proto.id.DatasetId; import io.cdap.cdap.security.spi.authorization.UnauthorizedException; import java.io.IOException; -import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Calendar; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; @@ -60,13 +60,16 @@ public class TimePartitionedFileSetTest { @ClassRule public static DatasetFrameworkTestUtil dsFrameworkUtil = new DatasetFrameworkTestUtil(); - static final DateFormat DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT, Locale.US); static final long MINUTE = TimeUnit.MINUTES.toMillis(1); static final long HOUR = TimeUnit.HOURS.toMillis(1); static final long MAX = Long.MAX_VALUE; private static final DatasetId TPFS_INSTANCE = DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("tpfs"); + private static long parseDateTime(String value) throws ParseException { + return new SimpleDateFormat("M/d/yy h:mm a", Locale.US).parse(value).getTime(); + } + @Before public void before() throws Exception { dsFrameworkUtil.createInstance("timePartitionedFileSet", TPFS_INSTANCE, FileSetProperties.builder() @@ -88,8 +91,7 @@ public void apply() throws Exception { // make sure the dataset has no partitions validateTimePartitions(tpfs, 0L, MAX, Collections.emptyMap()); - Date date = DATE_FORMAT.parse("6/4/12 10:00 am"); - long time = date.getTime(); + long time = parseDateTime("6/4/12 10:00 am"); // keep track of all the metadata added Map allMetadata = Maps.newHashMap(); @@ -147,7 +149,7 @@ public void testAddGetPartitions() throws Exception { @Override public void apply() throws Exception { // this is an arbitrary data to use as the test time - long time = DATE_FORMAT.parse("12/10/14 5:10 am").getTime(); + long time = parseDateTime("12/10/14 5:10 am"); long time2 = time + HOUR; String firstPath = "first/partition"; String secondPath = "second/partition"; @@ -220,9 +222,9 @@ public void apply() throws Exception { @Test public void testOutputPartitionPath() throws Exception { // test specifying output time - Date date = DATE_FORMAT.parse("1/1/15 8:42 pm"); + long outputTime = parseDateTime("1/1/15 8:42 pm"); Map args = Maps.newHashMap(); - TimePartitionedFileSetArguments.setOutputPartitionTime(args, date.getTime()); + TimePartitionedFileSetArguments.setOutputPartitionTime(args, outputTime); TimeZone timeZone = Calendar.getInstance().getTimeZone(); TimePartitionedFileSetArguments.setOutputPathFormat(args, "yyyy-MM-dd/HH_mm", timeZone.getID()); TimePartitionedFileSet ds = dsFrameworkUtil.getInstance(TPFS_INSTANCE, args); @@ -278,8 +280,7 @@ public void apply() throws Exception { } }); - Date date = DATE_FORMAT.parse("6/4/12 10:00 am"); - final long time = date.getTime(); + final long time = parseDateTime("6/4/12 10:00 am"); txnl.execute(new TransactionExecutor.Subroutine() { @Override public void apply() throws Exception { @@ -337,8 +338,8 @@ public void apply() throws Exception { public void testPartitionsForTimeRange() throws Exception { for (Object[] test : rangeTests) { try { - long start = test[0] instanceof Long ? (Long) test[0] : DATE_FORMAT.parse((String) test[0]).getTime(); - long stop = test[1] instanceof Long ? (Long) test[1] : DATE_FORMAT.parse((String) test[1]).getTime(); + long start = test[0] instanceof Long ? (Long) test[0] : parseDateTime((String) test[0]); + long stop = test[1] instanceof Long ? (Long) test[1] : parseDateTime((String) test[1]); List filters = TimePartitionedFileSetDataset.partitionFiltersForTimeRange(start, stop); //Assert.assertEquals(test.length - 2, filters.size()); Set expectedSet = Sets.newHashSet(); @@ -500,8 +501,8 @@ private void validateTimePartitions(TimePartitionedFileSet dataset, @Test public void testTimePartitionedInputArguments() throws Exception { - final long time8 = DATE_FORMAT.parse("10/17/2014 8:42 am").getTime(); - final long time9 = DATE_FORMAT.parse("10/17/2014 9:42 am").getTime(); + final long time8 = parseDateTime("10/17/2014 8:42 am"); + final long time9 = parseDateTime("10/17/2014 9:42 am"); final String path8 = "8:42"; final String path9 = "9:42"; diff --git a/cdap-docs-gen/pom.xml b/cdap-docs-gen/pom.xml index 62cd7af2c9c8..31a0f90c6473 100644 --- a/cdap-docs-gen/pom.xml +++ b/cdap-docs-gen/pom.xml @@ -48,6 +48,7 @@ com.google.inject guice + no_aop diff --git a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml index ee782162c70c..fb8b0b6748c8 100644 --- a/cdap-docs/user-guide/source/pipelines/plugins/pom.xml +++ b/cdap-docs/user-guide/source/pipelines/plugins/pom.xml @@ -617,8 +617,7 @@ maven-compiler-plugin 3.7.0 - 1.8 - 1.8 + 11 diff --git a/cdap-kms/pom.xml b/cdap-kms/pom.xml index 2ce525c0dd5a..2dcc1d955373 100644 --- a/cdap-kms/pom.xml +++ b/cdap-kms/pom.xml @@ -56,6 +56,7 @@ com.google.inject guice + no_aop junit diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/startup/LogPipelineCheck.java b/cdap-master/src/main/java/io/cdap/cdap/master/startup/LogPipelineCheck.java index 9a3557a996a2..e102d8ef3c3c 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/startup/LogPipelineCheck.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/startup/LogPipelineCheck.java @@ -78,7 +78,8 @@ public void run() throws Exception { // cConf.clear(); // cConf.addResource(inputStream); // new LogPipelineLoader(cConf).validate(); - ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), null); + ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), + ClassLoader.getPlatformClassLoader()); Class cConfClass = classLoader.loadClass(CConfiguration.class.getName()); Object cConf = cConfClass.getMethod("create").invoke(null); diff --git a/cdap-master/src/test/java/io/cdap/cdap/master/startup/LogPipelineCheckTest.java b/cdap-master/src/test/java/io/cdap/cdap/master/startup/LogPipelineCheckTest.java index af1618bb1913..db3086221e5e 100644 --- a/cdap-master/src/test/java/io/cdap/cdap/master/startup/LogPipelineCheckTest.java +++ b/cdap-master/src/test/java/io/cdap/cdap/master/startup/LogPipelineCheckTest.java @@ -74,6 +74,41 @@ public void testValid() throws Exception { new LogPipelineCheck(cConf).run(); } + /** + * Test appender referencing Java 11 platform modules moved out of the bootstrap classloader. + */ + public static class PlatformCheckingAppender extends NoopAppender { + + public PlatformCheckingAppender() { + // Force resolution of java.sql and java.security.jgss classes when instantiated. + if (java.sql.Driver.class.getName().isEmpty() + || org.ietf.jgss.GSSManager.class.getName().isEmpty()) { + throw new IllegalStateException("Unexpected empty platform class name"); + } + } + } + + @Test + public void testPlatformClassLoaderAccess() throws Exception { + File libDir = TEMP_FOLDER.newFolder(); + String appenderClassName = "PlatformCheckingAppender2"; + try (JarOutputStream jarOutput = new JarOutputStream( + new FileOutputStream(new File(libDir, "lib.jar")))) { + jarOutput.putNextEntry(new JarEntry(appenderClassName + ".class")); + jarOutput.write(renameClass(PlatformCheckingAppender.class, appenderClassName)); + } + + File configDir = TEMP_FOLDER.newFolder(); + generateLogConfig("ALL", Collections.emptyMap(), + appenderClassName, new File(configDir, "test-platform.xml")); + + CConfiguration cConf = CConfiguration.create(); + cConf.set(Constants.Logging.PIPELINE_LIBRARY_DIR, libDir.getAbsolutePath()); + cConf.set(Constants.Logging.PIPELINE_CONFIG_DIR, configDir.getAbsolutePath()); + + new LogPipelineCheck(cConf).run(); + } + @Test (expected = InvalidPipelineException.class) public void testInvalid() throws Exception { // Sets an invalid config for the CDAPLogAppender diff --git a/cdap-runtime-ext-emr/pom.xml b/cdap-runtime-ext-emr/pom.xml index 12ca2f471464..6525611ac8e5 100644 --- a/cdap-runtime-ext-emr/pom.xml +++ b/cdap-runtime-ext-emr/pom.xml @@ -89,6 +89,23 @@ commons-cli ${commons.cli.version} + + + javax.xml.bind + jaxb-api + 2.3.1 + + + org.glassfish.jaxb + jaxb-runtime + 2.3.8 + + + jakarta.xml.bind + jakarta.xml.bind-api + + + junit junit diff --git a/cdap-runtime-ext-emr/src/main/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisioner.java b/cdap-runtime-ext-emr/src/main/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisioner.java index 56c560a2e83c..b0d1cfad6c59 100644 --- a/cdap-runtime-ext-emr/src/main/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisioner.java +++ b/cdap-runtime-ext-emr/src/main/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisioner.java @@ -58,55 +58,64 @@ public ProvisionerSpecification getSpec() { @Override public void validateProperties(Map properties) { - EMRConf.fromProperties(properties); + callWithExtensionClassLoader(() -> EMRConf.fromProperties(properties)); } @Override public Cluster createCluster(ProvisionerContext context) throws Exception { - // Generates and set the ssh key - SSHKeyPair sshKeyPair = context.getSSHContext().generate("ec2-user"); // or 'hadoop' - context.getSSHContext().setSSHKeyPair(sshKeyPair); - - EMRConf conf = EMRConf.fromProvisionerContext(context); - String clusterName = getClusterName(context.getProgramRunInfo()); - - try (EMRClient client = EMRClient.fromConf(conf)) { - // if it already exists, it means this is a retry. We can skip actually making the request - Optional existing = client.getUnterminatedClusterByName(clusterName); - if (existing.isPresent()) { - return client.getCluster(existing.get().getId()).get(); + return callWithExtensionClassLoader(() -> { + // Generates and set the ssh key + SSHKeyPair sshKeyPair = context.getSSHContext().generate("ec2-user"); // or 'hadoop' + context.getSSHContext().setSSHKeyPair(sshKeyPair); + + EMRConf conf = EMRConf.fromProvisionerContext(context); + String clusterName = getClusterName(context.getProgramRunInfo()); + + try (EMRClient client = EMRClient.fromConf(conf)) { + // if it already exists, it means this is a retry. We can skip actually making the request + Optional existing = client.getUnterminatedClusterByName(clusterName); + if (existing.isPresent()) { + return client.getCluster(existing.get().getId()).get(); + } + String clusterId = client.createCluster(clusterName); + return new Cluster(clusterId, ClusterStatus.CREATING, Collections.emptyList(), + Collections.emptyMap()); } - String clusterId = client.createCluster(clusterName); - return new Cluster(clusterId, ClusterStatus.CREATING, Collections.emptyList(), - Collections.emptyMap()); - } + }); } @Override public ClusterStatus getClusterStatus(ProvisionerContext context, Cluster cluster) throws Exception { - EMRConf conf = EMRConf.fromProvisionerContext(context); - try (EMRClient client = EMRClient.fromConf(conf)) { - return client.getClusterStatus(cluster.getName()); - } + return callWithExtensionClassLoader(() -> { + EMRConf conf = EMRConf.fromProvisionerContext(context); + try (EMRClient client = EMRClient.fromConf(conf)) { + return client.getClusterStatus(cluster.getName()); + } + }); } @Override public Cluster getClusterDetail(ProvisionerContext context, Cluster cluster) throws Exception { - EMRConf conf = EMRConf.fromProvisionerContext(context); - try (EMRClient client = EMRClient.fromConf(conf)) { - Optional existing = client.getCluster(cluster.getName()); - return existing.orElseGet(() -> new Cluster(cluster, ClusterStatus.NOT_EXISTS)); - } + return callWithExtensionClassLoader(() -> { + EMRConf conf = EMRConf.fromProvisionerContext(context); + try (EMRClient client = EMRClient.fromConf(conf)) { + Optional existing = client.getCluster(cluster.getName()); + return existing.orElseGet(() -> new Cluster(cluster, ClusterStatus.NOT_EXISTS)); + } + }); } @Override public void deleteCluster(ProvisionerContext context, Cluster cluster) throws Exception { - EMRConf conf = EMRConf.fromProvisionerContext(context); - try (EMRClient client = EMRClient.fromConf(conf)) { - client.deleteCluster(cluster.getName()); - } + callWithExtensionClassLoader(() -> { + EMRConf conf = EMRConf.fromProvisionerContext(context); + try (EMRClient client = EMRClient.fromConf(conf)) { + client.deleteCluster(cluster.getName()); + } + return null; + }); } @Override @@ -144,6 +153,31 @@ static String getClusterName(ProgramRunInfo programRunInfo) { return CLUSTER_PREFIX + cleanedAppName + "-" + programRunInfo.getRun(); } + /** + * Runs the call with this extension's classloader as the thread context classloader. On Java 11, + * JAXB is packaged inside this extension, and the AWS SDK looks it up through the context + * classloader. + */ + private static T callWithExtensionClassLoader( + ExtensionClassLoaderCall call) throws E { + Thread currentThread = Thread.currentThread(); + ClassLoader oldClassLoader = currentThread.getContextClassLoader(); + currentThread.setContextClassLoader(ElasticMapReduceProvisioner.class.getClassLoader()); + try { + return call.call(); + } finally { + currentThread.setContextClassLoader(oldClassLoader); + } + } + + /** + * A call that runs with the extension classloader as the thread context classloader. + */ + private interface ExtensionClassLoaderCall { + + T call() throws E; + } + @Override public Capabilities getCapabilities() { return new Capabilities( diff --git a/cdap-runtime-ext-emr/src/test/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisionerTest.java b/cdap-runtime-ext-emr/src/test/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisionerTest.java index 71e0522c5222..1bd612ec48d5 100644 --- a/cdap-runtime-ext-emr/src/test/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisionerTest.java +++ b/cdap-runtime-ext-emr/src/test/java/io/cdap/cdap/runtime/spi/provisioner/emr/ElasticMapReduceProvisionerTest.java @@ -16,8 +16,23 @@ package io.cdap.cdap.runtime.spi.provisioner.emr; +import com.amazonaws.Request; +import com.amazonaws.services.ec2.model.ImportKeyPairRequest; +import com.amazonaws.services.ec2.model.transform.ImportKeyPairRequestMarshaller; +import com.amazonaws.util.Base64; import io.cdap.cdap.runtime.spi.ProgramRunInfo; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; import org.junit.Assert; import org.junit.Test; @@ -26,6 +41,66 @@ */ public class ElasticMapReduceProvisionerTest { + @XmlRootElement(name = "keyPairPayload") + public static class SampleXmlPayload { + @XmlElement + public String keyName; + } + + @Test + public void testJaxbRuntimeAvailableOnJava11() throws Exception { + JAXBContext jaxbContext = JAXBContext.newInstance(SampleXmlPayload.class); + SampleXmlPayload payload = new SampleXmlPayload(); + payload.keyName = "cdap-emr-key"; + StringWriter writer = new StringWriter(); + jaxbContext.createMarshaller().marshal(payload, writer); + SampleXmlPayload roundTrip = (SampleXmlPayload) jaxbContext.createUnmarshaller() + .unmarshal(new StringReader(writer.toString())); + Assert.assertEquals("cdap-emr-key", roundTrip.keyName); + + // AWS SDK Base64 and EC2 request marshalling need JAXB on Java 11. + Assert.assertEquals("AQID", Base64.encodeAsString(new byte[]{1, 2, 3})); + Request request = new ImportKeyPairRequestMarshaller() + .marshall(new ImportKeyPairRequest("cdap-key", "ssh-rsa AAAAB3Nza")); + Assert.assertEquals("cdap-key", request.getParameters().get("KeyName").get(0)); + } + + @Test + public void testValidatePropertiesUsesExtensionClassLoader() throws Exception { + ElasticMapReduceProvisioner provisioner = new ElasticMapReduceProvisioner(); + ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader emptyClassLoader = new URLClassLoader(new URL[0], null)) { + Thread.currentThread().setContextClassLoader(emptyClassLoader); + + AtomicReference observedClassLoader = new AtomicReference<>(); + Map properties = new HashMap() { + @Override + public String get(Object key) { + observedClassLoader.set(Thread.currentThread().getContextClassLoader()); + return super.get(key); + } + }; + properties.put("accessKey", "ak"); + properties.put("secretKey", "sk"); + properties.put("ec2SubnetId", "subnet-1"); + properties.put("additionalMasterSecurityGroup", "sg-1"); + + provisioner.validateProperties(properties); + Assert.assertSame(ElasticMapReduceProvisioner.class.getClassLoader(), + observedClassLoader.get()); + Assert.assertSame(emptyClassLoader, Thread.currentThread().getContextClassLoader()); + + try { + provisioner.validateProperties(Collections.emptyMap()); + Assert.fail("Expected IllegalArgumentException for missing properties"); + } catch (IllegalArgumentException expected) { + Assert.assertSame(emptyClassLoader, Thread.currentThread().getContextClassLoader()); + } + } finally { + Thread.currentThread().setContextClassLoader(oldClassLoader); + } + } + @Test public void testClusterName() { // test basic diff --git a/cdap-security/src/test/java/io/cdap/cdap/security/authorization/AuthEnforceRewriterTest.java b/cdap-security/src/test/java/io/cdap/cdap/security/authorization/AuthEnforceRewriterTest.java index a03b25121860..dc148bea10f3 100644 --- a/cdap-security/src/test/java/io/cdap/cdap/security/authorization/AuthEnforceRewriterTest.java +++ b/cdap-security/src/test/java/io/cdap/cdap/security/authorization/AuthEnforceRewriterTest.java @@ -159,6 +159,7 @@ private void testRewrite(Method method, Object rewrittenObject, EntityId entityI Class expectedException, Object... args) throws NoSuchMethodException { try { method.invoke(rewrittenObject, args); + Assert.fail(String.format("Expected %s to be thrown", expectedException.getName())); } catch (Exception e) { // Since the above method is invoked through reflection any exception thrown will be wrapped in // InvocationTargetException so verify that the root cause is the expected exception confirming that enforce diff --git a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java index 70e0de1238a9..ccd213c67fb9 100644 --- a/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java +++ b/cdap-spark-core-base/src/main/java/io/cdap/cdap/app/runtime/spark/classloader/SparkClassRewriter.java @@ -263,7 +263,7 @@ private byte[] rewriteOutputMetrics(Type type, InputStream byteCodeStream) throw ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); Method setBytesWrittenMethod = new Method("setBytesWritten", Type.VOID_TYPE, new Type[]{Type.LONG_TYPE}); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { @@ -272,7 +272,7 @@ public MethodVisitor visitMethod(int access, String name, String descriptor, return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, descriptor) { + return new AdviceAdapter(Opcodes.ASM7, mv, access, name, descriptor) { private final Label skipZeroLabel = newLabel(); @@ -357,11 +357,11 @@ private byte[] rewriteDStreamGraph(InputStream byteCodeStream) throws IOExceptio ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM7, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { super.visitMethodInsn(opcode, owner, name, desc, itf); @@ -443,7 +443,7 @@ private byte[] rewriteWritAheadLogHandler(InputStream byteCodeStream) throws IOE Method shutdownMethod = new Method("shutdown", Type.VOID_TYPE, new Type[0]); String executorClass = "scala/concurrent/ExecutionContextExecutorService"; - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String descriptor, @@ -519,7 +519,7 @@ private byte[] rewriteExecutorClassLoader(InputStream byteCodeStream) throws IOE new Type[] { Type.getType(String.class) }); resourceMethods.put(method, null); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { private boolean hasParentLoader; private boolean rewriteInit; @@ -544,7 +544,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si if (!rewriteInit || !"".equals(name)) { return mv; } - return new GeneratorAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new GeneratorAdapter(Opcodes.ASM7, mv, access, name, desc) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { @@ -673,7 +673,7 @@ private byte[] rewriteConstructor(final Type classType, InputStream byteCodeStre ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, final String name, @@ -686,7 +686,7 @@ public MethodVisitor visitMethod(int access, final String name, return mv; } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new AdviceAdapter(Opcodes.ASM7, mv, access, name, desc) { boolean calledThis; @@ -739,11 +739,11 @@ private byte[] rewriteSetProperties(InputStream byteCodeStream) throws IOExcepti ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM7, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { // If we see a call to System.setProperty, change it to SparkRuntimeEnv.setProperty @@ -777,7 +777,7 @@ private byte[] rewriteTempFileNameForCheckpoint(InputStream byteCodeStream) thro ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { boolean hasCheckpointTimeField; @@ -798,7 +798,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); - return new GeneratorAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new GeneratorAdapter(Opcodes.ASM7, mv, access, name, desc) { boolean tempStringAddedToStack; final Method hadoopPathConstructorMethod = @@ -882,7 +882,7 @@ private byte[] rewritePythonRunner(InputStream byteCodeStream) throws IOExceptio // Intercept the static void main(String[] args) method. final Method mainMethod = new Method("main", Type.VOID_TYPE, new Type[] { Type.getType(String[].class) }); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { MethodVisitor mv = super.visitMethod(access, name, desc, signature, exceptions); @@ -901,7 +901,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si // [local_mode] throw (t instance of SparkUserAppException) ? new RuntimeException(t) : t; // [distributed_mode] throw t; // } - return new AdviceAdapter(Opcodes.ASM5, mv, access, name, desc) { + return new AdviceAdapter(Opcodes.ASM7, mv, access, name, desc) { final Type sparkUserAppExceptionType = Type.getObjectType("org/apache/spark/SparkUserAppException"); final Type throwableType = Type.getType(Throwable.class); @@ -983,7 +983,7 @@ private byte[] rewritePythonRunnerCompanion(InputStream byteCodeStream) throws I ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { // Intercept all methods @@ -1003,7 +1003,7 @@ private byte[] rewritePythonWorkerFactory(InputStream byteCodeStream) throws IOE ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, @@ -1013,7 +1013,7 @@ public MethodVisitor visitMethod(int access, String name, access, name, desc, distributed); final GeneratorAdapter adapter = new GeneratorAdapter(mv, access, name, desc); - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM7, mv) { @Override public void visitFieldInsn(int opcode, String owner, String name, String desc) { @@ -1077,7 +1077,7 @@ private byte[] rewriteAkkaRemoting(InputStream byteCodeStream) throws IOExceptio ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(0); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { // Call super so that the method signature is registered with the ClassWriter (parent) @@ -1088,7 +1088,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si return mv; } - return new MethodVisitor(Opcodes.ASM5, mv) { + return new MethodVisitor(Opcodes.ASM7, mv) { @Override public void visitMethodInsn(int opcode, String owner, String name, String desc, boolean itf) { // Detect if it is making call "import scala.concurrent.ExecutionContext.Implicits.global", @@ -1150,7 +1150,7 @@ private byte[] rewriteSparkNetworkClass(InputStream byteCodeStream) throws IOExc // Scan for class type and methods to see if the rewriting is needed final AtomicBoolean rewritten = new AtomicBoolean(false); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { private Type classType; private boolean isFileRegion; @@ -1252,7 +1252,7 @@ private Type determineAkkaDispatcherReturnType() { final AtomicReference result = new AtomicReference<>(); ClassReader cr = new ClassReader(is); - cr.accept(new ClassVisitor(Opcodes.ASM5) { + cr.accept(new ClassVisitor(Opcodes.ASM7) { @Override public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) { if (name.equals("dispatcher") && Type.getArgumentTypes(desc).length == 0) { @@ -1302,7 +1302,7 @@ private byte[] rewriteClient(InputStream byteCodeStream) throws IOException { ClassReader cr = new ClassReader(byteCodeStream); ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); - cr.accept(new ClassVisitor(Opcodes.ASM5, cw) { + cr.accept(new ClassVisitor(Opcodes.ASM7, cw) { @Override public MethodVisitor visitMethod(final int access, final String name, final String desc, String signature, String[] exceptions) { @@ -1418,7 +1418,7 @@ private static final class OutputRedirectMethodVisitor extends MethodVisitor { private final boolean distributed; OutputRedirectMethodVisitor(MethodVisitor mv, int access, String name, String desc, boolean distributed) { - super(Opcodes.ASM5, mv); + super(Opcodes.ASM7, mv); this.adapter = new GeneratorAdapter(mv, access, name, desc); this.distributed = distributed; } diff --git a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java index 5fd0bb2f4c01..bf7d46a046ef 100644 --- a/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java +++ b/cdap-spark-core-base/src/test/java/io/cdap/cdap/app/runtime/spark/classloader/SparkRunnerClassLoaderTest.java @@ -18,9 +18,7 @@ import com.google.common.io.Closeables; import io.cdap.cdap.common.lang.ClassLoaders; -import org.junit.Assert; -import org.junit.Test; - +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; @@ -33,6 +31,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; +import org.junit.Assert; +import org.junit.Test; +import org.objectweb.asm.ClassReader; /** * Unit tests for {@link SparkRunnerClassLoader}. @@ -143,4 +144,37 @@ public void testCloseResourceStream() throws IOException { Closeables.closeQuietly(cl); is.read(); } + + /** + * Nested class compiled to Java 11 with {@code NestHost} attribute for ASM7 rewriter testing. + */ + public static class SampleNestedClass { + + public void obtainTokensForNamenodes() { + // no-op target for YarnSparkHadoopUtil rewrite check + } + } + + @Test + public void testSparkClassRewriterHandlesJava11Nestmates() throws IOException { + String resource = SampleNestedClass.class.getName().replace('.', '/') + ".class"; + byte[] originalBytes; + try (InputStream is = getClass().getClassLoader().getResourceAsStream(resource)) { + Assert.assertNotNull(is); + originalBytes = is.readAllBytes(); + } + + SparkClassRewriter rewriter = new SparkClassRewriter(name -> null, false, false); + byte[] rewrittenYarnUtil = rewriter.rewriteClass( + "org.apache.spark.deploy.yarn.YarnSparkHadoopUtil", + new ByteArrayInputStream(originalBytes)); + Assert.assertNotNull(rewrittenYarnUtil); + Assert.assertEquals(new ClassReader(originalBytes).getClassName(), + new ClassReader(rewrittenYarnUtil).getClassName()); + + byte[] rewrittenSubmit = rewriter.rewriteClass( + "org.apache.spark.deploy.SparkSubmit$Nested", + new ByteArrayInputStream(originalBytes)); + Assert.assertNotNull(rewrittenSubmit); + } } diff --git a/cdap-standalone/bin/cdap.bat b/cdap-standalone/bin/cdap.bat index 29d6439e03ac..e6184f25bcf6 100644 --- a/cdap-standalone/bin/cdap.bat +++ b/cdap-standalone/bin/cdap.bat @@ -85,23 +85,41 @@ if ["%JAVA_HOME%"] == [] ( REM Check for Java version setlocal ENABLEDELAYEDEXPANSION -set /a counter=0 +REM Use the first line of the form: ... version "" ... +REM Lines such as "Picked up JAVA_TOOL_OPTIONS: ..." can come first and are skipped +set "ver=" for /f "tokens=* delims= " %%f in ('"%JAVACMD%" -version 2^>^&1') do @( - if "!counter!"=="0" set line=%%f - set /a counter+=1 + if not defined ver ( + for /f tokens^=1^,2^ delims^=^" %%a in ("%%f") do ( + set "head=%%a" + if "!head:~-8!"=="version " set "ver=%%b" + ) + ) ) -set line=%line:*version "1.=!!% -set line=%line:~0,1% -set java_minimum=7 -set java_maximum=8 -if NOT "%line%" == "%java_minimum%" ( - if NOT "%line%" == "%java_maximum%" ( - echo ERROR: Java version ^(%line%^) is not supported. - echo Please install Java %java_minimum% or %java_maximum%: other versions of Java are not supported. - endlocal - EXIT /B 1 +set "java_major=" +for /f "tokens=1,2 delims=.-_+" %%a in ("!ver!") do ( + if "%%a"=="1" ( + set "java_major=%%b" + ) else ( + set "java_major=%%a" ) ) +set java_minimum=11 +REM LSS falls back to a string compare for non-numbers, so check for digits first +set "non_digit=" +if "!java_major!"=="" set "non_digit=x" +for /f "delims=0123456789" %%i in ("!java_major!") do set "non_digit=%%i" +if defined non_digit ( + echo ERROR: Could not detect Java version from ^(!ver!^). + endlocal + EXIT /B 1 +) +if !java_major! LSS %java_minimum% ( + echo ERROR: Java version ^(!ver!^) is not supported. + echo Please install Java %java_minimum% or higher: older versions of Java are not supported. + endlocal + EXIT /B 1 +) endlocal GOTO :EOF @@ -186,6 +204,7 @@ REM Note the trailing semi-colon so that the last directory entry is interpreted SET "CLASSPATH=%CDAP_HOME%\lib\*;%CDAP_HOME%\conf\;" CALL :CHECK_WINDOWS CALL :CHECK_JAVA +IF %ERRORLEVEL% NEQ 0 GOTO FINALLY CALL :CREATE_LOG_DIR CALL :SET_ACCESS_TOKEN @@ -216,6 +235,7 @@ REM See TX_DEBUGGER for notes on setting CLASSPATH SET "CLASSPATH=%CDAP_HOME%\libexec\io.cdap.cdap.cdap-cli-%CDAP_VERSION%.jar;%CDAP_HOME%\lib\io.cdap.cdap.cdap-cli-%CDAP_VERSION%.jar;%CDAP_HOME%\conf\;" CALL :CHECK_WINDOWS CALL :CHECK_JAVA +IF %ERRORLEVEL% NEQ 0 GOTO FINALLY set class=io.cdap.cdap.cli.CLIMain diff --git a/pom.xml b/pom.xml index f2723505236d..63271a096c1a 100644 --- a/pom.xml +++ b/pom.xml @@ -124,6 +124,7 @@ 0.1.0 2.8.9 13.0.1 + 4.0 4.5.14 4.4.16 @@ -290,6 +291,10 @@ tephra-core ${tephra.version} + + com.google.inject + guice + org.slf4j slf4j-log4j12 @@ -309,16 +314,29 @@ com.google.inject guice ${guice.version} + no_aop com.google.inject.extensions guice-assistedinject ${guice.version} + + + com.google.inject + guice + + com.google.inject.extensions guice-multibindings ${guice.version} + + + com.google.inject + guice + + org.jboss.resteasy @@ -329,6 +347,12 @@ org.jboss.resteasy resteasy-guice ${resteasy.version} + + + com.google.inject + guice + + org.xerial.snappy @@ -452,11 +476,31 @@ logback-classic ${logback.version} + + org.ow2.asm + asm + ${asm.version} + org.ow2.asm asm-commons ${asm.version} + + org.ow2.asm + asm-tree + ${asm.version} + + + org.ow2.asm + asm-analysis + ${asm.version} + + + org.ow2.asm + asm-util + ${asm.version} + org.apache.kafka kafka_2.12 @@ -802,6 +846,10 @@ hadoop-yarn-common ${hadoop.version} + + com.google.inject + guice + com.google.protobuf protobuf-java @@ -896,6 +944,10 @@ hadoop-mapreduce-client-app ${hadoop.version} + + com.google.inject + guice + com.google.protobuf protobuf-java @@ -1178,6 +1230,16 @@ ${tephra.version} test-jar test + + + com.google.inject + guice + + + org.ow2.asm + asm-all + + @@ -1211,6 +1273,10 @@ ${hadoop.version} test + + com.google.inject + guice + log4j log4j @@ -1396,6 +1462,10 @@ common-io ${cdap.common.version} + + com.google.inject + guice + org.ow2.asm asm-all @@ -1407,6 +1477,10 @@ hadoop-mapreduce-client-jobclient ${hadoop.version} + + com.google.inject + guice + com.google.protobuf protobuf-java @@ -1528,10 +1602,14 @@ maven-compiler-plugin 3.7.0 - 1.8 - 1.8 + 11 + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + org.apache.maven.plugins maven-surefire-plugin @@ -1744,6 +1822,13 @@ org.apache.maven.plugins maven-jar-plugin + + + jar + false + none + + org.apache.maven.plugins @@ -1865,8 +1950,10 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 + none + 11 cdap-distributions/src/javadoc/stylesheet.css Cask Data, Inc. Licensed under the Apache License, Version 2.0.]]> @@ -1907,7 +1994,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 @@ -2537,25 +2624,18 @@ - + findbugs - org.codehaus.mojo - findbugs-maven-plugin - 3.0.5 - - - com.thoughtworks.xstream - xstream - 1.4.20 - - + com.github.spotbugs + spotbugs-maven-plugin + 4.8.6.4 - findbugs-check + spotbugs-check check @@ -2674,9 +2754,10 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.0 + 3.6.3 - -Xdoclint:none + none + 11 false @@ -2727,9 +2808,10 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.6.3 - -Xdoclint:none + none + 11 false cdap-distributions/src/javadoc/stylesheet.css