Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
types: [opened, synchronize, reopened, labeled]

jobs:
build:

Check warning on line 25 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

excessive-permissions

build.yml:25: overly broad permissions: default permissions used due to no permissions: block
runs-on: k8s-runner-build

# We allow builds:
Expand All @@ -45,11 +45,16 @@
gcloud config configurations activate emulator
gcloud emulators spanner start &
echo "SPANNER_EMULATOR_HOST=localhost:9010" >> $GITHUB_ENV
- uses: actions/checkout@v4

Check failure on line 48 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

zizmor/unpinned-uses

unpinned action reference: action is not pinned to a hash (required by blanket policy)

Check failure on line 48 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

unpinned-uses

build.yml:48: unpinned action reference: action is not pinned to a hash (required by blanket policy)
with:
submodules: recursive
- name: Set up JDK 11
uses: actions/setup-java@v4

Check failure on line 52 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

zizmor/unpinned-uses

unpinned action reference: action is not pinned to a hash (required by blanket policy)

Check failure on line 52 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

unpinned-uses

build.yml:52: unpinned action reference: action is not pinned to a hash (required by blanket policy)
with:
distribution: 'temurin'
java-version: '11'
- name: Cache
uses: actions/cache@v4

Check failure on line 57 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

zizmor/unpinned-uses

unpinned action reference: action is not pinned to a hash (required by blanket policy)

Check failure on line 57 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

unpinned-uses

build.yml:57: unpinned action reference: action is not pinned to a hash (required by blanket policy)
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
Expand All @@ -59,7 +64,7 @@
# TODO(CDAP-21120): Enable PreviewDataPipelineTest after fixing the flaky test.
run: mvn clean test -fae -T 2 -B -V -DcloudBuild -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dtest='!io.cdap.cdap.datapipeline.preview.PreviewDataPipelineTest,!io.cdap.cdap.test.base.TestFrameworkTestBase,!io.cdap.cdap.datapipeline.ReducibleAggregatorTestBase' -Dsurefire.failIfNoSpecifiedTests=false
- name: Archive build artifacts
uses: actions/upload-artifact@v4

Check failure on line 67 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

zizmor/unpinned-uses

unpinned action reference: action is not pinned to a hash (required by blanket policy)

Check failure on line 67 in .github/workflows/build.yml

View workflow job for this annotation

GitHub Actions / zizmor-output

unpinned-uses

build.yml:67: unpinned action reference: action is not pinned to a hash (required by blanket policy)
if: always()
with:
name: reports-${{ github.run_id }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import javax.annotation.Nullable;
import org.apache.twill.filesystem.Location;

Expand All @@ -48,7 +46,7 @@ protected AbstractArtifactManager(CConfiguration cConf) {
File tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR),
cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile();
this.tmpDir = DirUtils.createTempDir(tmpDir);
this.bootstrapClassLoader = new URLClassLoader(new URL[0], null);
this.bootstrapClassLoader = ClassLoader.getPlatformClassLoader();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@

package io.cdap.cdap.internal.app.runtime.distributed;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.io.Closeables;
import com.google.common.reflect.TypeToken;
Expand Down Expand Up @@ -70,6 +72,7 @@
import java.net.ProxySelector;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.Security;
import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
Expand All @@ -96,7 +99,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.bridge.SLF4JBridgeHandler;
import sun.net.InetAddressCachePolicy;

/**
* A {@link TwillRunnable} for running a program through a {@link ProgramRunner}.
Expand All @@ -106,6 +108,10 @@
public abstract class AbstractProgramTwillRunnable<T extends ProgramRunner> implements
TwillRunnable {

static final String NETWORKADDRESS_CACHE_TTL = "networkaddress.cache.ttl";
static final String SUN_NET_INETADDR_TTL = "sun.net.inetaddr.ttl";
static final String DEFAULT_DNS_CACHE_TTL_SECONDS = "30";

private static final Logger LOG = LoggerFactory.getLogger(AbstractProgramTwillRunnable.class);
private static final Gson GSON = ApplicationSpecificationAdapter.addTypeAdapters(
new GsonBuilder())
Expand Down Expand Up @@ -163,6 +169,19 @@ public final void initialize(TwillContext context) {
}
}

/**
* Sets a default DNS cache TTL unless the user already set one. Without an explicit TTL,
* {@code InetAddressCachePolicy} caches DNS lookups forever once a {@link SecurityManager} is
* installed, so this must run before {@link System#setSecurityManager}. See CDAP-20781.
*/
@VisibleForTesting
static void initializeDnsCachePolicy() {
if (Strings.nullToEmpty(Security.getProperty(NETWORKADDRESS_CACHE_TTL)).trim().isEmpty()
&& Strings.nullToEmpty(System.getProperty(SUN_NET_INETADDR_TTL)).trim().isEmpty()) {
Security.setProperty(NETWORKADDRESS_CACHE_TTL, DEFAULT_DNS_CACHE_TTL_SECONDS);
}
}

/**
* Prepares this instance to execute a program.
*
Expand All @@ -173,15 +192,8 @@ private void doInitialize(File programOptionFile) throws Exception {
controllerFuture = new CompletableFuture<>();
programCompletion = new CompletableFuture<>();

// Make sure InetAddressCachePolicy is not set to FOREVER.
// This is needed because if InetAddressCachePolicy is loaded after System#setSecurityManager() call,
// caching policy is set to FOREVER. With FOREVER, caching policy, this twill runnable will not be able to
// reach out to other services within the cluster as IP is cached forever.
// Look at CDAP-20781 for more information.
if (InetAddressCachePolicy.get() == InetAddressCachePolicy.FOREVER) {
LOG.warn("InetAddress cache policy is set to FOREVER. This will ips to be cached forever. "
+ "Ensure cache policy is not set to FOREVER.");
}
// Must run before installing the SecurityManager below (CDAP-20781).
initializeDnsCachePolicy();

// Setup process wide settings
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -465,9 +465,25 @@ private String generateLaunchScript(
args.put(ClusterProperties.KERBEROS_KEYTAB, kerberosKeytab);
}

scriptWriter.println("if [ -n \"${JAVA_HOME}\" ] && [ -x \"${JAVA_HOME}/bin/java\" ]; then");
scriptWriter.println(" JAVA_CMD=\"${JAVA_HOME}/bin/java\"");
scriptWriter.println("else");
scriptWriter.println(" JAVA_CMD=\"java\"");
scriptWriter.println("fi");
scriptWriter.println("JAVA_VER_RAW=$(\"${JAVA_CMD}\" -version 2>&1"
+ " | awk -F '\"' '!/^Picked up / && /version \"/ {print $2; exit}')");
scriptWriter.println("JAVA_MAJOR=$(echo \"${JAVA_VER_RAW}\""
+ " | awk -F '[._+-]' '{if ($1 == \"1\") print $2; else print $1}')");
scriptWriter.println("case \"${JAVA_MAJOR}\" in ''|*[!0-9]*) JAVA_MAJOR=0 ;; esac");
scriptWriter.println("if [ \"${JAVA_MAJOR}\" -lt 11 ]; then");
scriptWriter.println(" echo \"Java 11+ required on the cluster"
+ " (found: ${JAVA_VER_RAW:-unknown})\" >&2");
scriptWriter.println(" exit 1");
scriptWriter.println("fi");

scriptWriter.printf(
"nohup java -Djava.io.tmpdir=tmp -Dcdap.runid=%s -cp %s/%s -Xmx%dm %s %s '%s' true %s "
+ ">%s/stdout 2>%s/stderr &\n",
"nohup \"${JAVA_CMD}\" -Djava.io.tmpdir=tmp -Dcdap.runid=%s -cp %s/%s -Xmx%dm %s %s '%s'"
+ " true %s >%s/stdout 2>%s/stderr &\n",
getProgramRunId().getRun(),
targetPath,
Constants.Files.LAUNCHER_JAR,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ private void inspectHandler(final TypeToken<?> delegateType, final TypeToken<?>
.getResourceAsStream(Type.getInternalName(rawType) + ".class")
) {
ClassReader classReader = new ClassReader(sourceBytes);
classReader.accept(new ClassVisitor(Opcodes.ASM5) {
classReader.accept(new ClassVisitor(Opcodes.ASM7) {

// Only need to visit @Path at the class level if we are inspecting the user handler class
private final boolean inspectDelegate = delegateType.equals(inspectType);
Expand All @@ -215,7 +215,7 @@ public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
if (inspectDelegate && type.equals(Type.getType(Path.class))) {
visitedPath = true;
AnnotationVisitor annotationVisitor = classWriter.visitAnnotation(desc, visible);
return new AnnotationVisitor(Opcodes.ASM5, annotationVisitor) {
return new AnnotationVisitor(Opcodes.ASM7, annotationVisitor) {
@Override
public void visit(String name, Object value) {
// "value" is the key for the Path annotation string.
Expand Down Expand Up @@ -374,7 +374,7 @@ private class HandlerMethodVisitor extends MethodVisitor {
HandlerMethodVisitor(TypeToken<?> delegateType, MethodVisitor mv, String desc,
String signature, int access, String name, String[] exceptions,
Type classType, ClassWriter classWriter, List<Class<?>> preservedClasses) {
super(Opcodes.ASM5, mv);
super(Opcodes.ASM7, mv);
this.delegateType = delegateType;
this.desc = desc;
this.signature = signature;
Expand All @@ -392,7 +392,7 @@ private class HandlerMethodVisitor extends MethodVisitor {
public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
// Memorize all visible annotations
if (visible) {
AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc);
AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM7, desc);
annotations.add(annotationNode);
return annotationNode;
}
Expand All @@ -404,7 +404,7 @@ public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, bo
// Memorize all visible annotations for each parameter.
// It needs to store in a Multimap because there can be multiple annotations per parameter.
if (visible) {
AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM5, desc);
AnnotationNode annotationNode = new AnnotationNode(Opcodes.ASM7, desc);
paramAnnotations.put(parameter, annotationNode);
return annotationNode;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright © 2026 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

package io.cdap.cdap.internal.app.runtime.artifact;

import io.cdap.cdap.api.artifact.ArtifactClasses;
import io.cdap.cdap.api.artifact.ArtifactInfo;
import io.cdap.cdap.api.artifact.ArtifactScope;
import io.cdap.cdap.api.artifact.CloseableClassLoader;
import io.cdap.cdap.common.conf.CConfiguration;
import io.cdap.cdap.common.conf.Constants;
import io.cdap.cdap.common.test.AppJarHelper;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.twill.filesystem.LocalLocationFactory;
import org.apache.twill.filesystem.Location;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
* Tests {@link AbstractArtifactManager} classloader behavior on Java 11.
*/
public class AbstractArtifactManagerTest {

@ClassRule
public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();

/**
* Standalone class packaged into the test artifact jar with no parent CDAP API dependencies.
*/
public static class SampleArtifactClass {
}

@Test
public void testNullParentLoadsPlatformModules() throws Exception {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());

Location jarLocation = AppJarHelper.createDeploymentJar(
new LocalLocationFactory(TMP_FOLDER.newFolder()), SampleArtifactClass.class);
ArtifactInfo artifactInfo = new ArtifactInfo(
"sample-artifact", "1.0.0", ArtifactScope.USER, ArtifactClasses.builder().build(),
Collections.emptyMap());

AbstractArtifactManager manager = new AbstractArtifactManager(cConf) {
@Override
public List<ArtifactInfo> listArtifacts() {
return Collections.singletonList(artifactInfo);
}

@Override
public List<ArtifactInfo> listArtifacts(String namespace) {
return listArtifacts();
}

@Override
protected Location getArtifactLocation(ArtifactInfo info, @Nullable String namespace)
throws IOException {
return jarLocation;
}
};

try (CloseableClassLoader classLoader = manager.createClassLoader(artifactInfo, null)) {
Assert.assertNotNull(classLoader.loadClass(SampleArtifactClass.class.getName()));
Assert.assertSame(Class.forName("java.sql.Driver"), classLoader.loadClass("java.sql.Driver"));
Assert.assertSame(Class.forName("org.ietf.jgss.GSSManager"),
classLoader.loadClass("org.ietf.jgss.GSSManager"));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@
import io.cdap.cdap.internal.app.deploy.pipeline.ApplicationWithPrograms;
import io.cdap.cdap.internal.app.runtime.BasicArguments;
import io.cdap.cdap.test.XSlowTests;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
Expand All @@ -57,7 +58,9 @@
@Category(XSlowTests.class)
public class MapReduceWithPartitionedTest extends MapReduceRunnerTestBase {

static final DateFormat DATE_FORMAT = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.SHORT, Locale.US);
private static long parseDateTime(String value) throws ParseException {
return new SimpleDateFormat("M/d/yy h:mm a", Locale.US).parse(value).getTime();
}


@Test
Expand All @@ -75,7 +78,7 @@ public void apply() {
}
});

final long time = DATE_FORMAT.parse("1/15/15 11:15 am").getTime();
final long time = parseDateTime("1/15/15 11:15 am");
final long time5 = time + TimeUnit.MINUTES.toMillis(5);

// run the partition writer m/r with this output partition time
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Copyright © 2026 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

package io.cdap.cdap.internal.app.runtime.distributed;

import java.security.Security;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

/**
* Unit tests for {@link AbstractProgramTwillRunnable#initializeDnsCachePolicy()}.
*/
public class AbstractProgramTwillRunnableTest {

private String originalSecurityTtl;
private String originalSystemTtl;

@Before
public void setUp() {
originalSecurityTtl = Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL);
originalSystemTtl = System.getProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);
}

@After
public void tearDown() {
Security.setProperty(
AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL,
originalSecurityTtl == null ? "" : originalSecurityTtl);
if (originalSystemTtl == null) {
System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);
} else {
System.setProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL, originalSystemTtl);
}
}

@Test
public void testDefaultsToThirtySecondsWhenUnset() {
Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "");
System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);

AbstractProgramTwillRunnable.initializeDnsCachePolicy();
Assert.assertEquals("30",
Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL));
}

@Test
public void testDefaultsWhenSecurityPropertyIsBlank() {
Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, " ");
System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);

AbstractProgramTwillRunnable.initializeDnsCachePolicy();
Assert.assertEquals("30",
Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL));
}

@Test
public void testPreservesPreSetSecurityProperty() {
Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "60");
System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);

AbstractProgramTwillRunnable.initializeDnsCachePolicy();
Assert.assertEquals("60",
Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL));
}

@Test
public void testPreservesPreSetSystemProperty() {
Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "");
System.setProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL, "45");

AbstractProgramTwillRunnable.initializeDnsCachePolicy();
Assert.assertEquals("",
Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL));
}

@Test
public void testPreservesExplicitForeverPolicy() {
Security.setProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL, "-1");
System.clearProperty(AbstractProgramTwillRunnable.SUN_NET_INETADDR_TTL);

AbstractProgramTwillRunnable.initializeDnsCachePolicy();
Assert.assertEquals("-1",
Security.getProperty(AbstractProgramTwillRunnable.NETWORKADDRESS_CACHE_TTL));
}
}
Loading
Loading