Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,15 @@
import io.qameta.allure.test.IsolatedLifecycle;
import io.qameta.allure.test.RunUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.opentest4j.AssertionFailedError;

import java.time.Instant;
import java.util.List;
import java.util.function.Consumer;
import java.util.stream.Stream;

import static io.qameta.allure.test.AllureTestCommonsUtils.attach;
import static io.qameta.allure.util.ResultsUtils.md5;
Expand Down Expand Up @@ -411,6 +415,7 @@ void shouldReportSuiteFailuresWithoutThrowable() {
"Citrus suite teardown failed"
);
assertThat(errors).allSatisfy(error -> {
assertThat(error.getStatus()).isEqualTo(Status.BROKEN);
assertThat(error.getTrace()).isNull();
assertThat(error.getTimestamp()).isPositive();
});
Expand All @@ -435,10 +440,38 @@ void shouldReportSuiteFailureComparisonDetails() {
final GlobalError error = errors.get(0);
assertThat(error.getMessage()).contains("Citrus suite teardown failed", "suite comparison failed");
assertThat(error.getTrace()).contains("org.opentest4j.AssertionFailedError: suite comparison failed");
assertThat(error.getStatus()).isEqualTo(Status.FAILED);
assertThat(error.getExpected()).isEqualTo(cause.getExpected().toString());
assertThat(error.getActual()).isEqualTo(cause.getActual().toString());
}

@ParameterizedTest
@MethodSource("suiteGlobalStatuses")
void shouldClassifyBothSuiteFailureCallbacks(final Throwable throwable, final Status status) {
final AllureResults results = reportSuiteFailures(adapter -> {
adapter.onStartFailure(throwable);
adapter.onFinishFailure(throwable);
});
final List<GlobalError> errors = results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream())
.toList();
assertThat(errors).hasSize(2).allSatisfy(error -> {
assertThat(error.getStatus()).isEqualTo(status);
if (throwable != null) {
assertThat(error.getTrace()).contains(throwable.getClass().getName());
}
});
assertThat(results.getTestResults()).isEmpty();
}

static Stream<Arguments> suiteGlobalStatuses() {
return Stream.of(
Arguments.of(new AssertionError("assertion"), Status.FAILED),
Arguments.of(new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED),
Arguments.of(new IllegalStateException("runtime"), Status.BROKEN),
Arguments.of(null, Status.BROKEN)
);
}

private CitrusContext.Builder suiteContext() {
// Only Allure output is needed; disable Citrus HTML and JUnit report generation.
return new CitrusContext.Builder().testReporters(new TestReporters());
Expand Down
1 change: 1 addition & 0 deletions allure-cucumber7-jvm/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dependencies {
testImplementation("io.github.glytching:junit-extensions")
testImplementation("org.assertj:assertj-core")
testImplementation("org.junit.jupiter:junit-jupiter-api")
testImplementation("org.junit.jupiter:junit-jupiter-params")
testImplementation("org.slf4j:slf4j-simple")
testImplementation(project(":allure-assertj"))
testImplementation(project(":allure-java-commons-test"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ private void handleTestRunFinished(final TestRunFinished event) {
final String context = scenarioStarted.get()
? "Cucumber test run failed"
: "Cucumber test run failed, scenarios did not run";
lifecycle.writeGlobals(createGlobalError(context, result.getError()));
lifecycle.writeGlobals(createGlobalError(context, result.getError(), translateTestCaseStatus(result)));
}

private void handleTestCaseFinished(final TestCaseFinished event) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import io.cucumber.core.runtime.FeaturePathFeatureSupplier;
import io.cucumber.core.runtime.Runtime;
import io.cucumber.core.runtime.TimeServiceEventBus;
import io.cucumber.plugin.event.Result;
import io.cucumber.plugin.event.TestRunFinished;
import io.github.glytching.junit.extension.system.SystemProperty;
import io.github.glytching.junit.extension.system.SystemPropertyExtension;
import io.qameta.allure.Description;
Expand All @@ -47,8 +49,13 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.parallel.ResourceLock;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.opentest4j.AssertionFailedError;

import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -57,6 +64,7 @@
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static io.qameta.allure.util.ResultsUtils.PACKAGE_LABEL_NAME;
import static io.qameta.allure.util.ResultsUtils.PARENT_SUITE_LABEL_NAME;
Expand Down Expand Up @@ -1057,6 +1065,60 @@ private AllureResults runFeature(final String featureResource,
});
}

@ParameterizedTest
@MethodSource("runErrorStatuses")
void shouldClassifyRunErrorsUsingCucumberOutcome(final io.cucumber.plugin.event.Status outcome,
final Throwable throwable, final Status status) {
final AllureResults results = runFinished(new Result(outcome, Duration.ZERO, throwable));
assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> {
assertThat(error.getStatus()).isEqualTo(status);
assertThat(error.getTrace()).contains(throwable.getClass().getName());
if (throwable instanceof AssertionFailedError) {
assertThat(error.getActual()).startsWith("actual (");
assertThat(error.getExpected()).startsWith("expected (");
}
});
assertThat(results.getTestResults()).isEmpty();
}

static Stream<Arguments> runErrorStatuses() {
return Stream.of(
Arguments.of(
io.cucumber.plugin.event.Status.FAILED,
new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED
),
Arguments.of(
io.cucumber.plugin.event.Status.FAILED,
new IllegalStateException("runtime"), Status.BROKEN
),
Arguments.of(io.cucumber.plugin.event.Status.SKIPPED, new AssertionError("skip"), Status.BROKEN),
Arguments.of(io.cucumber.plugin.event.Status.PENDING, new AssertionError("pending"), Status.BROKEN),
Arguments.of(io.cucumber.plugin.event.Status.AMBIGUOUS, new AssertionError("ambiguous"), Status.BROKEN),
Arguments.of(io.cucumber.plugin.event.Status.UNDEFINED, new AssertionError("undefined"), Status.BROKEN),
Arguments.of(io.cucumber.plugin.event.Status.PASSED, new AssertionError("runtime signal"), Status.BROKEN)
);
}

@Test
void shouldNotCreateGlobalWhenRunResultHasNoThrowable() {
assertThat(runFinished(new Result(io.cucumber.plugin.event.Status.FAILED, Duration.ZERO, null)).getGlobals())
.isEmpty();
}

@Test
void shouldNotCreateGlobalWhenRunResultIsAbsent() {
assertThat(runFinished(null).getGlobals()).isEmpty();
}

private static AllureResults runFinished(final Result result) {
return RunUtils.runTests(lifecycle -> {
final EventBus bus = new TimeServiceEventBus(Clock.systemUTC(), UUID::randomUUID);
final AllureCucumber7Jvm adapter = new AllureCucumber7Jvm(lifecycle);
adapter.setEventPublisher(bus);
bus.send(result == null ? new TestRunFinished(Instant.now()) : new TestRunFinished(Instant.now(), result));
});
}

private static List<GlobalError> getGlobalErrors(final AllureResults results) {
return results.getGlobals().stream()
.flatMap(globals -> globals.getErrors().stream())
Expand Down
25 changes: 22 additions & 3 deletions allure-java-commons/src/main/java/io/qameta/allure/Allure.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@

/**
* The class contains some useful methods to work with {@link AllureLifecycle}.
* This facade intentionally exposes the runtime API through a single entry point.
*/
@SuppressWarnings("PMD.TooManyMethods")
@SuppressWarnings({"PMD.TooManyMethods", "PMD.GodClass"})
public final class Allure {

private static final String TEXT_PLAIN = "text/plain";
Expand Down Expand Up @@ -422,21 +423,39 @@ public static void descriptionHtml(final String descriptionHtml) {
public static void globalError(final Throwable throwable) {
Objects.requireNonNull(throwable, "throwable");
final StatusDetails statusDetails = getStatusDetails(throwable).orElseThrow();
globalError(statusDetails);
globalError(getStatus(throwable).orElse(Status.BROKEN), statusDetails);
}

/**
* Adds an error with the given status details that belongs to the test run rather than to a test or fixture.
* The error is passed synchronously to the configured results writer before this method returns.
* Writers that support globals store it in a new globals artifact.
* The status detail fields are copied without normalization, and the timestamp is assigned when this method
* is called.
* is called. The error defaults to broken; use the explicit status overload to report failed.
*
* @param statusDetails the error details to add
*/
public static void globalError(final StatusDetails statusDetails) {
globalError(Status.BROKEN, statusDetails);
}

/**
* Adds a run-level error with an explicit failed or broken status.
* Details are copied without mutation and the error is written synchronously with a fresh timestamp.
*
* @param status the error status, must be failed or broken
* @param statusDetails the error details to add
* @throws NullPointerException if either argument is null
* @throws IllegalArgumentException if the status is neither failed nor broken
*/
public static void globalError(final Status status, final StatusDetails statusDetails) {
Objects.requireNonNull(status, "status");
Objects.requireNonNull(statusDetails, "statusDetails");
if (status != Status.FAILED && status != Status.BROKEN) {
throw new IllegalArgumentException("Global error status must be failed or broken");
}
final GlobalError globalError = toGlobalError(statusDetails)
.setStatus(status)
.setTimestamp(System.currentTimeMillis());
final Globals globals = new Globals();
globals.getErrors().add(globalError);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.Locale;

import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
import static com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS;
import static com.fasterxml.jackson.databind.MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME;
import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;

Expand Down Expand Up @@ -56,6 +57,7 @@ public static ObjectMapper createMapper() {
return JsonMapper
.builder()
.configure(USE_WRAPPER_NAME_AS_PROPERTY_NAME, true)
.configure(ACCEPT_CASE_INSENSITIVE_ENUMS, true)
.serializationInclusion(NON_NULL)
.configure(INDENT_OUTPUT, Boolean.getBoolean(INDENT_OUTPUT_PROPERTY_NAME))
.build()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,22 @@ public static Optional<StatusDetails> getStatusDetails(final Throwable e) {
* @return the globals artifact holding a single error
*/
public static Globals createGlobalError(final String context, final Throwable throwable) {
return createGlobalError(context, throwable, getStatus(throwable).orElse(Status.BROKEN));
}

/**
* Returns a globals artifact with an adapter-resolved error status.
* Only failed is preserved; any other status is normalized to broken.
* Resolve the status before passing a redacted or otherwise transformed throwable.
*
* @param context the description of what failed
* @param throwable the failure details, may be null
* @param status the adapter-resolved status, may be null
* @return the globals artifact holding a single error
*/
public static Globals createGlobalError(final String context, final Throwable throwable, final Status status) {
final GlobalError error = new GlobalError()
.setStatus(status == Status.FAILED ? Status.FAILED : Status.BROKEN)
.setTimestamp(System.currentTimeMillis())
.setMessage(context);

Expand Down
Loading
Loading