diff --git a/allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java b/allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java index c08929336..39e533ac6 100644 --- a/allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java +++ b/allure-citrus/src/test/java/io/qameta/allure/citrus/AllureCitrusTest.java @@ -46,11 +46,15 @@ import io.qameta.allure.test.IsolatedLifecycle; import io.qameta.allure.test.RunUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.opentest4j.AssertionFailedError; import java.time.Instant; import java.util.List; import java.util.function.Consumer; +import java.util.stream.Stream; import static io.qameta.allure.test.AllureTestCommonsUtils.attach; import static io.qameta.allure.util.ResultsUtils.md5; @@ -411,6 +415,7 @@ void shouldReportSuiteFailuresWithoutThrowable() { "Citrus suite teardown failed" ); assertThat(errors).allSatisfy(error -> { + assertThat(error.getStatus()).isEqualTo(Status.BROKEN); assertThat(error.getTrace()).isNull(); assertThat(error.getTimestamp()).isPositive(); }); @@ -435,10 +440,38 @@ void shouldReportSuiteFailureComparisonDetails() { final GlobalError error = errors.get(0); assertThat(error.getMessage()).contains("Citrus suite teardown failed", "suite comparison failed"); assertThat(error.getTrace()).contains("org.opentest4j.AssertionFailedError: suite comparison failed"); + assertThat(error.getStatus()).isEqualTo(Status.FAILED); assertThat(error.getExpected()).isEqualTo(cause.getExpected().toString()); assertThat(error.getActual()).isEqualTo(cause.getActual().toString()); } + @ParameterizedTest + @MethodSource("suiteGlobalStatuses") + void shouldClassifyBothSuiteFailureCallbacks(final Throwable throwable, final Status status) { + final AllureResults results = reportSuiteFailures(adapter -> { + adapter.onStartFailure(throwable); + adapter.onFinishFailure(throwable); + }); + final List errors = results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()) + .toList(); + assertThat(errors).hasSize(2).allSatisfy(error -> { + assertThat(error.getStatus()).isEqualTo(status); + if (throwable != null) { + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream suiteGlobalStatuses() { + return Stream.of( + Arguments.of(new AssertionError("assertion"), Status.FAILED), + Arguments.of(new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED), + Arguments.of(new IllegalStateException("runtime"), Status.BROKEN), + Arguments.of(null, Status.BROKEN) + ); + } + private CitrusContext.Builder suiteContext() { // Only Allure output is needed; disable Citrus HTML and JUnit report generation. return new CitrusContext.Builder().testReporters(new TestReporters()); diff --git a/allure-cucumber7-jvm/build.gradle.kts b/allure-cucumber7-jvm/build.gradle.kts index 376658b19..756df3c10 100644 --- a/allure-cucumber7-jvm/build.gradle.kts +++ b/allure-cucumber7-jvm/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { testImplementation("io.github.glytching:junit-extensions") testImplementation("org.assertj:assertj-core") testImplementation("org.junit.jupiter:junit-jupiter-api") + testImplementation("org.junit.jupiter:junit-jupiter-params") testImplementation("org.slf4j:slf4j-simple") testImplementation(project(":allure-assertj")) testImplementation(project(":allure-java-commons-test")) diff --git a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java index 501eed491..d7699cf68 100644 --- a/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java +++ b/allure-cucumber7-jvm/src/main/java/io/qameta/allure/cucumber7jvm/AllureCucumber7Jvm.java @@ -246,7 +246,7 @@ private void handleTestRunFinished(final TestRunFinished event) { final String context = scenarioStarted.get() ? "Cucumber test run failed" : "Cucumber test run failed, scenarios did not run"; - lifecycle.writeGlobals(createGlobalError(context, result.getError())); + lifecycle.writeGlobals(createGlobalError(context, result.getError(), translateTestCaseStatus(result))); } private void handleTestCaseFinished(final TestCaseFinished event) { diff --git a/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java index b87094cde..2fd9e8342 100644 --- a/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java +++ b/allure-cucumber7-jvm/src/test/java/io/qameta/allure/cucumber7jvm/AllureCucumber7JvmTest.java @@ -24,6 +24,8 @@ import io.cucumber.core.runtime.FeaturePathFeatureSupplier; import io.cucumber.core.runtime.Runtime; import io.cucumber.core.runtime.TimeServiceEventBus; +import io.cucumber.plugin.event.Result; +import io.cucumber.plugin.event.TestRunFinished; import io.github.glytching.junit.extension.system.SystemProperty; import io.github.glytching.junit.extension.system.SystemPropertyExtension; import io.qameta.allure.Description; @@ -47,8 +49,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -57,6 +64,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.qameta.allure.util.ResultsUtils.PACKAGE_LABEL_NAME; import static io.qameta.allure.util.ResultsUtils.PARENT_SUITE_LABEL_NAME; @@ -1057,6 +1065,60 @@ private AllureResults runFeature(final String featureResource, }); } + @ParameterizedTest + @MethodSource("runErrorStatuses") + void shouldClassifyRunErrorsUsingCucumberOutcome(final io.cucumber.plugin.event.Status outcome, + final Throwable throwable, final Status status) { + final AllureResults results = runFinished(new Result(outcome, Duration.ZERO, throwable)); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + if (throwable instanceof AssertionFailedError) { + assertThat(error.getActual()).startsWith("actual ("); + assertThat(error.getExpected()).startsWith("expected ("); + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream runErrorStatuses() { + return Stream.of( + Arguments.of( + io.cucumber.plugin.event.Status.FAILED, + new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED + ), + Arguments.of( + io.cucumber.plugin.event.Status.FAILED, + new IllegalStateException("runtime"), Status.BROKEN + ), + Arguments.of(io.cucumber.plugin.event.Status.SKIPPED, new AssertionError("skip"), Status.BROKEN), + Arguments.of(io.cucumber.plugin.event.Status.PENDING, new AssertionError("pending"), Status.BROKEN), + Arguments.of(io.cucumber.plugin.event.Status.AMBIGUOUS, new AssertionError("ambiguous"), Status.BROKEN), + Arguments.of(io.cucumber.plugin.event.Status.UNDEFINED, new AssertionError("undefined"), Status.BROKEN), + Arguments.of(io.cucumber.plugin.event.Status.PASSED, new AssertionError("runtime signal"), Status.BROKEN) + ); + } + + @Test + void shouldNotCreateGlobalWhenRunResultHasNoThrowable() { + assertThat(runFinished(new Result(io.cucumber.plugin.event.Status.FAILED, Duration.ZERO, null)).getGlobals()) + .isEmpty(); + } + + @Test + void shouldNotCreateGlobalWhenRunResultIsAbsent() { + assertThat(runFinished(null).getGlobals()).isEmpty(); + } + + private static AllureResults runFinished(final Result result) { + return RunUtils.runTests(lifecycle -> { + final EventBus bus = new TimeServiceEventBus(Clock.systemUTC(), UUID::randomUUID); + final AllureCucumber7Jvm adapter = new AllureCucumber7Jvm(lifecycle); + adapter.setEventPublisher(bus); + bus.send(result == null ? new TestRunFinished(Instant.now()) : new TestRunFinished(Instant.now(), result)); + }); + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-java-commons/src/main/java/io/qameta/allure/Allure.java b/allure-java-commons/src/main/java/io/qameta/allure/Allure.java index c9e79c859..eec88334f 100644 --- a/allure-java-commons/src/main/java/io/qameta/allure/Allure.java +++ b/allure-java-commons/src/main/java/io/qameta/allure/Allure.java @@ -46,8 +46,9 @@ /** * The class contains some useful methods to work with {@link AllureLifecycle}. + * This facade intentionally exposes the runtime API through a single entry point. */ -@SuppressWarnings("PMD.TooManyMethods") +@SuppressWarnings({"PMD.TooManyMethods", "PMD.GodClass"}) public final class Allure { private static final String TEXT_PLAIN = "text/plain"; @@ -422,7 +423,7 @@ public static void descriptionHtml(final String descriptionHtml) { public static void globalError(final Throwable throwable) { Objects.requireNonNull(throwable, "throwable"); final StatusDetails statusDetails = getStatusDetails(throwable).orElseThrow(); - globalError(statusDetails); + globalError(getStatus(throwable).orElse(Status.BROKEN), statusDetails); } /** @@ -430,13 +431,31 @@ public static void globalError(final Throwable throwable) { * The error is passed synchronously to the configured results writer before this method returns. * Writers that support globals store it in a new globals artifact. * The status detail fields are copied without normalization, and the timestamp is assigned when this method - * is called. + * is called. The error defaults to broken; use the explicit status overload to report failed. * * @param statusDetails the error details to add */ public static void globalError(final StatusDetails statusDetails) { + globalError(Status.BROKEN, statusDetails); + } + + /** + * Adds a run-level error with an explicit failed or broken status. + * Details are copied without mutation and the error is written synchronously with a fresh timestamp. + * + * @param status the error status, must be failed or broken + * @param statusDetails the error details to add + * @throws NullPointerException if either argument is null + * @throws IllegalArgumentException if the status is neither failed nor broken + */ + public static void globalError(final Status status, final StatusDetails statusDetails) { + Objects.requireNonNull(status, "status"); Objects.requireNonNull(statusDetails, "statusDetails"); + if (status != Status.FAILED && status != Status.BROKEN) { + throw new IllegalArgumentException("Global error status must be failed or broken"); + } final GlobalError globalError = toGlobalError(statusDetails) + .setStatus(status) .setTimestamp(System.currentTimeMillis()); final Globals globals = new Globals(); globals.getErrors().add(globalError); diff --git a/allure-java-commons/src/main/java/io/qameta/allure/internal/Allure2ModelJackson.java b/allure-java-commons/src/main/java/io/qameta/allure/internal/Allure2ModelJackson.java index 2bb7c13e4..71a3ded45 100644 --- a/allure-java-commons/src/main/java/io/qameta/allure/internal/Allure2ModelJackson.java +++ b/allure-java-commons/src/main/java/io/qameta/allure/internal/Allure2ModelJackson.java @@ -29,6 +29,7 @@ import java.util.Locale; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS; import static com.fasterxml.jackson.databind.MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME; import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT; @@ -56,6 +57,7 @@ public static ObjectMapper createMapper() { return JsonMapper .builder() .configure(USE_WRAPPER_NAME_AS_PROPERTY_NAME, true) + .configure(ACCEPT_CASE_INSENSITIVE_ENUMS, true) .serializationInclusion(NON_NULL) .configure(INDENT_OUTPUT, Boolean.getBoolean(INDENT_OUTPUT_PROPERTY_NAME)) .build() diff --git a/allure-java-commons/src/main/java/io/qameta/allure/util/ResultsUtils.java b/allure-java-commons/src/main/java/io/qameta/allure/util/ResultsUtils.java index 6e39b0d2a..5937c747a 100644 --- a/allure-java-commons/src/main/java/io/qameta/allure/util/ResultsUtils.java +++ b/allure-java-commons/src/main/java/io/qameta/allure/util/ResultsUtils.java @@ -748,7 +748,22 @@ public static Optional getStatusDetails(final Throwable e) { * @return the globals artifact holding a single error */ public static Globals createGlobalError(final String context, final Throwable throwable) { + return createGlobalError(context, throwable, getStatus(throwable).orElse(Status.BROKEN)); + } + + /** + * Returns a globals artifact with an adapter-resolved error status. + * Only failed is preserved; any other status is normalized to broken. + * Resolve the status before passing a redacted or otherwise transformed throwable. + * + * @param context the description of what failed + * @param throwable the failure details, may be null + * @param status the adapter-resolved status, may be null + * @return the globals artifact holding a single error + */ + public static Globals createGlobalError(final String context, final Throwable throwable, final Status status) { final GlobalError error = new GlobalError() + .setStatus(status == Status.FAILED ? Status.FAILED : Status.BROKEN) .setTimestamp(System.currentTimeMillis()) .setMessage(context); diff --git a/allure-java-commons/src/test/java/io/qameta/allure/AllureTest.java b/allure-java-commons/src/test/java/io/qameta/allure/AllureTest.java index e9354eb4f..6401f50b5 100644 --- a/allure-java-commons/src/test/java/io/qameta/allure/AllureTest.java +++ b/allure-java-commons/src/test/java/io/qameta/allure/AllureTest.java @@ -19,6 +19,7 @@ import io.qameta.allure.http.HttpExchangeRequest; import io.qameta.allure.model.Attachment; import io.qameta.allure.model.FixtureResult; +import io.qameta.allure.model.GlobalError; import io.qameta.allure.model.Label; import io.qameta.allure.model.Link; import io.qameta.allure.model.Parameter; @@ -30,6 +31,11 @@ import io.qameta.allure.test.IsolatedLifecycle; import io.qameta.allure.util.ObjectUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.NullSource; +import org.opentest4j.AssertionFailedError; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; @@ -37,6 +43,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; import static io.qameta.allure.Allure.addHttpExchange; import static io.qameta.allure.Allure.attachment; @@ -58,6 +65,8 @@ import static io.qameta.allure.test.ThreadLocalEnhancedRandom.current; import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNullPointerException; import static org.assertj.core.api.Assertions.fail; import static org.assertj.core.api.Assertions.tuple; @SuppressWarnings("unchecked") @@ -68,6 +77,101 @@ class AllureTest { private static final String TEST_KEY_NAMESPACE = "test:test"; private static final String FIXTURE_KEY_NAMESPACE = "test:fixture"; + @ParameterizedTest + @MethodSource("globalErrorThrowables") + void shouldClassifyGlobalThrowableBeforeCopyingDetails(final Throwable throwable) { + final AllureResults results = runTests(ignored -> Allure.globalError(throwable)); + final GlobalError error = results.getGlobals().get(0).getErrors().get(0); + assertThat(error.getStatus()).isEqualTo( + throwable instanceof AssertionError ? Status.FAILED : Status.BROKEN + ); + assertThat(error.getMessage()).isEqualTo(throwable.getMessage()); + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + if (throwable instanceof AssertionFailedError) { + assertThat(error.getActual()).startsWith("actual ("); + assertThat(error.getExpected()).startsWith("expected ("); + } + } + + static Stream globalErrorThrowables() { + return Stream.of( + new AssertionError("assertion"), + new AssertionFailedError("comparison", "expected", "actual"), + new IllegalStateException("runtime") + ); + } + + @Test + void shouldDefaultUnclassifiedGlobalDetailsToBroken() { + final StatusDetails input = new StatusDetails().setMessage("assertion-looking details") + .setActual("actual").setExpected("expected"); + final AllureResults results = runTests(ignored -> Allure.globalError(input)); + assertThat(results.getGlobals().get(0).getErrors().get(0).getStatus()).isEqualTo(Status.BROKEN); + } + + @ParameterizedTest + @EnumSource( + value = Status.class, + names = {"FAILED", "BROKEN"} + ) + void shouldWriteExplicitGlobalStatusWithoutMutatingDetails(final Status status) { + final GlobalError input = new GlobalError().setStatus(Status.BROKEN) + .setKnown(true).setMuted(true).setFlaky(true).setMessage("comparison") + .setTrace("trace").setActual("actual").setExpected("expected").setTimestamp(1L); + final long before = System.currentTimeMillis(); + final AllureResults results = runTests(ignored -> Allure.globalError(status, input)); + final GlobalError error = results.getGlobals().get(0).getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error).isNotSameAs(input); + assertThat(error.getTimestamp()).isBetween(before, System.currentTimeMillis()); + assertThat(error.isKnown()).isTrue(); + assertThat(error.isMuted()).isTrue(); + assertThat(error.isFlaky()).isTrue(); + assertThat(error.getMessage()).isEqualTo("comparison"); + assertThat(error.getTrace()).isEqualTo("trace"); + assertThat(error.getActual()).isEqualTo("actual"); + assertThat(error.getExpected()).isEqualTo("expected"); + assertThat(input.getStatus()).isEqualTo(Status.BROKEN); + assertThat(input.getTimestamp()).isEqualTo(1L); + } + + @ParameterizedTest + @EnumSource(Status.class) + @NullSource + void shouldTreatGlobalErrorInputAsDetails(final Status status) { + final GlobalError input = new GlobalError().setStatus(status).setMessage("run error").setTimestamp(1L); + final AllureResults results = runTests(ignored -> Allure.globalError(input)); + final GlobalError error = results.getGlobals().get(0).getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(Status.BROKEN); + assertThat(error.getMessage()).isEqualTo(input.getMessage()); + assertThat(error).isNotSameAs(input); + assertThat(input.getStatus()).isEqualTo(status); + assertThat(input.getTimestamp()).isEqualTo(1L); + } + + @ParameterizedTest + @EnumSource( + value = Status.class, + names = {"PASSED", "SKIPPED"} + ) + void shouldRejectUnsupportedExplicitGlobalStatus(final Status status) { + final AllureResults results = runTests( + ignored -> assertThatIllegalArgumentException().isThrownBy(() -> Allure.globalError(status, new StatusDetails())) + ); + assertThat(results.getGlobals()).isEmpty(); + } + + @Test + void shouldRejectNullExplicitGlobalStatusAndDetails() { + final AllureResults results = runTests(ignored -> { + assertThatNullPointerException() + .isThrownBy(() -> Allure.globalError(null, new StatusDetails())).withMessage("status"); + assertThatNullPointerException() + .isThrownBy(() -> Allure.globalError(Status.FAILED, null)).withMessage("statusDetails"); + }); + assertThat(results.getGlobals()).isEmpty(); + } + @Test void shouldAddSteps() { final AllureResults results = runWithinTestContext( diff --git a/allure-java-commons/src/test/java/io/qameta/allure/FileSystemResultsWriterTest.java b/allure-java-commons/src/test/java/io/qameta/allure/FileSystemResultsWriterTest.java index d729f872e..55bcdd882 100644 --- a/allure-java-commons/src/test/java/io/qameta/allure/FileSystemResultsWriterTest.java +++ b/allure-java-commons/src/test/java/io/qameta/allure/FileSystemResultsWriterTest.java @@ -20,6 +20,7 @@ import io.qameta.allure.model.GlobalAttachment; import io.qameta.allure.model.GlobalError; import io.qameta.allure.model.Globals; +import io.qameta.allure.model.Status; import io.qameta.allure.model.StatusDetails; import io.qameta.allure.model.TestResult; import io.qameta.allure.model.TestResultContainer; @@ -27,6 +28,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.MockedStatic; @@ -97,8 +99,12 @@ void shouldWriteTestResultContainer(@TempDir final Path folder) { .isRegularFile(); } - @Test - void shouldWriteGlobals(@TempDir final Path folder) throws IOException { + @ParameterizedTest + @EnumSource( + value = Status.class, + names = {"FAILED", "BROKEN"} + ) + void shouldWriteGlobals(final Status status, @TempDir final Path folder) throws IOException { final FileSystemResultsWriter writer = new FileSystemResultsWriter(folder); final GlobalAttachment attachment = new GlobalAttachment() .setName("setup log") @@ -107,6 +113,7 @@ void shouldWriteGlobals(@TempDir final Path folder) throws IOException { .setSize(12L) .setTimestamp(123L); final GlobalError error = new GlobalError() + .setStatus(status) .setKnown(true) .setMuted(false) .setFlaky(true) @@ -140,7 +147,8 @@ void shouldWriteGlobals(@TempDir final Path folder) throws IOException { assertThat(payload.path("errors").size()).isEqualTo(1); final JsonNode errorPayload = payload.path("errors").path(0); - assertThat(errorPayload.size()).isEqualTo(8); + assertThat(errorPayload.size()).isEqualTo(9); + assertThat(errorPayload.path("status").textValue()).isEqualTo(status.value()); assertThat(errorPayload.path("known").booleanValue()).isTrue(); assertThat(errorPayload.path("muted").booleanValue()).isFalse(); assertThat(errorPayload.path("flaky").booleanValue()).isTrue(); @@ -161,6 +169,7 @@ void shouldWriteGlobals(@TempDir final Path folder) throws IOException { assertThat(written.getErrors()).hasSize(1); final GlobalError writtenError = written.getErrors().get(0); + assertThat(writtenError.getStatus()).isEqualTo(status); assertThat(writtenError.isKnown()).isTrue(); assertThat(writtenError.isMuted()).isFalse(); assertThat(writtenError.isFlaky()).isTrue(); @@ -171,6 +180,17 @@ void shouldWriteGlobals(@TempDir final Path folder) throws IOException { assertThat(writtenError.getTimestamp()).isEqualTo(456L); } + @Test + void shouldDefaultAbsentGlobalErrorStatusToBroken() throws IOException { + final Globals globals = Allure2ModelJackson.createMapper().readValue( + "{\"errors\":[{\"message\":\"run error\",\"timestamp\":123}]}", Globals.class + ); + final GlobalError error = globals.getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(Status.BROKEN); + assertThat(error.getMessage()).isEqualTo("run error"); + assertThat(error.getTimestamp()).isEqualTo(123L); + } + @Test void shouldWriteEachGlobalsArtifactToDistinctFile(@TempDir final Path folder) throws IOException { final FileSystemResultsWriter writer = new FileSystemResultsWriter(folder); diff --git a/allure-java-commons/src/test/java/io/qameta/allure/ResultsUtilsTest.java b/allure-java-commons/src/test/java/io/qameta/allure/ResultsUtilsTest.java index 32c594931..ecc2273e4 100644 --- a/allure-java-commons/src/test/java/io/qameta/allure/ResultsUtilsTest.java +++ b/allure-java-commons/src/test/java/io/qameta/allure/ResultsUtilsTest.java @@ -19,6 +19,7 @@ import io.github.glytching.junit.extension.system.SystemPropertyExtension; import io.qameta.allure.model.GlobalError; import io.qameta.allure.model.Globals; +import io.qameta.allure.model.Status; import io.qameta.allure.model.StatusDetails; import io.qameta.allure.util.ResultsUtils; import org.junit.jupiter.api.Test; @@ -606,6 +607,57 @@ void shouldCreateGlobalErrorWithoutThrowable() { assertThat(error.getTrace()).isNull(); } + @ParameterizedTest + @MethodSource("globalErrorClassifications") + void shouldClassifyGlobalErrorsFromOriginalThrowable(final Throwable throwable, final Status status) { + final GlobalError error = ResultsUtils.createGlobalError("run error", throwable).getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTimestamp()).isPositive(); + } + + static Stream globalErrorClassifications() { + return Stream.of( + Arguments.of(new AssertionError("assertion"), Status.FAILED), + Arguments.of(new AssertionFailedError("comparison", "expected", "actual"), Status.FAILED), + Arguments.of(new RuntimeException("runtime"), Status.BROKEN), + Arguments.of(null, Status.BROKEN) + ); + } + + @ParameterizedTest + @MethodSource("explicitGlobalErrorStatuses") + void shouldNormalizeExplicitGlobalErrorStatus(final Status input, final Status expected) { + final GlobalError error = ResultsUtils.createGlobalError( + "run error", new AssertionFailedError("comparison", "expected", "actual"), input + ).getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(expected); + assertThat(error.getMessage()).isEqualTo("run error: comparison"); + assertThat(error.getTrace()).contains("AssertionFailedError"); + assertThat(error.getActual()).startsWith("actual ("); + assertThat(error.getExpected()).startsWith("expected ("); + } + + static Stream explicitGlobalErrorStatuses() { + return Stream.of( + Arguments.of(Status.FAILED, Status.FAILED), + Arguments.of(Status.BROKEN, Status.BROKEN), + Arguments.of(Status.PASSED, Status.BROKEN), + Arguments.of(Status.SKIPPED, Status.BROKEN), + Arguments.of(null, Status.BROKEN) + ); + } + + @Test + void shouldKeepExplicitStatusWithoutThrowableDetails() { + final GlobalError error = ResultsUtils.createGlobalError("private run error", null, Status.FAILED) + .getErrors().get(0); + assertThat(error.getStatus()).isEqualTo(Status.FAILED); + assertThat(error.getMessage()).isEqualTo("private run error"); + assertThat(error.getTrace()).isNull(); + assertThat(error.getActual()).isNull(); + assertThat(error.getExpected()).isNull(); + } + private static final class LambdaSubject { private final String name; diff --git a/allure-jbehave5/build.gradle.kts b/allure-jbehave5/build.gradle.kts index 17bf720cb..11ebb226f 100644 --- a/allure-jbehave5/build.gradle.kts +++ b/allure-jbehave5/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { testImplementation(project(":allure-assertj")) testImplementation("org.jbehave:jbehave-core:$jbehaveVersion") testImplementation("org.junit.jupiter:junit-jupiter-api") + testImplementation("org.junit.jupiter:junit-jupiter-params") testImplementation("org.mockito:mockito-core") testImplementation("org.slf4j:slf4j-simple") testImplementation(project(":allure-java-commons-test")) diff --git a/allure-jbehave5/src/test/java/io/qameta/allure/jbehave5/AllureJbehave5Test.java b/allure-jbehave5/src/test/java/io/qameta/allure/jbehave5/AllureJbehave5Test.java index e3911bad4..d5f8efc8b 100644 --- a/allure-jbehave5/src/test/java/io/qameta/allure/jbehave5/AllureJbehave5Test.java +++ b/allure-jbehave5/src/test/java/io/qameta/allure/jbehave5/AllureJbehave5Test.java @@ -46,13 +46,19 @@ import org.jbehave.core.embedder.Embedder; import org.jbehave.core.embedder.EmbedderControls; import org.jbehave.core.embedder.NullEmbedderMonitor; +import org.jbehave.core.failures.UUIDExceptionWrapper; import org.jbehave.core.io.LoadFromClasspath; import org.jbehave.core.reporters.NullStoryReporter; import org.jbehave.core.reporters.StoryReporterBuilder; import org.jbehave.core.steps.InjectableStepsFactory; import org.jbehave.core.steps.InstanceStepsFactory; +import org.jbehave.core.steps.StepCollector; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; import java.io.File; import java.nio.file.Path; @@ -61,6 +67,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.qameta.allure.Allure.step; import static io.qameta.allure.util.ResultsUtils.md5; @@ -608,6 +615,50 @@ private AllureResults runStories(final List additionalSteps, final Strin })); } + @ParameterizedTest + @MethodSource("lifecycleGlobalStatuses") + void shouldClassifyUnwrappedLifecycleFailure(final StepCollector.Stage stage, + final Throwable throwable, final Status status) { + final AllureResults results = RunUtils.runTests(lifecycle -> { + final AllureJbehave5 adapter = new AllureJbehave5(lifecycle); + adapter.beforeStoriesSteps(stage); + adapter.failed("lifecycle step", throwable); + adapter.afterStoriesSteps(stage); + }); + final Throwable unwrapped = throwable instanceof UUIDExceptionWrapper ? throwable.getCause() : throwable; + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + if (unwrapped == null) { + assertThat(error.getTrace()).isNull(); + } else { + assertThat(error.getTrace()).contains(unwrapped.getClass().getName()); + assertThat(error.getTrace()).doesNotContain(UUIDExceptionWrapper.class.getName()); + if (unwrapped instanceof AssertionFailedError) { + assertThat(error.getActual()).startsWith("actual ("); + assertThat(error.getExpected()).startsWith("expected ("); + } + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream lifecycleGlobalStatuses() { + return Stream.of(StepCollector.Stage.BEFORE, StepCollector.Stage.AFTER).flatMap( + stage -> Stream.of( + Arguments.of(stage, new AssertionError("assertion"), Status.FAILED), + Arguments.of(stage, new IllegalStateException("runtime"), Status.BROKEN), + Arguments.of( + stage, new UUIDExceptionWrapper( + new AssertionFailedError("comparison", "expected", "actual") + ), Status.FAILED + ), + Arguments.of(stage, new UUIDExceptionWrapper(new IllegalStateException("runtime")), Status.BROKEN), + Arguments.of(stage, new UUIDExceptionWrapper((Throwable) null), Status.BROKEN), + Arguments.of(stage, null, Status.BROKEN) + ) + ); + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java index e79db5f8b..1550ab0a6 100644 --- a/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java +++ b/allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java @@ -333,7 +333,7 @@ public void executionFinished(final TestIdentifier testIdentifier, if (testIdentifier.isTest()) { stopTest(testIdentifier, status, statusDetails); } else if (testExecutionResult.getStatus() != TestExecutionResult.Status.SUCCESSFUL) { - reportContainerFailure(testIdentifier, testExecutionResult); + reportContainerFailure(testIdentifier, testExecutionResult, status); } getLifecycle().writeScope(scopeKey(testIdentifier.getUniqueId())); } @@ -351,7 +351,8 @@ public void executionFinished(final TestIdentifier testIdentifier, * because the event does not say whether some or all of the children completed.

*/ private void reportContainerFailure(final TestIdentifier testIdentifier, - final TestExecutionResult testExecutionResult) { + final TestExecutionResult testExecutionResult, + final Status status) { final String outcome = testExecutionResult.getStatus() == TestExecutionResult.Status.ABORTED ? "was aborted" : "failed"; @@ -362,7 +363,8 @@ private void reportContainerFailure(final TestIdentifier testIdentifier, getLifecycle().writeGlobals( createGlobalError( context, - testExecutionResult.getThrowable().orElse(null) + testExecutionResult.getThrowable().orElse(null), + status ) ); } diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java index 176eb54fc..708856bac 100644 --- a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java @@ -30,6 +30,8 @@ import io.qameta.allure.junitplatform.features.DisabledRepeatedTests; import io.qameta.allure.junitplatform.features.DisabledTests; import io.qameta.allure.junitplatform.features.DynamicTests; +import io.qameta.allure.junitplatform.features.FailedInAfterAllTests; +import io.qameta.allure.junitplatform.features.FailedInBeforeAllTests; import io.qameta.allure.junitplatform.features.FailedTests; import io.qameta.allure.junitplatform.features.FlakyMutedTest; import io.qameta.allure.junitplatform.features.JupiterUniqueIdTest; @@ -91,10 +93,16 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.engine.support.descriptor.EngineDescriptor; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.core.LauncherConfig; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; @@ -104,6 +112,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.qameta.allure.junitplatform.AllureJunitPlatform.JUNIT_PLATFORM_UNIQUE_ID; import static io.qameta.allure.junitplatform.AllureJunitPlatformTestUtils.runClasses; @@ -1663,6 +1672,94 @@ void shouldReportBrokenBeforeAllOfAParameterizedClassAsGlobalError() { ); } + @ParameterizedTest + @MethodSource("classFixtureFailures") + void shouldClassifyClassFixtureGlobalErrors(final Class fixture, final Status status) { + final AllureResults results = runClasses(fixture); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTrace()).isNotBlank(); + }); + } + + static Stream classFixtureFailures() { + return Stream.of( + Arguments.of(FailedInBeforeAllTests.class, Status.FAILED), + Arguments.of(FailedInAfterAllTests.class, Status.FAILED), + Arguments.of(BrokenInBeforeAllTests.class, Status.BROKEN), + Arguments.of(BrokenInAfterAllTests.class, Status.BROKEN), + Arguments.of(SkippedInBeforeAllTests.class, Status.BROKEN) + ); + } + + @ParameterizedTest + @MethodSource("containerOutcomes") + void shouldUseContainerOutcomeForGlobalError(final TestExecutionResult outcome, final Status status) { + final AllureResults results = RunUtils.runTests(lifecycle -> { + final AllureJunitPlatform listener = new AllureJunitPlatform(lifecycle); + final TestIdentifier container = containerIdentifier(); + listener.executionStarted(container); + listener.executionFinished(container, outcome); + }); + assertThat(getGlobalErrors(results)).singleElement().satisfies( + error -> assertThat(error.getStatus()).isEqualTo(status) + ); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream containerOutcomes() { + return Stream.of( + Arguments.of(TestExecutionResult.failed(null), Status.FAILED), + Arguments.of(TestExecutionResult.failed(new AssertionError("assertion")), Status.FAILED), + Arguments.of(TestExecutionResult.failed(new IllegalStateException("runtime")), Status.BROKEN), + Arguments.of(TestExecutionResult.aborted(new AssertionError("abort")), Status.BROKEN) + ); + } + + @ParameterizedTest + @MethodSource("overriddenContainerStatuses") + void shouldHonorContainerStatusOverride(final Throwable throwable, final Status status) { + final AllureResults results = RunUtils.runTests(lifecycle -> { + final AllureJunitPlatform listener = new AllureJunitPlatform(lifecycle) { + @Override + protected Status getStatus(final Throwable error) { + return status; + } + }; + final TestIdentifier container = containerIdentifier(); + listener.executionStarted(container); + listener.executionFinished(container, TestExecutionResult.failed(throwable)); + }); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + }); + } + + static Stream overriddenContainerStatuses() { + return Stream.of( + Arguments.of(new AssertionError("assertion"), Status.BROKEN), + Arguments.of(new IllegalStateException("runtime"), Status.FAILED) + ); + } + + @Test + void shouldNormalizeUnenumeratedSkippedGlobalToBroken() { + final AllureResults results = runClasses(DisabledClassWithTemplates.class); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(Status.BROKEN); + assertThat(error.getTrace()).isNull(); + }); + assertThat(results.getTestResults()).extracting(TestResult::getStatus).containsOnly(Status.SKIPPED); + } + + private static TestIdentifier containerIdentifier() { + final EngineDescriptor engine = new EngineDescriptor(UniqueId.forEngine("junit-jupiter"), "engine"); + final EngineDescriptor container = new EngineDescriptor(engine.getUniqueId().append("class", "Sample"), "sample"); + engine.addChild(container); + return TestIdentifier.from(container); + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInAfterAllTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInAfterAllTests.java new file mode 100644 index 000000000..f75bc1997 --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInAfterAllTests.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform.features; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +public class FailedInAfterAllTests { + + @AfterAll + static void failCleanup() { + throw new AssertionError("assertion in after all"); + } + + @Test + void test() { + } +} diff --git a/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInBeforeAllTests.java b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInBeforeAllTests.java new file mode 100644 index 000000000..4057a1e3d --- /dev/null +++ b/allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/features/FailedInBeforeAllTests.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.junitplatform.features; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class FailedInBeforeAllTests { + + @BeforeAll + static void failSetup() { + throw new AssertionError("assertion in before all"); + } + + @Test + void test() { + } +} diff --git a/allure-junit4/build.gradle.kts b/allure-junit4/build.gradle.kts index 5fc83e7bb..9b79c8922 100644 --- a/allure-junit4/build.gradle.kts +++ b/allure-junit4/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { testImplementation("org.assertj:assertj-core") testImplementation(project(":allure-assertj")) testImplementation("org.junit.jupiter:junit-jupiter-api") + testImplementation("org.junit.jupiter:junit-jupiter-params") testImplementation("org.mockito:mockito-core") testImplementation("org.slf4j:slf4j-simple") testImplementation(project(":allure-java-commons-test")) diff --git a/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4Test.java b/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4Test.java index 063ab9302..6a00d89ab 100644 --- a/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4Test.java +++ b/allure-junit4/src/test/java/io/qameta/allure/junit4/AllureJunit4Test.java @@ -62,13 +62,20 @@ import io.qameta.allure.testfilter.TestPlan; import io.qameta.allure.testfilter.TestPlanV1_0; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.AssumptionViolatedException; +import org.junit.ComparisonFailure; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.runner.JUnitCore; import org.junit.runner.Request; +import org.junit.runner.notification.Failure; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.qameta.allure.junit4.samples.MetadataTagTest.ALLURE_LABEL_TAG; import static io.qameta.allure.junit4.samples.MetadataTagTest.PLAIN_TAG; @@ -792,6 +799,54 @@ void shouldNotInventInitializationErrorForInvalidTestClass() { .contains("Method bad should have no parameters"); } + @ParameterizedTest + @MethodSource("runFailureStatuses") + void shouldClassifyRunFailureGlobals(final boolean initialization, final Throwable throwable, final Status status) { + final org.junit.runner.Description description = initialization + ? org.junit.runner.Description.createTestDescription(AllureJunit4Test.class, "initializationError") + : org.junit.runner.Description.createSuiteDescription(AllureJunit4Test.class); + if (!initialization) { + description.addChild(org.junit.runner.Description.createTestDescription(AllureJunit4Test.class, "test")); + } + final AllureResults results = RunUtils.runTests( + lifecycle -> new AllureJunit4(lifecycle).testFailure(new Failure(description, throwable)) + ); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + if (throwable instanceof ComparisonFailure) { + assertThat(error.getActual()).isEqualTo("actual"); + assertThat(error.getExpected()).isEqualTo("expected"); + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream runFailureStatuses() { + return Stream.of(true, false).flatMap( + initialization -> Stream.of( + Arguments.of(initialization, new AssertionError("assertion"), Status.FAILED), + Arguments.of(initialization, new ComparisonFailure("comparison", "expected", "actual"), Status.FAILED), + Arguments.of(initialization, new IllegalStateException("runtime"), Status.BROKEN), + Arguments.of(initialization, null, Status.BROKEN) + ) + ); + } + + @Test + void shouldClassifyClassAssumptionAsBrokenGlobal() { + final org.junit.runner.Description description = org.junit.runner.Description.createSuiteDescription(AllureJunit4Test.class); + description.addChild(org.junit.runner.Description.createTestDescription(AllureJunit4Test.class, "test")); + final AllureResults results = RunUtils.runTests( + lifecycle -> new AllureJunit4(lifecycle).testAssumptionFailure( + new Failure(description, new AssumptionViolatedException("class assumption")) + ) + ); + assertThat(getGlobalErrors(results)).singleElement().satisfies( + error -> assertThat(error.getStatus()).isEqualTo(Status.BROKEN) + ); + assertThat(results.getTestResults()).isEmpty(); + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java index 23e04c8f1..990de7407 100644 --- a/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java +++ b/allure-karate/src/main/java/io/qameta/allure/karate/AllureKarate.java @@ -354,10 +354,12 @@ private void reportFeatureErrors(final ScenarioResult result, final ReportedScen final StepResult step = steps.get(i); // beforeScenario and afterScenario hooks belong to scenario results, not globals. if (step.isFailed() && !step.isHook()) { + final Status status = getStatus(step.isFailed(), step.getError()); lifecycle.writeGlobals( createGlobalError( redact ? context + ": " + ScenarioResult.SUPPRESSED_FAILURE_MESSAGE : context, - redact ? null : step.getError() + redact ? null : step.getError(), + status ) ); } diff --git a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java index 47a9555c1..94a19f666 100644 --- a/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java +++ b/allure-karate/src/test/java/io/qameta/allure/karate/AllureKarateGlobalErrorsTest.java @@ -15,14 +15,27 @@ */ package io.qameta.allure.karate; +import io.karatelabs.core.FeatureResult; +import io.karatelabs.core.FeatureRunEvent; +import io.karatelabs.core.FeatureRuntime; import io.karatelabs.core.ScenarioResult; +import io.karatelabs.core.StepResult; +import io.karatelabs.gherkin.Feature; +import io.karatelabs.gherkin.Scenario; import io.qameta.allure.Description; import io.qameta.allure.model.GlobalError; +import io.qameta.allure.model.Status; import io.qameta.allure.model.TestResult; import io.qameta.allure.test.AllureResults; +import io.qameta.allure.test.RunUtils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.AssertionFailedError; import java.util.List; +import java.util.stream.Stream; import static io.qameta.allure.model.Status.BROKEN; import static io.qameta.allure.model.Status.FAILED; @@ -216,7 +229,57 @@ void shouldReportOnlyFeatureHookFailuresAsGlobals() { assertThat(error.getTrace()).contains("feature cleanup failed"); } + @ParameterizedTest + @MethodSource("featureErrorStatuses") + void shouldResolveFeatureErrorStatusBeforeRedaction(final Throwable throwable, final Status status, + final boolean redact) { + final AllureResults results = RunUtils.runTests(lifecycle -> { + final Feature feature = Feature.read("classpath:testdata/global-errors/after-feature-failure.feature"); + final ScenarioResult scenario = new ScenarioResult(Scenario.createError(feature, "preparation", 0)); + scenario.setReportDisabled(redact); + scenario.addStepResult(StepResult.fakeFailure("preparation", 0, throwable)); + final FeatureResult result = new FeatureResult(feature); + result.addScenarioResult(scenario); + new AllureKarate(lifecycle).onEvent(FeatureRunEvent.exit(new FeatureRuntime(feature), result)); + }); + assertThat(globalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + if (redact) { + assertThat(error.getMessage()).contains(ScenarioResult.SUPPRESSED_FAILURE_MESSAGE) + .doesNotContain("private message", "private actual", "private expected"); + assertThat(error.getTrace()).isNull(); + assertThat(error.getActual()).isNull(); + assertThat(error.getExpected()).isNull(); + } else if (throwable != null) { + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + assertThat(error.getMessage()).contains(throwable.getMessage()); + if (throwable instanceof AssertionFailedError) { + assertThat(error.getActual()).startsWith("private actual ("); + assertThat(error.getExpected()).startsWith("private expected ("); + } + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream featureErrorStatuses() { + return Stream.of(true, false).flatMap( + redact -> Stream.of( + Arguments.of(new AssertionError("private message"), Status.FAILED, redact), + Arguments.of( + new AssertionFailedError("private message", "private expected", "private actual"), + Status.FAILED, redact + ), + Arguments.of(new IllegalStateException("private message"), Status.BROKEN, redact), + Arguments.of(null, Status.BROKEN, redact) + ) + ); + } + private List globalErrors(final AllureResults results) { - return results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()).toList(); + final List errors = results.getGlobals().stream().flatMap(globals -> globals.getErrors().stream()) + .toList(); + assertThat(errors).allSatisfy(error -> assertThat(error.getStatus()).isIn(FAILED, BROKEN)); + return errors; } } diff --git a/allure-kotlin-extensions/src/main/kotlin/io/qameta/allure/kotlin/extensions/Allure.kt b/allure-kotlin-extensions/src/main/kotlin/io/qameta/allure/kotlin/extensions/Allure.kt index e2fa2500f..dc0b321c2 100644 --- a/allure-kotlin-extensions/src/main/kotlin/io/qameta/allure/kotlin/extensions/Allure.kt +++ b/allure-kotlin-extensions/src/main/kotlin/io/qameta/allure/kotlin/extensions/Allure.kt @@ -139,6 +139,12 @@ public fun globalError(throwable: Throwable): Unit = Allure.globalError(throwabl /** Adds a run-level error with [statusDetails]. */ public fun globalError(statusDetails: StatusDetails): Unit = Allure.globalError(statusDetails) +/** Adds a run-level error with an explicit failed or broken [status] and [statusDetails]. */ +public fun globalError( + status: Status, + statusDetails: StatusDetails, +): Unit = Allure.globalError(status, statusDetails) + /** * Adds UTF-8 [content] as an Allure attachment. * diff --git a/allure-kotlin-extensions/src/test/kotlin/io/qameta/allure/kotlin/extensions/AllureExtensionsTest.kt b/allure-kotlin-extensions/src/test/kotlin/io/qameta/allure/kotlin/extensions/AllureExtensionsTest.kt index 8359dd1fc..abb149e82 100644 --- a/allure-kotlin-extensions/src/test/kotlin/io/qameta/allure/kotlin/extensions/AllureExtensionsTest.kt +++ b/allure-kotlin-extensions/src/test/kotlin/io/qameta/allure/kotlin/extensions/AllureExtensionsTest.kt @@ -322,20 +322,26 @@ internal class AllureExtensionsTest { } @Test - @Description("Both global-error overloads write run-level errors through the configured Allure lifecycle.") + @Description("Global-error overloads write classified run-level errors through the configured Allure lifecycle.") fun shouldWriteThrowableAndStatusDetailsAsGlobalErrors() { val results = RunUtils.runTests( Allure.ThrowableContextRunnableVoid { globalError(IllegalStateException("startup failed")) globalError(StatusDetails().setMessage("environment unavailable").setKnown(true)) + globalError(AssertionError("assertion")) + globalError(Status.FAILED, StatusDetails().setMessage("explicit assertion")) + globalError(Status.BROKEN, StatusDetails().setMessage("explicit runtime")) }, ) val recordedErrors = results.globals.flatMap { it.errors } - assertThat(recordedErrors).hasSize(2) - assertThat(recordedErrors.map { it.message }).containsExactly("startup failed", "environment unavailable") + assertThat(recordedErrors).hasSize(5) + assertThat(recordedErrors.map { it.status }) + .containsExactly(Status.BROKEN, Status.BROKEN, Status.FAILED, Status.FAILED, Status.BROKEN) + assertThat(recordedErrors.map { it.message }) + .containsExactly("startup failed", "environment unavailable", "assertion", "explicit assertion", "explicit runtime") assertThat(recordedErrors[0].trace).contains("IllegalStateException: startup failed") assertThat(recordedErrors[1].isKnown).isTrue() assertThat(recordedErrors.map { it.timestamp }).allSatisfy { timestamp -> diff --git a/allure-model/src/main/java/io/qameta/allure/model/GlobalError.java b/allure-model/src/main/java/io/qameta/allure/model/GlobalError.java index 6590dbd40..ee2d709ad 100644 --- a/allure-model/src/main/java/io/qameta/allure/model/GlobalError.java +++ b/allure-model/src/main/java/io/qameta/allure/model/GlobalError.java @@ -26,8 +26,29 @@ public class GlobalError extends StatusDetails { private static final long serialVersionUID = 1L; + private Status status = Status.BROKEN; private long timestamp; + /** + * Gets the error status. + * + * @return the status, normally failed or broken + */ + public Status getStatus() { + return status; + } + + /** + * Sets the error status. + * + * @param value the status, normally failed or broken + * @return self for method chaining + */ + public GlobalError setStatus(final Status value) { + this.status = value; + return this; + } + /** * Gets the time when the error was added. * @@ -127,6 +148,7 @@ public boolean equals(final Object o) { } final GlobalError that = (GlobalError) o; return timestamp == that.timestamp + && status == that.status && isKnown() == that.isKnown() && isMuted() == that.isMuted() && isFlaky() == that.isFlaky(); @@ -137,6 +159,6 @@ && isMuted() == that.isMuted() */ @Override public int hashCode() { - return Objects.hash(super.hashCode(), isKnown(), isMuted(), isFlaky(), timestamp); + return Objects.hash(super.hashCode(), isKnown(), isMuted(), isFlaky(), status, timestamp); } } diff --git a/allure-model/src/test/java/io/qameta/allure/model/GlobalErrorTest.java b/allure-model/src/test/java/io/qameta/allure/model/GlobalErrorTest.java index e843f19ef..3e051bc19 100644 --- a/allure-model/src/test/java/io/qameta/allure/model/GlobalErrorTest.java +++ b/allure-model/src/test/java/io/qameta/allure/model/GlobalErrorTest.java @@ -21,6 +21,30 @@ class GlobalErrorTest { + @Test + void shouldDefaultGlobalErrorToBroken() { + assertThat(new GlobalError().getStatus()).isEqualTo(Status.BROKEN); + } + + @Test + void shouldKeepExplicitGlobalErrorStatus() { + final GlobalError error = new GlobalError(); + assertThat(error.setStatus(Status.FAILED)).isSameAs(error); + assertThat(error.getStatus()).isEqualTo(Status.FAILED); + assertThat(error.setStatus(Status.BROKEN)).isSameAs(error); + assertThat(error.getStatus()).isEqualTo(Status.BROKEN); + } + + @Test + void shouldIncludeStatusInGlobalErrorValueSemantics() { + final GlobalError failed = globalError("error", 123L).setStatus(Status.FAILED); + final GlobalError equalError = globalError("error", 123L).setStatus(Status.FAILED); + final GlobalError broken = globalError("error", 123L).setStatus(Status.BROKEN); + + assertThat(failed).isEqualTo(equalError).isNotEqualTo(broken); + assertThat(failed.hashCode()).isEqualTo(equalError.hashCode()); + } + @Test void shouldCompareStatusDetailsAndTimestamp() { final GlobalError error = globalError("global setup failed", 123L); diff --git a/allure-scalatest/src/main/scala/io/qameta/allure/scalatest/AllureScalatest.scala b/allure-scalatest/src/main/scala/io/qameta/allure/scalatest/AllureScalatest.scala index 0aef8a5d0..844c4bc00 100644 --- a/allure-scalatest/src/main/scala/io/qameta/allure/scalatest/AllureScalatest.scala +++ b/allure-scalatest/src/main/scala/io/qameta/allure/scalatest/AllureScalatest.scala @@ -120,7 +120,7 @@ class AllureScalatest(val lifecycle: AllureLifecycle) extends Reporter { private def reportAbort(context: String, message: String, throwable: Option[Throwable]): Unit = { val errorContext = if (throwable.exists(_.getMessage == message)) context else s"$context: $message" - lifecycle.writeGlobals(createGlobalError(errorContext, throwable.orNull)) + lifecycle.writeGlobals(createGlobalError(errorContext, throwable.orNull, getStatus(throwable.orNull))) } def startTest(event: TestStarting): Unit = { @@ -134,12 +134,14 @@ class AllureScalatest(val lifecycle: AllureLifecycle) extends Reporter { ) } + private def getStatus(throwable: Throwable): Status = throwable match { + case _: TestFailedException => Status.FAILED + case _ => Status.BROKEN + } + def failTestCase(event: TestFailed): Unit = { val throwable = event.throwable.getOrElse(new RuntimeException(event.message)) - val status = throwable match { - case _: TestFailedException => Status.FAILED - case _ => Status.BROKEN - } + val status = getStatus(throwable) val statusDetails = getStatusDetails(throwable) .orElse(new StatusDetails().setMessage(event.message)) diff --git a/allure-scalatest/src/test/scala/io/qameta/allure/scalatest/AllureScalatestTest.scala b/allure-scalatest/src/test/scala/io/qameta/allure/scalatest/AllureScalatestTest.scala index fedb1b904..7a9295499 100644 --- a/allure-scalatest/src/test/scala/io/qameta/allure/scalatest/AllureScalatestTest.scala +++ b/allure-scalatest/src/test/scala/io/qameta/allure/scalatest/AllureScalatestTest.scala @@ -251,6 +251,7 @@ class AllureScalatestTest { val error = errors.head error.getMessage should include("BeforeAllFailureSpec") error.getMessage should include("beforeAll failed") + error.getStatus shouldBe Status.BROKEN error.getTrace should include("java.lang.IllegalStateException: beforeAll failed") error.getTrace should include("BeforeAllFailureSpec.beforeAll") error.getTimestamp should be >= started @@ -268,6 +269,7 @@ class AllureScalatestTest { val error = errors.head error.getMessage should include("AfterAllFailureSpec") error.getMessage should include("afterAll failed") + error.getStatus shouldBe Status.BROKEN error.getTrace should include("java.lang.IllegalStateException: afterAll failed") error.getTrace should include("AfterAllFailureSpec.afterAll") } @@ -284,6 +286,7 @@ class AllureScalatestTest { error.getMessage should include("ScalaTest run") error.getTrace should include("ConstructorFailureSpec") error.getTrace should include("suite construction failed") + error.getStatus shouldBe Status.BROKEN } @Test @@ -305,6 +308,7 @@ class AllureScalatestTest { val error = errors.head error.getMessage should include("fixture suite") error.getMessage should include("suite initialization was interrupted") + error.getStatus shouldBe Status.BROKEN error.getTrace shouldBe null error.getTimestamp should be > 0L } @@ -326,6 +330,7 @@ class AllureScalatestTest { val error = errors.head error.getMessage should include("ScalaTest run") error.getMessage should include("test discovery was interrupted") + error.getStatus shouldBe Status.BROKEN error.getTrace shouldBe null error.getTimestamp should be > 0L } @@ -349,10 +354,44 @@ class AllureScalatestTest { error.getMessage should include("ScalaTest initialization failed") error.getMessage should include("global comparison failed") error.getTrace should include("org.opentest4j.AssertionFailedError: global comparison failed") + error.getStatus shouldBe Status.BROKEN error.getExpected shouldBe cause.getExpected.toString error.getActual shouldBe cause.getActual.toString } + @Test + def shouldClassifyScalaTestAssertionsAsFailedAborts(): Unit = { + val cause = org.scalatest.Assertions.intercept[org.scalatest.exceptions.TestFailedException] { + org.scalatest.Assertions.fail("ScalaTest assertion") + } + checkAbortStatuses(cause, Status.FAILED) + } + + @Test + def shouldClassifyRawAssertionsAsBrokenAborts(): Unit = { + checkAbortStatuses(new AssertionError("raw assertion"), Status.BROKEN) + } + + @Test + def shouldClassifyRuntimeExceptionsAsBrokenAborts(): Unit = { + checkAbortStatuses(new IllegalStateException("runtime failure"), Status.BROKEN) + } + + private def checkAbortStatuses(cause: Throwable, status: Status): Unit = { + val events = Seq( + SuiteAborted(new Ordinal(0), "suite aborted", "fixture suite", "fixture-suite", None, throwable = Some(cause)), + RunAborted(new Ordinal(0), "run aborted", throwable = Some(cause)) + ) + events.foreach { event => + val results = report(event) + val errors = results.getGlobals.asScala.flatMap(_.getErrors.asScala) + errors should have length 1 + errors.head.getStatus shouldBe status + errors.head.getTrace should include(cause.getClass.getName) + results.getTestResults should have length 0 + } + } + private def report(event: Event): AllureResults = { val results = new AllureResultsWriterStub() val reporter = new AllureScalatest(new AllureLifecycle(results)) diff --git a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java index dd256429e..13c814a3d 100644 --- a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java +++ b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/AllureSpock2Test.java @@ -29,6 +29,8 @@ import io.qameta.allure.model.TestResult; import io.qameta.allure.model.TestResultContainer; import io.qameta.allure.spock2.samples.ActualExpectedStatusDetailsTest; +import io.qameta.allure.spock2.samples.AssertedCleanupSpec; +import io.qameta.allure.spock2.samples.AssertedSetupSpec; import io.qameta.allure.spock2.samples.BlockFailureTest; import io.qameta.allure.spock2.samples.BrokenTest; import io.qameta.allure.spock2.samples.DataDrivenTest; @@ -66,6 +68,9 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Triple; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.platform.commons.util.ReflectionUtils; import org.junit.platform.engine.discovery.ClassSelector; import org.junit.platform.engine.discovery.DiscoverySelectors; @@ -1172,6 +1177,36 @@ void shouldNotInventIterationsForDataDrivenFeaturesWhenSetupSpecFails() { ); } + @ParameterizedTest + @MethodSource("specGlobalStatuses") + void shouldUseFixtureClassificationForSpecGlobals(final Class spec, final Status status, + final Status testStatus) { + final AllureResults results = runClasses(spec); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTrace()).isNotEmpty(); + }); + assertThat(results.getTestResults()).isNotEmpty().allSatisfy( + test -> assertThat(test.getStatus()).isEqualTo(testStatus) + ); + final List failedFixtures = results.getTestResultContainers().stream() + .flatMap(container -> Stream.concat(container.getBefores().stream(), container.getAfters().stream())) + .filter(fixture -> fixture.getName().equals("setup spec") || fixture.getName().equals("cleanup spec")) + .collect(Collectors.toList()); + assertThat(failedFixtures).singleElement().satisfies( + fixture -> assertThat(fixture.getStatus()).isEqualTo(status) + ); + } + + static Stream specGlobalStatuses() { + return Stream.of( + Arguments.of(AssertedSetupSpec.class, Status.FAILED, Status.SKIPPED), + Arguments.of(AssertedCleanupSpec.class, Status.FAILED, Status.PASSED), + Arguments.of(FailedSetupSpec.class, Status.BROKEN, Status.SKIPPED), + Arguments.of(FailedCleanupSpec.class, Status.BROKEN, Status.PASSED) + ); + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedCleanupSpec.groovy b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedCleanupSpec.groovy new file mode 100644 index 000000000..964f8a9b3 --- /dev/null +++ b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedCleanupSpec.groovy @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.spock2.samples + +import spock.lang.Specification + +class AssertedCleanupSpec extends Specification { + + def cleanupSpec() { + assert false: "cleanup spec assertion" + } + + def "regular feature"() { + expect: + true + } +} diff --git a/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedSetupSpec.groovy b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedSetupSpec.groovy new file mode 100644 index 000000000..883ee006b --- /dev/null +++ b/allure-spock2/src/test/groovy/io/qameta/allure/spock2/samples/AssertedSetupSpec.groovy @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.spock2.samples + +import spock.lang.Specification + +class AssertedSetupSpec extends Specification { + + def setupSpec() { + assert false: "setup spec assertion" + } + + def "regular feature"() { + expect: + true + } +} diff --git a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java index c66503863..72a81fb28 100644 --- a/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java +++ b/allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java @@ -682,7 +682,8 @@ private boolean reportUnknownInvocations(final ITestResult result) { + "have had are unknown", getQualifiedName(method) ), - result.getThrowable() + result.getThrowable(), + getStatus(result.getThrowable()) ) ); return true; @@ -955,7 +956,7 @@ public void onConfigurationFailure(final ITestResult itr) { getQualifiedName(method) ); - getLifecycle().writeGlobals(createGlobalError(context, itr.getThrowable())); + getLifecycle().writeGlobals(createGlobalError(context, itr.getThrowable(), getStatus(itr.getThrowable()))); } /** diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java index cb452e2da..33a388859 100644 --- a/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java +++ b/allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java @@ -40,8 +40,12 @@ import io.qameta.allure.testfilter.TestPlan; import io.qameta.allure.testfilter.TestPlanV1_0; import io.qameta.allure.testng.config.AllureTestNgConfig; +import io.qameta.allure.testng.samples.AssertedAfterSuite; +import io.qameta.allure.testng.samples.AssertedBeforeSuite; import io.qameta.allure.testng.samples.CustomListenerAttachments; +import io.qameta.allure.testng.samples.FailedAfterSuite; import io.qameta.allure.testng.samples.FailedBeforeClassWithDataProvider; +import io.qameta.allure.testng.samples.FailedBeforeSuite; import io.qameta.allure.testng.samples.FailedDependency; import io.qameta.allure.testng.samples.InheritedDataProviderDependency; import io.qameta.allure.testng.samples.PriorityTests; @@ -1800,6 +1804,82 @@ private static Map identitiesByHistoryId(final AllureResults res ); } + @ParameterizedTest + @MethodSource("suiteGlobalFailures") + void shouldClassifySuiteGlobalErrors(final Class fixture, final Status status) { + final AllureResults results = runTestPlan(null, fixture); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(status); + assertThat(error.getTrace()).isNotBlank(); + }); + } + + static Stream suiteGlobalFailures() { + return Stream.of( + Arguments.of(AssertedBeforeSuite.class, Status.FAILED), + Arguments.of(AssertedAfterSuite.class, Status.FAILED), + Arguments.of(FailedBeforeSuite.class, Status.BROKEN), + Arguments.of(FailedAfterSuite.class, Status.BROKEN) + ); + } + + @ParameterizedTest + @MethodSource("globalCallbackStatuses") + void shouldUseTestNgClassifierForGlobalCallbacks(final int callback, final Throwable throwable, + final Status override, final Status expected) + throws Exception { + final ITestNGMethod method = mock(ITestNGMethod.class); + when(method.getRealClass()).thenReturn(AllureTestNgTest.class); + when(method.getMethodName()).thenReturn("valueDependentTest"); + when(method.getGroups()).thenReturn(new String[0]); + when(method.getConstructorOrMethod()).thenReturn( + new ConstructorOrMethod(getClass().getMethod("valueDependentTest", String.class)) + ); + final ITestResult result = mock(ITestResult.class); + when(result.getMethod()).thenReturn(method); + when(result.getParameters()).thenReturn(new Object[0]); + when(result.getThrowable()).thenReturn(throwable); + final AllureResults results = RunUtils.runTests(lifecycle -> { + final AllureTestNg adapter = new AllureTestNg(lifecycle) { + @Override + protected Status getStatus(final Throwable error) { + return override == null ? super.getStatus(error) : override; + } + }; + if (callback == 0) { + adapter.onConfigurationFailure(result); + } else if (callback == 1) { + adapter.onTestSkipped(result); + } else { + adapter.onTestFailure(result); + } + }); + assertThat(getGlobalErrors(results)).singleElement().satisfies(error -> { + assertThat(error.getStatus()).isEqualTo(expected); + if (throwable == null) { + assertThat(error.getTrace()).isNull(); + } else { + assertThat(error.getTrace()).contains(throwable.getClass().getName()); + } + }); + assertThat(results.getTestResults()).isEmpty(); + } + + static Stream globalCallbackStatuses() { + return Stream.of(0, 1, 2).flatMap( + callback -> Stream.of( + Arguments.of(callback, new AssertionError("assertion"), null, Status.FAILED), + Arguments.of(callback, new IllegalStateException("runtime"), null, Status.BROKEN), + Arguments.of(callback, null, null, Status.BROKEN), + Arguments.of(callback, new AssertionError("override"), Status.BROKEN, Status.BROKEN), + Arguments.of(callback, new IllegalStateException("override"), Status.FAILED, Status.FAILED) + ) + ); + } + + public void valueDependentTest(final String value) { + } + private static List getGlobalErrors(final AllureResults results) { return results.getGlobals().stream() .flatMap(globals -> globals.getErrors().stream()) diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedAfterSuite.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedAfterSuite.java new file mode 100644 index 000000000..37078a829 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedAfterSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.AfterSuite; +import org.testng.annotations.Test; + +public class AssertedAfterSuite { + + @AfterSuite + public void failCleanup() { + throw new AssertionError("suite cleanup assertion"); + } + + @Test + public void test() { + } +} diff --git a/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedBeforeSuite.java b/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedBeforeSuite.java new file mode 100644 index 000000000..519730156 --- /dev/null +++ b/allure-testng/src/test/java/io/qameta/allure/testng/samples/AssertedBeforeSuite.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016-2026 Qameta Software Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.qameta.allure.testng.samples; + +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.Test; + +public class AssertedBeforeSuite { + + @BeforeSuite + public void failSetup() { + throw new AssertionError("suite setup assertion"); + } + + @Test + public void test() { + } +}