diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java index ccd826c201..c765d2ebc4 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/util/FailsafeSummaryXmlUtils.java @@ -35,6 +35,7 @@ import org.xml.sax.InputSource; import static java.lang.Boolean.parseBoolean; +import static java.lang.Float.parseFloat; import static java.lang.Integer.parseInt; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; @@ -48,6 +49,8 @@ * @since 2.20 */ public final class FailsafeSummaryXmlUtils { + private static final float ONE_SECOND = 1000.0f; + private static final String FAILSAFE_SUMMARY_XML_SCHEMA_LOCATION = "https://maven.apache.org/surefire/maven-surefire-plugin/xsd/failsafe-summary.xsd"; @@ -56,6 +59,7 @@ public final class FailsafeSummaryXmlUtils { private static final String MESSAGE_ELEMENT = "%s"; + // TODO What happened to flakes? private static final String FAILSAFE_SUMMARY_XML_TEMPLATE = "\n" + "%d\n" + " %d\n" + " %d\n" + + " \n" + " %s\n" + ""; @@ -82,6 +87,7 @@ public static RunResult toRunResult(File failsafeSummaryXml) throws Exception { String errors = xpath.evaluate("/failsafe-summary/errors", root); String failures = xpath.evaluate("/failsafe-summary/failures", root); String skipped = xpath.evaluate("/failsafe-summary/skipped", root); + String elapsed = xpath.evaluate("/failsafe-summary/time", root); String failureMessage = xpath.evaluate("/failsafe-summary/failureMessage", root); String timeout = xpath.evaluate("/failsafe-summary/@timeout", root); @@ -90,6 +96,8 @@ public static RunResult toRunResult(File failsafeSummaryXml) throws Exception { parseInt(errors), parseInt(failures), parseInt(skipped), + 0, + isBlank(elapsed) ? null : ((int) (parseFloat(elapsed) * ONE_SECOND)), isBlank(failureMessage) ? null : unescapeXml(failureMessage), parseBoolean(timeout)); } @@ -107,6 +115,7 @@ public static void fromRunResultToFile(RunResult fromRunResult, File toFailsafeS fromRunResult.getErrors(), fromRunResult.getFailures(), fromRunResult.getSkipped(), + fromRunResult.getElapsed() != null ? String.valueOf(fromRunResult.getElapsed() / ONE_SECOND) : "", msg); try (FileOutputStream os = new FileOutputStream(toFailsafeSummaryXml)) { diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/MarshallerUnmarshallerTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/MarshallerUnmarshallerTest.java index 1872bda854..c26a097fa7 100644 --- a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/MarshallerUnmarshallerTest.java +++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/MarshallerUnmarshallerTest.java @@ -32,6 +32,7 @@ */ public class MarshallerUnmarshallerTest { @Test + @SuppressWarnings("checkstyle:MagicNumber") public void shouldUnmarshallExistingXmlFile() throws Exception { File xml = new File("target/test-classes/org/apache/maven/plugin/failsafe/failsafe-summary.xml"); RunResult summary = FailsafeSummaryXmlUtils.toRunResult(xml); @@ -44,6 +45,8 @@ public void shouldUnmarshallExistingXmlFile() throws Exception { assertThat(summary.getSkipped()).isEqualTo(3); + assertThat(summary.getElapsed()).isEqualTo(1500); + assertThat(summary.getFailure()) .contains("There was an error in the forked processtest " + "subsystem#no method RuntimeException Hi There!"); @@ -56,6 +59,7 @@ public void shouldUnmarshallExistingXmlFile() throws Exception { } @Test + @SuppressWarnings("checkstyle:MagicNumber") public void shouldMarshallAndUnmarshallSameXml() throws Exception { RunResult expected = new RunResult( 7, @@ -63,6 +67,7 @@ public void shouldMarshallAndUnmarshallSameXml() throws Exception { 2, 3, 2, + 1500, "There was an error in the forked processtest " + "subsystem#no method RuntimeException Hi There! $&>>" + "\n\tat org.apache.maven.plugin.surefire.booterclient.ForkStarter" @@ -84,6 +89,8 @@ public void shouldMarshallAndUnmarshallSameXml() throws Exception { assertThat(actual.getFailures()).isEqualTo(expected.getFailures()); + assertThat(actual.getElapsed()).isEqualTo(expected.getElapsed()); + assertThat(actual.getSkipped()).isEqualTo(expected.getSkipped()); assertThat(actual.getFailure()).isEqualTo(expected.getFailure()); diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java index 1bd4145bc1..8c42ddd4bd 100644 --- a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java +++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/RunResultTest.java @@ -56,18 +56,18 @@ public void testSerialization() throws Exception { @Test public void testFailures() throws Exception { - writeReadCheck(new RunResult(0, 1, 2, 3, "stacktraceHere", false)); + writeReadCheck(new RunResult(0, 1, 2, 3, 220, "stacktraceHere", false)); } @Test public void testSkipped() throws Exception { - writeReadCheck(new RunResult(3, 2, 1, 0, null, true)); + writeReadCheck(new RunResult(3, 2, 1, 0, 50, null, true)); } @Test public void testAppendSerialization() throws Exception { RunResult simpleAggregate = getSimpleAggregate(); - RunResult additional = new RunResult(2, 1, 2, 2, "msg " + ((char) 0x0E01), true); + RunResult additional = new RunResult(2, 1, 2, 2, 500, "msg " + ((char) 0x0E01), true); File summary = SureFireFileManager.createTempFile("failsafe", "test"); FailsafeSummaryXmlUtils.writeSummary(simpleAggregate, summary, false); @@ -88,6 +88,8 @@ public void testAppendSerialization() throws Exception { assertThat(expected.getFlakes()).isEqualTo(2); + assertThat(expected.getElapsed()).isEqualTo(500); + assertThat(expected.getFailure()).isEqualTo("msg " + ((char) 0x0E01)); assertThat(expected.isTimeout()).isTrue(); diff --git a/maven-failsafe-plugin/src/test/resources/org/apache/maven/plugin/failsafe/failsafe-summary.xml b/maven-failsafe-plugin/src/test/resources/org/apache/maven/plugin/failsafe/failsafe-summary.xml index 2b15bca782..fcdd918736 100644 --- a/maven-failsafe-plugin/src/test/resources/org/apache/maven/plugin/failsafe/failsafe-summary.xml +++ b/maven-failsafe-plugin/src/test/resources/org/apache/maven/plugin/failsafe/failsafe-summary.xml @@ -1 +1 @@ - 7 1 2 3 \ No newline at end of file + 7 1 2 3 diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java index 762e1348cc..66f205a94a 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java @@ -263,7 +263,7 @@ private void mergeTestHistoryResult() { } // Update globalStatistics by iterating through mergedTestHistoryResult - int completedCount = 0, skipped = 0; + int completedCount = 0, skipped = 0, elapsed = -1; for (Map.Entry> entry : mergedTestHistoryResult.entrySet()) { List testMethodStats = entry.getValue(); @@ -273,6 +273,12 @@ private void mergeTestHistoryResult() { List resultTypes = new ArrayList<>(); for (TestMethodStats methodStats : testMethodStats) { resultTypes.add(methodStats.getResultType()); + if (methodStats.getElapsed() != null) { + if (elapsed == -1) { + elapsed = 0; + } + elapsed += methodStats.getElapsed(); + } } switch (getTestResultType(resultTypes, reportConfiguration.getRerunFailingTestsCount())) { @@ -303,7 +309,7 @@ private void mergeTestHistoryResult() { } } - globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size()); + globalStats.set(completedCount, errorTests.size(), failedTests.size(), skipped, flakyTests.size(), (elapsed != -1 ? elapsed : null)); } /** diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java index d3abb53a79..6033509c92 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestMethodStats.java @@ -34,10 +34,13 @@ public class TestMethodStats { private final StackTraceWriter stackTraceWriter; - public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter) { + private final Integer elapsed; + + public TestMethodStats(String testClassMethodName, ReportEntryType resultType, StackTraceWriter stackTraceWriter, Integer elapsed) { this.testClassMethodName = testClassMethodName; this.resultType = resultType; this.stackTraceWriter = stackTraceWriter; + this.elapsed = elapsed; } public String getTestClassMethodName() { @@ -51,4 +54,8 @@ public ReportEntryType getResultType() { public StackTraceWriter getStackTraceWriter() { return stackTraceWriter; } + + public Integer getElapsed() { + return elapsed; + } } diff --git a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java index 5e90fb0559..391bc0d949 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/TestSetRunListener.java @@ -288,7 +288,8 @@ private void addTestMethodStats() { TestMethodStats methodStats = new TestMethodStats( reportEntry.getClassMethodName(), reportEntry.getReportEntryType(), - reportEntry.getStackTraceWriter()); + reportEntry.getStackTraceWriter(), + reportEntry.getElapsed()); testMethodStats.add(methodStats); } } diff --git a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java index 00abbf2482..6436b88923 100644 --- a/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java +++ b/maven-surefire-common/src/main/java/org/apache/maven/surefire/report/RunStatistics.java @@ -18,12 +18,25 @@ */ package org.apache.maven.surefire.report; +import java.text.MessageFormat; +import java.util.Locale; + import org.apache.maven.surefire.api.suite.RunResult; /** * @author Kristian Rosenvold */ public final class RunStatistics { + private static final float ONE_SECOND = 1000.0f; + + /* + * Rationale: The idea is to always display four digits for visually consistent output + * Important: Keep in sync with maven-surefire-report-plugin/src/main/resources/surefire-report.properties + */ + private final MessageFormat elapsedTimeFormat = new MessageFormat( + "{0,choice,0#0|0.0<{0,number,0.000}|10#{0,number,0.00}|100#{0,number,0.0}|1000#{0,number,0}} s", + Locale.ROOT); + private int completedCount; private int errors; @@ -34,6 +47,8 @@ public final class RunStatistics { private int flakes; + private Integer elapsed; + public synchronized int getCompletedCount() { return completedCount; } @@ -54,12 +69,17 @@ public synchronized int getFlakes() { return flakes; } - public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes) { + public synchronized Integer getElapsed() { + return elapsed; + } + + public synchronized void set(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) { this.completedCount = completedCount; this.errors = errors; this.failures = failures; this.skipped = skipped; this.flakes = flakes; + this.elapsed = elapsed; } public synchronized RunResult getRunResult() { @@ -72,6 +92,7 @@ public synchronized String getSummary() { if (flakes > 0) { summary += ", Flakes: " + flakes; } + summary += ", Time elapsed: " + (elapsed != null ? elapsedTimeFormat.format(new Object[] {elapsed / ONE_SECOND}): "(unknown)"); return summary; } } diff --git a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java index 9d4c9f924a..11e5e59c14 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java @@ -72,6 +72,7 @@ public class DefaultReporterFactoryTest extends TestCase { private static final String ERROR = "error"; + @SuppressWarnings("checkstyle:MagicNumber") public void testMergeTestHistoryResult() throws Exception { MessageUtils.setColorEnabled(false); File target = new File(System.getProperty("user.dir"), "target"); @@ -100,26 +101,28 @@ public void testMergeTestHistoryResult() throws Exception { // First run, four tests failed and one passed Queue firstRunStats = new ArrayDeque<>(); - firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR))); - firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR))); + //CHECKSTYLE:OFF + firstRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null)); + firstRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null)); firstRunStats.add( - new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL))); + new TestMethodStats(TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50)); firstRunStats.add( - new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL))); - firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null)); + new TestMethodStats(TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50)); + firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null, 50)); // Second run, two tests passed Queue secondRunStats = new ArrayDeque<>(); secondRunStats.add( - new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL))); - secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null)); - secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR))); - secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null)); + new TestMethodStats(TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter(ASSERTION_FAIL), 50)); + secondRunStats.add(new TestMethodStats(TEST_TWO, ReportEntryType.SUCCESS, null, 25)); + secondRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), 50)); + secondRunStats.add(new TestMethodStats(TEST_FOUR, ReportEntryType.SUCCESS, null, 25)); // Third run, another test passed Queue thirdRunStats = new ArrayDeque<>(); - thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null)); - thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR))); + thirdRunStats.add(new TestMethodStats(TEST_ONE, ReportEntryType.SUCCESS, null, 10)); + thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR), null)); + //CHECKSTYLE:ON TestSetRunListener firstRunListener = mock(TestSetRunListener.class); TestSetRunListener secondRunListener = mock(TestSetRunListener.class); @@ -402,7 +405,7 @@ public void testCreateReporterWithZeroStatistics() { assertEquals(0, statistics.getErrors()); assertEquals(0, statistics.getSkipped()); assertEquals(0, statistics.getFlakes()); - assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary()); + assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", statistics.getSummary()); assertEquals(0, statistics.getCompletedCount()); List messages = reporter.getMessages(); @@ -413,7 +416,7 @@ public void testCreateReporterWithZeroStatistics() { assertEquals("", messages.get(4)); assertEquals("Results:", messages.get(5)); assertEquals("", messages.get(6)); - assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get(7)); + assertEquals("Tests run: 0, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: (unknown)", messages.get(7)); assertEquals("", messages.get(8)); assertEquals(9, messages.size()); } diff --git a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java index a3206e00a3..eb68619666 100644 --- a/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java +++ b/maven-surefire-common/src/test/java/org/apache/maven/surefire/report/RunStatisticsTest.java @@ -23,10 +23,11 @@ /** * */ +@SuppressWarnings("checkstyle:MagicNumber") public class RunStatisticsTest extends TestCase { public void testSetRunStatistics() { RunStatistics statistics = new RunStatistics(); - statistics.set(10, 5, 2, 1, 2); + statistics.set(10, 5, 2, 1, 2, 5500); assertEquals(10, statistics.getCompletedCount()); assertEquals(5, statistics.getErrors()); assertEquals(2, statistics.getFailures()); diff --git a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java index ceb182aee6..fee3412123 100644 --- a/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java +++ b/surefire-api/src/main/java/org/apache/maven/surefire/api/suite/RunResult.java @@ -40,6 +40,8 @@ public class RunResult { private final int flakes; + private final Integer elapsed; + private final String failure; private final boolean timeout; @@ -58,37 +60,44 @@ public static RunResult failure(RunResult accumulatedAtTimeout, Exception cause) return errorCode(accumulatedAtTimeout, getStackTrace(cause), accumulatedAtTimeout.isTimeout()); } + // TODO What happened to flakes? private static RunResult errorCode(RunResult other, String failure, boolean timeout) { return new RunResult( other.getCompletedCount(), other.getErrors(), other.getFailures(), other.getSkipped(), + other.getElapsed(), failure, timeout); } public RunResult(int completedCount, int errors, int failures, int skipped) { - this(completedCount, errors, failures, skipped, null, false); + this(completedCount, errors, failures, skipped, null, null, false); } public RunResult(int completedCount, int errors, int failures, int skipped, int flakes) { - this(completedCount, errors, failures, skipped, flakes, null, false); + this(completedCount, errors, failures, skipped, flakes, null, null, false); + } + + public RunResult(int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed) { + this(completedCount, errors, failures, skipped, flakes, elapsed, null, false); } - public RunResult(int completedCount, int errors, int failures, int skipped, String failure, boolean timeout) { - this(completedCount, errors, failures, skipped, 0, failure, timeout); + public RunResult(int completedCount, int errors, int failures, int skipped, Integer elapsed, String failure, boolean timeout) { + this(completedCount, errors, failures, skipped, 0, elapsed, failure, timeout); } public RunResult( - int completedCount, int errors, int failures, int skipped, int flakes, String failure, boolean timeout) { + int completedCount, int errors, int failures, int skipped, int flakes, Integer elapsed, String failure, boolean timeout) { this.completedCount = completedCount; this.errors = errors; this.failures = failures; this.skipped = skipped; + this.flakes = flakes; + this.elapsed = elapsed; this.failure = failure; this.timeout = timeout; - this.flakes = flakes; } private static String getStackTrace(Exception e) { @@ -122,6 +131,10 @@ public int getSkipped() { return skipped; } + public Integer getElapsed() { + return elapsed; + } + public Integer getFailsafeCode() // Only used for compatibility reasons. { if (completedCount == 0) { @@ -167,13 +180,24 @@ public RunResult aggregate(RunResult other) { int ign = getSkipped() + other.getSkipped(); int err = getErrors() + other.getErrors(); int flakes = getFlakes() + other.getFlakes(); - return new RunResult(completed, err, fail, ign, flakes, failureMessage, timeout); + Integer elapsed; + if (getElapsed() == null) { + elapsed = other.getElapsed(); + } else { + if (other.getElapsed() == null) { + elapsed = getElapsed(); + } else { + elapsed = getElapsed() + other.getElapsed(); + } + } + return new RunResult(completed, err, fail, ign, flakes, elapsed, failureMessage, timeout); } public static RunResult noTestsRun() { return new RunResult(0, 0, 0, 0); } + // TODO What happened to flakes? @Override @SuppressWarnings("RedundantIfStatement") public boolean equals(Object o) { @@ -208,6 +232,7 @@ public boolean equals(Object o) { return true; } + // TODO What happened to flakes? @Override public int hashCode() { int result = completedCount;