diff --git a/karate-core/src/main/java/com/intuit/karate/Results.java b/karate-core/src/main/java/com/intuit/karate/Results.java index fa7d06bb9..cf5270672 100644 --- a/karate-core/src/main/java/com/intuit/karate/Results.java +++ b/karate-core/src/main/java/com/intuit/karate/Results.java @@ -28,6 +28,7 @@ import com.intuit.karate.core.TagResults; import com.intuit.karate.core.TimelineResults; import com.intuit.karate.report.ReportUtils; + import java.io.File; import java.util.ArrayList; import java.util.HashMap; @@ -37,7 +38,6 @@ import java.util.stream.Stream; /** - * * @author pthomas3 */ public class Results { @@ -48,6 +48,8 @@ public class Results { private final int featuresSkipped; private final int scenariosPassed; private final int scenariosFailed; + private final int scenariosSkipped; + private final int scenariosTotal; private final double timeTakenMillis; private final long endTime; private final List errors = new ArrayList(); @@ -66,6 +68,8 @@ private Results(Suite suite) { AtomicInteger ff = new AtomicInteger(); AtomicInteger sp = new AtomicInteger(); AtomicInteger sf = new AtomicInteger(); + AtomicInteger ss = new AtomicInteger(); + AtomicInteger st = new AtomicInteger(); AtomicInteger time = new AtomicInteger(); TimelineResults timeline = new TimelineResults(); TagResults tags = new TagResults(); @@ -84,12 +88,17 @@ private Results(Suite suite) { } sp.addAndGet(fr.getPassedCount()); sf.addAndGet(fr.getFailedCount()); + st.addAndGet(fr.getTotalCount()); + ss.addAndGet(fr.getSkippedCount()); + errors.addAll(fr.getErrors()); }); featuresPassed = fp.get(); featuresFailed = ff.get(); scenariosPassed = sp.get(); scenariosFailed = sf.get(); + scenariosSkipped = ss.get(); + scenariosTotal = st.get(); timeTakenMillis = time.get(); saveStatsJson(); printStats(); @@ -100,9 +109,9 @@ private Results(Suite suite) { // last so that path can be printed to the console File file = suite.suiteReports.summaryReport(suite, this).render(); System.out.println("\nHTML report: (paste into browser to view) | Karate version: " - + FileUtils.KARATE_VERSION + displayEnv - + file.toPath().toUri() - + "\n===================================================================\n"); + + FileUtils.KARATE_VERSION + displayEnv + + file.toPath().toUri() + + "\n===================================================================\n"); } } @@ -125,10 +134,10 @@ private void printStats() { System.out.println("Karate version: " + FileUtils.KARATE_VERSION + displayEnv); System.out.println("======================================================"); System.out.println(String.format("elapsed: %6.2f | threads: %4d | thread time: %.2f ", - getElapsedTime() / 1000, suite.threadCount, timeTakenMillis / 1000)); + getElapsedTime() / 1000, suite.threadCount, timeTakenMillis / 1000)); System.out.println(String.format("features: %5d | skipped: %4d | efficiency: %.2f", getFeaturesTotal(), featuresSkipped, getEfficiency())); System.out.println(String.format("scenarios: %4d | passed: %5d | failed: %d", - getScenariosTotal(), scenariosPassed, scenariosFailed)); + getRunScenariosTotal(), scenariosPassed, scenariosFailed)); System.out.println("======================================================"); if (!errors.isEmpty()) { System.out.println(">>> failed features:"); @@ -146,7 +155,9 @@ public Map toKarateJson() { map.put("featuresFailed", featuresFailed); map.put("featuresSkipped", featuresSkipped); map.put("scenariosPassed", scenariosPassed); - map.put("scenariosfailed", errors.size()); + map.put("scenariosFailed", getScenariosFailed()); + map.put("scenariosSkipped", scenariosSkipped); + map.put("scenariosTotal", scenariosTotal); map.put("elapsedTime", getElapsedTime()); map.put("totalTime", getTimeTakenMillis()); map.put("efficiency", getEfficiency()); @@ -179,7 +190,7 @@ public int getScenariosFailed() { return scenariosFailed; } - public int getScenariosTotal() { + public int getRunScenariosTotal() { return scenariosPassed + scenariosFailed; } diff --git a/karate-core/src/main/java/com/intuit/karate/Suite.java b/karate-core/src/main/java/com/intuit/karate/Suite.java index 4a52a7525..79b07dab0 100644 --- a/karate-core/src/main/java/com/intuit/karate/Suite.java +++ b/karate-core/src/main/java/com/intuit/karate/Suite.java @@ -283,7 +283,7 @@ public void saveFeatureResults(FeatureResult fr) { } private void onFeatureDone(FeatureResult fr, int index) { - if (fr.getScenarioCount() > 0) { // possible that zero scenarios matched tags + if (fr.getRunCount() > 0) { // possible that zero scenarios matched tags try { // edge case that reports are not writable saveFeatureResults(fr); String status = fr.isFailed() ? "fail" : "pass"; diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java index d2f5a028c..a18cff08f 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java @@ -37,7 +37,6 @@ import java.util.Map; /** - * * @author pthomas3 */ public class FeatureResult { @@ -53,6 +52,7 @@ public class FeatureResult { private Config config; private int loopIndex = -1; private int callDepth; + private int skippedCount; public FeatureResult(Feature feature) { this.feature = feature; @@ -64,7 +64,7 @@ public void printStats() { StringBuilder sb = new StringBuilder(); sb.append("---------------------------------------------------------\n"); sb.append("feature: ").append(featureName).append('\n'); - sb.append(String.format("scenarios: %2d | passed: %2d | failed: %2d | time: %.4f\n", getScenarioCount(), getPassedCount(), getFailedCount(), getDurationMillis() / 1000)); + sb.append(String.format("scenarios: %2d | passed: %2d | failed: %2d | time: %.4f\n", getRunCount(), getPassedCount(), getFailedCount(), getDurationMillis() / 1000)); sb.append("---------------------------------------------------------\n"); System.out.println(sb); } @@ -92,13 +92,14 @@ public static FeatureResult fromKarateJson(File workingDir, Map fr.loopIndex = (Integer) map.get("loopIndex"); fr.resultDate = (String) map.get("resultDate"); fr.callDepth = (Integer) map.get("callDepth"); + fr.setSkippedCount((Integer) map.get("skippedCount")); List> list = (List) map.get("scenarioResults"); if (list != null) { for (Map srMap : list) { ScenarioResult sr = ScenarioResult.fromKarateJson(workingDir, feature, srMap); if (!sr.getStepResults().isEmpty()) { fr.addResult(sr); - } + } } } return fr; @@ -125,7 +126,10 @@ public Map toSummaryJson() { map.put("durationMillis", getDurationMillis()); map.put("passedCount", getPassedCount()); map.put("failedCount", getFailedCount()); - map.put("scenarioCount", getScenarioCount()); + map.put("skippedCount", getSkippedCount()); + map.put("totalRunCount", getRunCount()); + map.put("totalCount", getTotalCount()); + map.put("scenarioCount", getRunCount()); map.put("packageQualifiedName", feature.getPackageQualifiedName()); map.put("relativePath", feature.getResource().getRelativePath()); return map; @@ -140,6 +144,9 @@ public Map toKarateJson() { map.put("durationMillis", getDurationMillis()); map.put("passedCount", getPassedCount()); map.put("failedCount", getFailedCount()); + map.put("skippedCount", getSkippedCount()); + map.put("totalRunCount", getRunCount()); + map.put("totalCount", getTotalCount()); map.put("packageQualifiedName", feature.getPackageQualifiedName()); map.put("relativePath", feature.getResource().getRelativePath()); //====================================================================== @@ -273,12 +280,12 @@ public boolean isEmpty() { return scenarioResults.isEmpty(); } - public int getScenarioCount() { + public int getRunCount() { return scenarioResults.size(); } public int getPassedCount() { - return getScenarioCount() - getFailedCount(); + return getRunCount() - getFailedCount(); } public boolean isFailed() { @@ -329,4 +336,15 @@ public String toString() { return displayName; } + public int getTotalCount() { + return getRunCount() + getSkippedCount(); + } + + public int getSkippedCount() { + return skippedCount; + } + + public void setSkippedCount(int skippedCount) { + this.skippedCount = skippedCount; + } } diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java index d4a5d7638..1ddb4a066 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java @@ -29,16 +29,17 @@ import com.intuit.karate.http.HttpClientFactory; import com.intuit.karate.resource.MemoryResource; import com.intuit.karate.resource.Resource; + import java.io.File; import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * * @author pthomas3 */ public class FeatureRuntime implements Runnable { @@ -63,6 +64,7 @@ public class FeatureRuntime implements Runnable { public final Map> SETUPONCE_CACHE = new HashMap(); private Runnable next; + private AtomicInteger skippedCount = new AtomicInteger(0); public Resource resolveFromThis(String path) { return featureCall.feature.getResource().resolve(path); @@ -208,6 +210,7 @@ public synchronized void afterFeature() { result.setVariables(lastExecutedScenario.engine.getAllVariablesAsMap()); result.setConfig(lastExecutedScenario.engine.getConfig()); } + result.setSkippedCount(this.skippedCount.get()); if (!result.isEmpty()) { for (RuntimeHook hook : suite.hooks) { hook.afterFeature(this); @@ -217,6 +220,9 @@ public synchronized void afterFeature() { next.run(); } } + public void incrementSkippedCount() { + this.skippedCount.incrementAndGet(); + } @Override public String toString() { diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java index 1dc10e223..d8ac31f93 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java @@ -30,7 +30,6 @@ import com.intuit.karate.ScenarioActions; import com.intuit.karate.StringUtils; import com.intuit.karate.graal.JsEngine; -import com.intuit.karate.graal.JsValue; import com.intuit.karate.http.ResourceType; import com.intuit.karate.shell.StringLogAppender; @@ -282,8 +281,8 @@ public Map getScenarioInfo() { protected void logError(String message) { if (currentStep != null) { message = currentStep.getDebugInfo() - + "\n" + currentStep.toString() - + "\n" + message; + + "\n" + currentStep.toString() + + "\n" + message; } logger.error("{}", message); } @@ -357,6 +356,7 @@ private static boolean isSelectedForExecution(FeatureRuntime fr, Scenario scenar public void beforeRun() { if (featureRuntime.caller.isNone() && featureRuntime.suite.isAborted()) { skipped = true; + featureRuntime.incrementSkippedCount(); return; } steps = skipBackground ? scenario.getSteps() : scenario.getStepsIncludingBackground(); @@ -372,9 +372,10 @@ public void beforeRun() { evalConfigJs(featureRuntime.suite.karateConfigEnv, "karate-config-" + featureRuntime.suite.env + ".js"); } skipped = !featureRuntime.suite.hooks.stream() - .map(h -> h.beforeScenario(this)) - .reduce(Boolean.TRUE, Boolean::logicalAnd); + .map(h -> h.beforeScenario(this)) + .reduce(Boolean.TRUE, Boolean::logicalAnd); if (skipped) { + featureRuntime.incrementSkippedCount(); logger.debug("beforeScenario hook returned false, will skip scenario: {}", scenario); } else { evaluateScenarioName(); @@ -544,9 +545,9 @@ public String toString() { public void evaluateScenarioName() { String scenarioName = scenario.getName(); boolean wrappedByBackTick = scenarioName != null - && scenarioName.length() > 1 - && '`' == scenarioName.charAt(0) - && '`' == scenarioName.charAt((scenarioName.length() - 1)); + && scenarioName.length() > 1 + && '`' == scenarioName.charAt(0) + && '`' == scenarioName.charAt((scenarioName.length() - 1)); boolean hasJavascriptPlaceholder = ScenarioEngine.hasJavaScriptPlacehoder(scenarioName); if (wrappedByBackTick || hasJavascriptPlaceholder) { String eval = scenarioName; diff --git a/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java b/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java index d41c620b9..5a29042f7 100644 --- a/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java +++ b/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java @@ -194,7 +194,7 @@ public static File saveJunitXml(String targetDir, FeatureResult result, String f Document doc = XmlUtils.newDocument(); Element root = doc.createElement("testsuite"); doc.appendChild(root); - root.setAttribute("tests", result.getScenarioCount() + ""); + root.setAttribute("tests", result.getRunCount() + ""); root.setAttribute("failures", result.getFailedCount() + ""); root.setAttribute("time", formatter.format(result.getDurationMillis() / 1000)); root.setAttribute("name", result.getDisplayName()); // will be uri diff --git a/karate-core/src/main/java/com/intuit/karate/report/karate-summary.html b/karate-core/src/main/java/com/intuit/karate/report/karate-summary.html index 0f24a2b5e..e8ef720ff 100644 --- a/karate-core/src/main/java/com/intuit/karate/report/karate-summary.html +++ b/karate-core/src/main/java/com/intuit/karate/report/karate-summary.html @@ -1,6 +1,6 @@ - + @@ -12,10 +12,10 @@ Karate Summary Report - - -
-
+ + +
+
-
-
+
+
- Tags| - Timeline + Tags| + Timeline
- + - - - - - - + + + + + + + + - - + + - - - - - - + + + + + + + + + + + + + + + + + + - +
FeatureTitlePassedFailedScenariosTime (ms)FeatureTitlePassedFailedSkippedScenariosPass (%)Time (ms)
- - + +
+ Totals:
-
- +
+ diff --git a/karate-core/src/test/java/com/intuit/karate/core/PerfHookTest.java b/karate-core/src/test/java/com/intuit/karate/core/PerfHookTest.java index 574ccf066..61d008509 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/PerfHookTest.java +++ b/karate-core/src/test/java/com/intuit/karate/core/PerfHookTest.java @@ -2,13 +2,19 @@ import com.intuit.karate.PerfHook; import com.intuit.karate.Runner; + import static com.intuit.karate.TestUtils.*; + import com.intuit.karate.http.HttpRequest; + import java.util.Collections; import java.util.Map; import java.util.UUID; + import org.junit.jupiter.api.AfterAll; + import static org.junit.jupiter.api.Assertions.*; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -16,7 +22,6 @@ import org.slf4j.LoggerFactory; /** - * * @author pthomas3 */ class PerfHookTest { @@ -28,8 +33,8 @@ class PerfHookTest { @BeforeAll static void beforeAll() { server = MockServer - .feature("classpath:com/intuit/karate/core/perf-mock.feature") - .http(0).build(); + .feature("classpath:com/intuit/karate/core/perf-mock.feature") + .http(0).build(); System.setProperty("karate.server.port", server.getPort() + ""); } @@ -54,9 +59,9 @@ void testPerfHook1() { assertNotNull(featureResult); assertFalse(featureResult.isEmpty()); assertFalse(featureResult.isFailed()); - assertEquals(featureResult.getScenarioCount(), 1); - assertEquals(featureResult.getPassedCount(), 1); - assertEquals(featureResult.getFailedCount(), 0); + assertEquals(1, featureResult.getRunCount()); + assertEquals(1, featureResult.getPassedCount()); + assertEquals(0, featureResult.getFailedCount()); matchContains(featureResult.getVariables(), "{ configSource: 'normal', responseStatus: 200, response: { foo: ['" + bar + "'] } }"); } @@ -70,9 +75,9 @@ void testPerfHook2() { assertNotNull(featureResult); assertFalse(featureResult.isEmpty()); assertTrue(featureResult.isFailed()); - assertEquals(featureResult.getScenarioCount(), 1); - assertEquals(featureResult.getPassedCount(), 0); - assertEquals(featureResult.getFailedCount(), 1); + assertEquals(1, featureResult.getRunCount()); + assertEquals(0, featureResult.getPassedCount()); + assertEquals(1, featureResult.getFailedCount()); matchContains(featureResult.getVariables(), "{ configSource: 'normal', responseStatus: 200, response: { foo: ['" + bar + "'] } }"); } @@ -86,9 +91,9 @@ void testPerfHook3() { assertNotNull(featureResult); assertFalse(featureResult.isEmpty()); assertTrue(featureResult.isFailed()); - assertEquals(featureResult.getScenarioCount(), 1); - assertEquals(featureResult.getPassedCount(), 0); - assertEquals(featureResult.getFailedCount(), 1); + assertEquals(1, featureResult.getRunCount()); + assertEquals(0, featureResult.getPassedCount()); + assertEquals(1, featureResult.getFailedCount()); matchContains(featureResult.getVariables(), "{ configSource: 'normal', responseStatus: 200, response: { foo: ['" + bar + "'] } }"); } @@ -100,9 +105,9 @@ void testPerfHook4() { assertNotNull(featureResult); assertFalse(featureResult.isEmpty()); assertTrue(featureResult.isFailed()); - assertEquals(featureResult.getScenarioCount(), 1); - assertEquals(featureResult.getPassedCount(), 0); - assertEquals(featureResult.getFailedCount(), 1); + assertEquals(1, featureResult.getRunCount()); + assertEquals(0, featureResult.getPassedCount()); + assertEquals(1, featureResult.getFailedCount()); match(featureResult.getVariables(), "{ configSource: 'normal', functionFromKarateBase: '#notnull' }"); } @@ -114,9 +119,9 @@ void testPerfHook5() { assertNotNull(featureResult); assertTrue(featureResult.isEmpty()); assertFalse(featureResult.isFailed()); - assertEquals(featureResult.getScenarioCount(), 0); - assertEquals(featureResult.getPassedCount(), 0); - assertEquals(featureResult.getFailedCount(), 0); + assertEquals(0, featureResult.getRunCount()); + assertEquals(0, featureResult.getPassedCount()); + assertEquals(0, featureResult.getFailedCount()); assertEquals(featureResult.getVariables(), Collections.emptyMap()); } @@ -163,9 +168,7 @@ public void afterFeature(FeatureResult fr) { @Override public void pause(Number millis) { - - } + } }; - } diff --git a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json index 863ff5cad..1aa7022a5 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/feature-result.json +++ b/karate-core/src/test/java/com/intuit/karate/core/feature-result.json @@ -135,6 +135,9 @@ "resultDate": "#string", "passedCount": 1, "failedCount": 0, + "skippedCount": 0, + "totalRunCount":1, + "totalCount": 1, "callDepth": 1, "loopIndex": -1 } @@ -291,6 +294,9 @@ "resultDate": "#string", "passedCount": 3, "failedCount": 0, + "skippedCount": 0, + "totalRunCount":3, + "totalCount":3, "callDepth": 0, "loopIndex": -1 } diff --git a/karate-core/src/test/java/com/intuit/karate/core/runner/FeatureResultTest.java b/karate-core/src/test/java/com/intuit/karate/core/runner/FeatureResultTest.java index 9951d1b36..a4d4f256f 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/runner/FeatureResultTest.java +++ b/karate-core/src/test/java/com/intuit/karate/core/runner/FeatureResultTest.java @@ -7,11 +7,14 @@ import com.intuit.karate.core.FeatureCall; import com.intuit.karate.core.FeatureResult; import com.intuit.karate.core.FeatureRuntime; + import java.io.File; import java.util.List; import java.util.Map; import java.util.function.IntBinaryOperator; + import static org.junit.jupiter.api.Assertions.*; + import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +42,9 @@ static String xml(FeatureResult result) { void testFailureMultiScenarioFeature() throws Exception { FeatureResult result = result("failed.feature"); assertEquals(2, result.getFailedCount()); - assertEquals(3, result.getScenarioCount()); + assertEquals(0, result.getSkippedCount()); + assertEquals(3, result.getRunCount()); + assertEquals(3, result.getTotalCount()); String contents = xml(result); assertTrue(contents.contains("did not evaluate to 'true': a != 1")); assertTrue(contents.contains("did not evaluate to 'true': a == 3")); @@ -63,7 +68,9 @@ void testFailureMultiScenarioFeature() throws Exception { void testAbortMultiScenarioFeature() throws Exception { FeatureResult result = result("aborted.feature"); assertEquals(0, result.getFailedCount()); - assertEquals(4, result.getScenarioCount()); + assertEquals(0, result.getSkippedCount()); + assertEquals(4, result.getRunCount()); + assertEquals(4, result.getTotalCount()); String contents = xml(result); // skip-pass and skip-fail both should have all steps as skipped @@ -96,7 +103,7 @@ void testLambdaFunctionsInScenarioFeature() throws Exception { // @Test // TODO fails in jdk 17 void testStackOverFlowError() { FeatureResult result = result("stackoverflow-error.feature"); - assertTrue(result.isFailed()); + assertTrue(result.isFailed()); assertTrue(result.getScenarioResults().get(0).getErrorMessage().contains("StackOverflowError")); } diff --git a/karate-core/src/test/java/com/intuit/karate/core/runner/hooks/ScenarioHookTest.java b/karate-core/src/test/java/com/intuit/karate/core/runner/hooks/ScenarioHookTest.java index 3313f3737..dcfd491dd 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/runner/hooks/ScenarioHookTest.java +++ b/karate-core/src/test/java/com/intuit/karate/core/runner/hooks/ScenarioHookTest.java @@ -25,7 +25,7 @@ void testHookForExamplesWithTags() { String path = "classpath:com/intuit/karate/core/runner/hooks/test-hook-multiexample.feature"; Results results = Runner.path(path).hook(new MandatoryTagHook()).parallel(1); assertEquals(1, results.getFeaturesTotal()); - assertEquals(7, results.getScenariosTotal()); + assertEquals(7, results.getRunScenariosTotal()); assertEquals(0, results.getFailCount()); }