2340: Enhance karate summary report

- Add skipped scenario count - Add pass percentage - Add totals row
karatelabs · Jan 13, 2024 · c7cc672 · c7cc672
1 parent 3affad9
commit c7cc672
Show file tree

Hide file tree

Showing 11 changed files with 150 additions and 80 deletions.
diff --git a/karate-core/src/main/java/com/intuit/karate/Results.java b/karate-core/src/main/java/com/intuit/karate/Results.java
@@ -28,6 +28,7 @@
 import com.intuit.karate.core.TagResults;
 import com.intuit.karate.core.TimelineResults;
 import com.intuit.karate.report.ReportUtils;
+
 import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -37,7 +38,6 @@
 import java.util.stream.Stream;
 
 /**
- *
  * @author pthomas3
  */
 public class Results {
@@ -48,6 +48,8 @@ public class Results {
     private final int featuresSkipped;
     private final int scenariosPassed;
     private final int scenariosFailed;
+    private final int scenariosSkipped;
+    private final int scenariosTotal;
     private final double timeTakenMillis;
     private final long endTime;
     private final List<String> errors = new ArrayList();
@@ -66,6 +68,8 @@ private Results(Suite suite) {
         AtomicInteger ff = new AtomicInteger();
         AtomicInteger sp = new AtomicInteger();
         AtomicInteger sf = new AtomicInteger();
+        AtomicInteger ss = new AtomicInteger();
+        AtomicInteger st = new AtomicInteger();
         AtomicInteger time = new AtomicInteger();
         TimelineResults timeline = new TimelineResults();
         TagResults tags = new TagResults();
@@ -84,12 +88,17 @@ private Results(Suite suite) {
             }
             sp.addAndGet(fr.getPassedCount());
             sf.addAndGet(fr.getFailedCount());
+            st.addAndGet(fr.getTotalCount());
+            ss.addAndGet(fr.getSkippedCount());
+
             errors.addAll(fr.getErrors());
         });
         featuresPassed = fp.get();
         featuresFailed = ff.get();
         scenariosPassed = sp.get();
         scenariosFailed = sf.get();
+        scenariosSkipped = ss.get();
+        scenariosTotal = st.get();
         timeTakenMillis = time.get();
         saveStatsJson();
         printStats();
@@ -100,9 +109,9 @@ private Results(Suite suite) {
             // last so that path can be printed to the console
             File file = suite.suiteReports.summaryReport(suite, this).render();
             System.out.println("\nHTML report: (paste into browser to view) | Karate version: "
-                    + FileUtils.KARATE_VERSION + displayEnv
-                    + file.toPath().toUri()
-                    + "\n===================================================================\n");
+                + FileUtils.KARATE_VERSION + displayEnv
+                + file.toPath().toUri()
+                + "\n===================================================================\n");
         }
     }
 
@@ -125,10 +134,10 @@ private void printStats() {
         System.out.println("Karate version: " + FileUtils.KARATE_VERSION + displayEnv);
         System.out.println("======================================================");
         System.out.println(String.format("elapsed: %6.2f | threads: %4d | thread time: %.2f ",
-                getElapsedTime() / 1000, suite.threadCount, timeTakenMillis / 1000));
+            getElapsedTime() / 1000, suite.threadCount, timeTakenMillis / 1000));
         System.out.println(String.format("features: %5d | skipped: %4d | efficiency: %.2f", getFeaturesTotal(), featuresSkipped, getEfficiency()));
         System.out.println(String.format("scenarios: %4d | passed: %5d | failed: %d",
-                getScenariosTotal(), scenariosPassed, scenariosFailed));
+            getRunScenariosTotal(), scenariosPassed, scenariosFailed));
         System.out.println("======================================================");
         if (!errors.isEmpty()) {
             System.out.println(">>> failed features:");
@@ -146,7 +155,9 @@ public Map<String, Object> toKarateJson() {
         map.put("featuresFailed", featuresFailed);
         map.put("featuresSkipped", featuresSkipped);
         map.put("scenariosPassed", scenariosPassed);
-        map.put("scenariosfailed", errors.size());
+        map.put("scenariosFailed", getScenariosFailed());
+        map.put("scenariosSkipped", scenariosSkipped);
+        map.put("scenariosTotal", scenariosTotal);
         map.put("elapsedTime", getElapsedTime());
         map.put("totalTime", getTimeTakenMillis());
         map.put("efficiency", getEfficiency());
@@ -179,7 +190,7 @@ public int getScenariosFailed() {
         return scenariosFailed;
     }
 
-    public int getScenariosTotal() {
+    public int getRunScenariosTotal() {
         return scenariosPassed + scenariosFailed;
     }
 

diff --git a/karate-core/src/main/java/com/intuit/karate/Suite.java b/karate-core/src/main/java/com/intuit/karate/Suite.java
@@ -283,7 +283,7 @@ public void saveFeatureResults(FeatureResult fr) {
     }
 
     private void onFeatureDone(FeatureResult fr, int index) {
-        if (fr.getScenarioCount() > 0) { // possible that zero scenarios matched tags
+        if (fr.getRunCount() > 0) { // possible that zero scenarios matched tags
             try { // edge case that reports are not writable     
                 saveFeatureResults(fr);
                 String status = fr.isFailed() ? "fail" : "pass";

diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java
@@ -37,7 +37,6 @@
 import java.util.Map;
 
 /**
- *
  * @author pthomas3
  */
 public class FeatureResult {
@@ -53,6 +52,7 @@ public class FeatureResult {
     private Config config;
     private int loopIndex = -1;
     private int callDepth;
+    private int skippedCount;
 
     public FeatureResult(Feature feature) {
         this.feature = feature;
@@ -64,7 +64,7 @@ public void printStats() {
         StringBuilder sb = new StringBuilder();
         sb.append("---------------------------------------------------------\n");
         sb.append("feature: ").append(featureName).append('\n');
-        sb.append(String.format("scenarios: %2d | passed: %2d | failed: %2d | time: %.4f\n", getScenarioCount(), getPassedCount(), getFailedCount(), getDurationMillis() / 1000));
+        sb.append(String.format("scenarios: %2d | passed: %2d | failed: %2d | time: %.4f\n", getRunCount(), getPassedCount(), getFailedCount(), getDurationMillis() / 1000));
         sb.append("---------------------------------------------------------\n");
         System.out.println(sb);
     }
@@ -92,13 +92,14 @@ public static FeatureResult fromKarateJson(File workingDir, Map<String, Object>
         fr.loopIndex = (Integer) map.get("loopIndex");
         fr.resultDate = (String) map.get("resultDate");
         fr.callDepth = (Integer) map.get("callDepth");
+        fr.setSkippedCount((Integer) map.get("skippedCount"));
         List<Map<String, Object>> list = (List) map.get("scenarioResults");
         if (list != null) {
             for (Map<String, Object> srMap : list) {
                 ScenarioResult sr = ScenarioResult.fromKarateJson(workingDir, feature, srMap);
                 if (!sr.getStepResults().isEmpty()) {
                     fr.addResult(sr);
-                }                
+                }
             }
         }
         return fr;
@@ -125,7 +126,10 @@ public Map<String, Object> toSummaryJson() {
         map.put("durationMillis", getDurationMillis());
         map.put("passedCount", getPassedCount());
         map.put("failedCount", getFailedCount());
-        map.put("scenarioCount", getScenarioCount());
+        map.put("skippedCount", getSkippedCount());
+        map.put("totalRunCount", getRunCount());
+        map.put("totalCount", getTotalCount());
+        map.put("scenarioCount", getRunCount());
         map.put("packageQualifiedName", feature.getPackageQualifiedName());
         map.put("relativePath", feature.getResource().getRelativePath());
         return map;
@@ -140,6 +144,9 @@ public Map<String, Object> toKarateJson() {
         map.put("durationMillis", getDurationMillis());
         map.put("passedCount", getPassedCount());
         map.put("failedCount", getFailedCount());
+        map.put("skippedCount", getSkippedCount());
+        map.put("totalRunCount", getRunCount());
+        map.put("totalCount", getTotalCount());
         map.put("packageQualifiedName", feature.getPackageQualifiedName());
         map.put("relativePath", feature.getResource().getRelativePath());
         //======================================================================
@@ -273,12 +280,12 @@ public boolean isEmpty() {
         return scenarioResults.isEmpty();
     }
 
-    public int getScenarioCount() {
+    public int getRunCount() {
         return scenarioResults.size();
     }
 
     public int getPassedCount() {
-        return getScenarioCount() - getFailedCount();
+        return getRunCount() - getFailedCount();
     }
 
     public boolean isFailed() {
@@ -329,4 +336,15 @@ public String toString() {
         return displayName;
     }
 
+    public int getTotalCount() {
+        return getRunCount() + getSkippedCount();
+    }
+
+    public int getSkippedCount() {
+        return skippedCount;
+    }
+
+    public void setSkippedCount(int skippedCount) {
+        this.skippedCount = skippedCount;
+    }
 }
diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureRuntime.java
@@ -29,16 +29,17 @@
 import com.intuit.karate.http.HttpClientFactory;
 import com.intuit.karate.resource.MemoryResource;
 import com.intuit.karate.resource.Resource;
+
 import java.io.File;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- *
  * @author pthomas3
  */
 public class FeatureRuntime implements Runnable {
@@ -63,6 +64,7 @@ public class FeatureRuntime implements Runnable {
     public final Map<String, Map<String, Object>> SETUPONCE_CACHE = new HashMap();
 
     private Runnable next;
+    private AtomicInteger skippedCount = new AtomicInteger(0);
 
     public Resource resolveFromThis(String path) {
         return featureCall.feature.getResource().resolve(path);
@@ -208,6 +210,7 @@ public synchronized void afterFeature() {
             result.setVariables(lastExecutedScenario.engine.getAllVariablesAsMap());
             result.setConfig(lastExecutedScenario.engine.getConfig());
         }
+        result.setSkippedCount(this.skippedCount.get());
         if (!result.isEmpty()) {
             for (RuntimeHook hook : suite.hooks) {
                 hook.afterFeature(this);
@@ -217,6 +220,9 @@ public synchronized void afterFeature() {
             next.run();
         }
     }
+    public void incrementSkippedCount() {
+        this.skippedCount.incrementAndGet();
+    }
 
     @Override
     public String toString() {

diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java
@@ -30,7 +30,6 @@
 import com.intuit.karate.ScenarioActions;
 import com.intuit.karate.StringUtils;
 import com.intuit.karate.graal.JsEngine;
-import com.intuit.karate.graal.JsValue;
 import com.intuit.karate.http.ResourceType;
 import com.intuit.karate.shell.StringLogAppender;
 
@@ -282,8 +281,8 @@ public Map<String, Object> getScenarioInfo() {
     protected void logError(String message) {
         if (currentStep != null) {
             message = currentStep.getDebugInfo()
-                    + "\n" + currentStep.toString()
-                    + "\n" + message;
+                + "\n" + currentStep.toString()
+                + "\n" + message;
         }
         logger.error("{}", message);
     }
@@ -357,6 +356,7 @@ private static boolean isSelectedForExecution(FeatureRuntime fr, Scenario scenar
     public void beforeRun() {
         if (featureRuntime.caller.isNone() && featureRuntime.suite.isAborted()) {
             skipped = true;
+            featureRuntime.incrementSkippedCount();
             return;
         }
         steps = skipBackground ? scenario.getSteps() : scenario.getStepsIncludingBackground();
@@ -372,9 +372,10 @@ public void beforeRun() {
                 evalConfigJs(featureRuntime.suite.karateConfigEnv, "karate-config-" + featureRuntime.suite.env + ".js");
             }
             skipped = !featureRuntime.suite.hooks.stream()
-                    .map(h -> h.beforeScenario(this))
-                    .reduce(Boolean.TRUE, Boolean::logicalAnd);
+                .map(h -> h.beforeScenario(this))
+                .reduce(Boolean.TRUE, Boolean::logicalAnd);
             if (skipped) {
+                featureRuntime.incrementSkippedCount();
                 logger.debug("beforeScenario hook returned false, will skip scenario: {}", scenario);
             } else {
                 evaluateScenarioName();
@@ -544,9 +545,9 @@ public String toString() {
     public void evaluateScenarioName() {
         String scenarioName = scenario.getName();
         boolean wrappedByBackTick = scenarioName != null
-                && scenarioName.length() > 1
-                && '`' == scenarioName.charAt(0)
-                && '`' == scenarioName.charAt((scenarioName.length() - 1));
+            && scenarioName.length() > 1
+            && '`' == scenarioName.charAt(0)
+            && '`' == scenarioName.charAt((scenarioName.length() - 1));
         boolean hasJavascriptPlaceholder = ScenarioEngine.hasJavaScriptPlacehoder(scenarioName);
         if (wrappedByBackTick || hasJavascriptPlaceholder) {
             String eval = scenarioName;

diff --git a/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java b/karate-core/src/main/java/com/intuit/karate/report/ReportUtils.java
@@ -194,7 +194,7 @@ public static File saveJunitXml(String targetDir, FeatureResult result, String f
         Document doc = XmlUtils.newDocument();
         Element root = doc.createElement("testsuite");
         doc.appendChild(root);
-        root.setAttribute("tests", result.getScenarioCount() + "");
+        root.setAttribute("tests", result.getRunCount() + "");
         root.setAttribute("failures", result.getFailedCount() + "");
         root.setAttribute("time", formatter.format(result.getDurationMillis() / 1000));
         root.setAttribute("name", result.getDisplayName()); // will be uri