From 05e9c111ab63ec307510d82f6bd8147ae338b395 Mon Sep 17 00:00:00 2001 From: Peter Thomas Date: Sun, 19 Aug 2018 16:38:54 -0500 Subject: [PATCH] cukexit wip the parallel runner works ! ref #444 the demos are working including reports TODO called steps support for the legacy cucumber-options, only tags and features new scenario-hooks strategy todo time reporting and hiding skipped features but it does look like the bulk of the work is over --- .../java/com/intuit/karate/CallContext.java | 11 +- .../java/com/intuit/karate/FileUtils.java | 46 +++-- .../main/java/com/intuit/karate/Match.java | 2 +- .../main/java/com/intuit/karate/Script.java | 4 +- .../main/java/com/intuit/karate/StepDefs.java | 7 +- .../java/com/intuit/karate/core/Engine.java | 37 ++-- .../intuit/karate/core/ExecutionContext.java | 53 +++++ .../karate/core/FeatureExecutionUnit.java | 29 +-- .../com/intuit/karate/core/FeatureParser.java | 15 +- .../com/intuit/karate/core/FeatureResult.java | 78 ++++++-- .../java/com/intuit/karate/core/Result.java | 2 +- .../karate/core/ScenarioExecutionUnit.java | 29 ++- .../com/intuit/karate/core/ScenarioHook.java | 41 ++++ .../karate/core/SectionExecutionUnit.java | 39 ++-- .../intuit/karate/core/StepExecutionUnit.java | 10 +- .../karate/core/StepListExecutionUnit.java | 2 +- .../karate/cucumber/CucumberRunner.java | 186 ++++++++---------- .../com/intuit/karate/filter/TagFilter.java | 32 --- .../karate/filter/TagFilterException.java | 12 -- .../java/com/intuit/karate/FileUtilsTest.java | 7 +- .../intuit/karate/core/MandatoryTagHook.java | 56 ++++++ .../intuit/karate/core/ScenarioHookTest.java | 55 ++++++ .../test-hook-multiexample.feature} | 0 .../karate/core/test-hook-notags.feature | 5 + .../karate/cucumber/CucumberRunnerTest.java | 44 ++--- .../karate/cucumber/FeatureResultTest.java | 60 +++--- .../karate/cucumber/FeatureReuseTest.java | 24 +-- .../filter/TagFilterMultiScenarioTest.java | 24 --- .../intuit/karate/filter/TagFilterTest.java | 30 --- .../karate/filter/TagFilterTestImpl.java | 65 ------ .../intuit/karate/filter/tag-filter.feature | 8 - .../src/test/java/demo/java/JavaApiTest.java | 9 +- .../intuit/karate/gatling/KarateAction.scala | 2 +- 33 files changed, 568 insertions(+), 456 deletions(-) create mode 100644 karate-core/src/main/java/com/intuit/karate/core/ExecutionContext.java create mode 100644 karate-core/src/main/java/com/intuit/karate/core/ScenarioHook.java delete mode 100644 karate-core/src/main/java/com/intuit/karate/filter/TagFilter.java delete mode 100644 karate-core/src/main/java/com/intuit/karate/filter/TagFilterException.java create mode 100644 karate-core/src/test/java/com/intuit/karate/core/MandatoryTagHook.java create mode 100644 karate-core/src/test/java/com/intuit/karate/core/ScenarioHookTest.java rename karate-core/src/test/java/com/intuit/karate/{filter/tag-filter-multiscenario.feature => core/test-hook-multiexample.feature} (100%) create mode 100644 karate-core/src/test/java/com/intuit/karate/core/test-hook-notags.feature delete mode 100644 karate-core/src/test/java/com/intuit/karate/filter/TagFilterMultiScenarioTest.java delete mode 100644 karate-core/src/test/java/com/intuit/karate/filter/TagFilterTest.java delete mode 100644 karate-core/src/test/java/com/intuit/karate/filter/TagFilterTestImpl.java delete mode 100644 karate-core/src/test/java/com/intuit/karate/filter/tag-filter.feature diff --git a/karate-core/src/main/java/com/intuit/karate/CallContext.java b/karate-core/src/main/java/com/intuit/karate/CallContext.java index 1e641e423..3c33aa0bf 100644 --- a/karate-core/src/main/java/com/intuit/karate/CallContext.java +++ b/karate-core/src/main/java/com/intuit/karate/CallContext.java @@ -23,6 +23,7 @@ */ package com.intuit.karate; +import com.intuit.karate.core.ScenarioHook; import com.intuit.karate.cucumber.ScenarioInfo; import com.intuit.karate.cucumber.StepInterceptor; import java.util.List; @@ -45,6 +46,7 @@ public class CallContext { public final Consumer asyncSystem; public final Runnable asyncNext; public final StepInterceptor stepInterceptor; + public final ScenarioHook scenarioHook; private List tags; private Map> tagValues; @@ -79,12 +81,16 @@ public boolean isCalled() { } public CallContext(Map callArg, boolean evalKarateConfig) { - this(null, 0, callArg, -1, false, evalKarateConfig, null, null, null, null); + this(null, 0, callArg, -1, false, evalKarateConfig, null, null, null, null, null); } + public CallContext(ScenarioHook scenarioHook) { + this(null, 0, null, -1, false, true, null, null, null, null, scenarioHook); + } + public CallContext(ScriptContext parentContext, int callDepth, Map callArg, int loopIndex, boolean reuseParentContext, boolean evalKarateConfig, String httpClientClass, - Consumer asyncSystem, Runnable asyncNext, StepInterceptor stepInterceptor) { + Consumer asyncSystem, Runnable asyncNext, StepInterceptor stepInterceptor, ScenarioHook scenarioHook) { this.parentContext = parentContext; this.callDepth = callDepth; this.callArg = callArg; @@ -95,6 +101,7 @@ public CallContext(ScriptContext parentContext, int callDepth, Map scanForFeatureFilesOnClassPath() { return scanForFeatureFiles(CLASSPATH_COLON_SLASH); } - public static List scanForFeatureFiles(String pathString) { - Path classPathRoot = getClassPathRoot(); - Path rootPath = getRootPathFor(pathString); + public static List scanForFeatureFiles(List paths) { + List list = new ArrayList(); + for (String path : paths) { + list.addAll(scanForFeatureFiles(path)); + } + return list; + } + + public static List scanForFeatureFiles(String pathString) { + boolean classpath = isClassPath(pathString); + Path rootPath = classpath ? getClassPathRoot() : getPathFor(null); + Path thisPath = getPathFor(pathString); List files = new ArrayList(); Stream stream; try { - stream = Files.walk(rootPath); + stream = Files.walk(thisPath); } catch (IOException e) { throw new RuntimeException(e); } @@ -484,8 +505,9 @@ public static List scanForFeatureFiles(String pathString) { Path path = paths.next(); if (path.getFileName().toString().endsWith(".feature")) { File file = path.toFile(); - Path relativePath = classPathRoot.relativize(path); - files.add(new FileResource(file, relativePath.toString())); + Path relativePath = rootPath.relativize(path); + String prefix = classpath ? CLASSPATH_COLON : ""; + files.add(new FileResource(file, prefix + relativePath.toString())); } } return files; diff --git a/karate-core/src/main/java/com/intuit/karate/Match.java b/karate-core/src/main/java/com/intuit/karate/Match.java index 26806f5ff..0861b596b 100644 --- a/karate-core/src/main/java/com/intuit/karate/Match.java +++ b/karate-core/src/main/java/com/intuit/karate/Match.java @@ -65,7 +65,7 @@ private static Match parse(String exp) { private Match() { ScriptEnv env = ScriptEnv.forEnvAndCurrentWorkingDir(null); CallContext callContext = new CallContext(null, 0, null, -1, false, false, - DummyHttpClient.class.getName(), null, null, null); + DummyHttpClient.class.getName(), null, null, null, null); context = new ScriptContext(env, callContext); } diff --git a/karate-core/src/main/java/com/intuit/karate/Script.java b/karate-core/src/main/java/com/intuit/karate/Script.java index 94726ac3a..7ab3c42c6 100755 --- a/karate-core/src/main/java/com/intuit/karate/Script.java +++ b/karate-core/src/main/java/com/intuit/karate/Script.java @@ -1688,8 +1688,8 @@ public static ScriptValue evalFeatureCall(Feature feature, Object callArg, Scrip private static ScriptValue evalFeatureCall(Feature feature, ScriptContext context, Map callArg, int loopIndex, boolean reuseParentConfig) { CallContext callContext = new CallContext(context, context.callDepth + 1, callArg, loopIndex, - reuseParentConfig, false, null, context.asyncSystem, null, context.stepInterceptor); -// if (context.env.reporter != null) { + reuseParentConfig, false, null, context.asyncSystem, null, context.stepInterceptor, null); +// if (context.env.reporter != null) { TODO call reporting // context.env.reporter.callBegin(feature, callContext); // } FeatureResult result = Engine.executeSync(null, feature, null, callContext); diff --git a/karate-core/src/main/java/com/intuit/karate/StepDefs.java b/karate-core/src/main/java/com/intuit/karate/StepDefs.java index 07a4167b8..06b81a84b 100755 --- a/karate-core/src/main/java/com/intuit/karate/StepDefs.java +++ b/karate-core/src/main/java/com/intuit/karate/StepDefs.java @@ -72,12 +72,15 @@ private static ScriptEnv getFeatureEnv() { return ideScriptEnv; } - public StepDefs(ScriptEnv scriptEnv, CallContext call) { - context = new ScriptContext(scriptEnv, call); + public StepDefs(ScriptEnv scriptEnv, CallContext callContext) { + this.callContext = callContext; + context = new ScriptContext(scriptEnv, callContext); request = new HttpRequestBuilder(); } public final ScriptContext context; + public final CallContext callContext; + private HttpRequestBuilder request; private HttpResponse response; diff --git a/karate-core/src/main/java/com/intuit/karate/core/Engine.java b/karate-core/src/main/java/com/intuit/karate/core/Engine.java index 458bcc06e..5897bd97d 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Engine.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Engine.java @@ -89,15 +89,11 @@ public static FeatureResult executeSync(String envString, Feature feature, Strin ScriptEnv env = ScriptEnv.forEnvTagsAndFeatureFile(envString, tagSelector, file); if (callContext == null) { callContext = new CallContext(null, true); - } - StepDefs stepDefs = new StepDefs(env, callContext); - String basePath = feature.getPackageQualifiedName(); - LogAppender appender = new FileLogAppender(getBuildDir() + "/surefire-reports/" + basePath + ".log", stepDefs.context.logger); - FeatureExecutionUnit unit = new FeatureExecutionUnit(feature, stepDefs, appender); - unit.submit(SYNC_EXECUTOR, NO_OP); - FeatureResult result = unit.getFeatureResult(); - result.setResultVars(stepDefs.context.getVars()); - return result; + } + ExecutionContext exec = new ExecutionContext(feature, env, callContext); + FeatureExecutionUnit unit = new FeatureExecutionUnit(exec); + unit.submit(SYNC_EXECUTOR, NO_OP); + return exec.result; } public static Result execute(Scenario scenario, Step step, StepDefs stepDefs) { @@ -126,19 +122,23 @@ public static Result execute(Scenario scenario, Step step, StepDefs stepDefs) { try { match.method.invoke(stepDefs, args); return Result.passed(getElapsedTime(startTime)); - } catch (KarateAbortException ke) { - return Result.aborted(getElapsedTime(startTime)); } catch (InvocationTargetException e) { // target will be KarateException - return Result.failed(getElapsedTime(startTime), e.getTargetException(), scenario, step); + if (e.getTargetException() instanceof KarateAbortException) { + return Result.aborted(getElapsedTime(startTime)); + } else { + return Result.failed(getElapsedTime(startTime), e.getTargetException(), scenario, step); + } } catch (Exception e) { return Result.failed(getElapsedTime(startTime), e, scenario, step); } } - public static void saveResultJson(String targetDir, FeatureResult result) { + public static File saveResultJson(String targetDir, FeatureResult result) { List single = Collections.singletonList(result); String json = JsonUtils.toPrettyJsonString(JsonUtils.toJsonDoc(single)); - FileUtils.writeToFile(new File(targetDir + "/" + result.getPackageQualifiedName() + ".json"), json); + File file = new File(targetDir + "/" + result.getPackageQualifiedName() + ".json"); + FileUtils.writeToFile(file, json); + return file; } private static String formatNanos(long nanos, DecimalFormat formatter) { @@ -172,7 +172,7 @@ private static Throwable appendSteps(List steps, StringBuilder sb) { return error; } - public static void saveResultXml(String targetDir, FeatureResult result) { + public static File saveResultXml(String targetDir, FeatureResult result) { DecimalFormat formatter = (DecimalFormat) NumberFormat.getNumberInstance(Locale.US); formatter.applyPattern("0.######"); Document doc = XmlUtils.newDocument(); @@ -232,7 +232,9 @@ public static void saveResultXml(String targetDir, FeatureResult result) { root.setAttribute("failures", failureCount + ""); root.setAttribute("time", formatNanos(totalDuration, formatter)); String xml = XmlUtils.toString(doc, true); - FileUtils.writeToFile(new File(targetDir + "/" + baseName + ".xml"), xml); + File file = new File(targetDir + "/" + baseName + ".xml"); + FileUtils.writeToFile(file, xml); + return file; } private static long getElapsedTime(long startTime) { @@ -251,6 +253,9 @@ private static List findMethodsMatching(String text) { } public static String fromCucumberOptionsTags(String ... tags) { + if (tags == null) { + return null; + } StringBuilder sb = new StringBuilder(); for (int i = 0; i < tags.length; i++) { String and = tags[i]; diff --git a/karate-core/src/main/java/com/intuit/karate/core/ExecutionContext.java b/karate-core/src/main/java/com/intuit/karate/core/ExecutionContext.java new file mode 100644 index 000000000..a3318c9a1 --- /dev/null +++ b/karate-core/src/main/java/com/intuit/karate/core/ExecutionContext.java @@ -0,0 +1,53 @@ +/* + * The MIT License + * + * Copyright 2018 Intuit Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +import com.intuit.karate.CallContext; +import com.intuit.karate.FileLogAppender; +import com.intuit.karate.LogAppender; +import com.intuit.karate.ScriptEnv; +import static com.intuit.karate.core.Engine.getBuildDir; + +/** + * + * @author pthomas3 + */ +public class ExecutionContext { + + public final Feature feature; + public final ScriptEnv env; + public final CallContext callContext; + public final FeatureResult result; + public final LogAppender appender; + + public ExecutionContext(Feature feature, ScriptEnv env, CallContext callContext) { + this.feature = feature; + result = new FeatureResult(feature); + this.env = env; + this.callContext = callContext; + String basePath = feature.getPackageQualifiedName(); + this.appender = new FileLogAppender(getBuildDir() + "/surefire-reports/" + basePath + ".log", env.logger); + } + +} diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureExecutionUnit.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureExecutionUnit.java index 786294134..5435fcfc4 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureExecutionUnit.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureExecutionUnit.java @@ -23,8 +23,6 @@ */ package com.intuit.karate.core; -import com.intuit.karate.LogAppender; -import com.intuit.karate.StepDefs; import com.intuit.karate.exception.KarateException; import java.util.Iterator; import java.util.function.BiConsumer; @@ -36,37 +34,28 @@ */ public class FeatureExecutionUnit implements ExecutionUnit { - private final StepDefs stepDefs; - private final LogAppender appender; - - private final FeatureResult featureResult; - + private final ExecutionContext exec; + private final Iterator iterator; - public FeatureExecutionUnit(Feature feature, StepDefs stepDefs, LogAppender appender) { - this.stepDefs = stepDefs; - this.appender = appender; - featureResult = new FeatureResult(feature); - iterator = feature.getSections().iterator(); - } - - public FeatureResult getFeatureResult() { - return featureResult; - } + public FeatureExecutionUnit(ExecutionContext exec) { + this.exec = exec; + iterator = exec.feature.getSections().iterator(); + } @Override public void submit(Consumer system, BiConsumer next) { if (iterator.hasNext()) { FeatureSection section = iterator.next(); - SectionExecutionUnit unit = new SectionExecutionUnit(section, stepDefs, featureResult, appender); + SectionExecutionUnit unit = new SectionExecutionUnit(section, exec); system.accept(() -> { unit.submit(system, (r, e) -> { FeatureExecutionUnit.this.submit(system, next); }); }); } else { - appender.close(); - next.accept(featureResult, null); + exec.appender.close(); + next.accept(exec.result, null); } } diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java index e309e5ebe..cdd32dc06 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureParser.java @@ -57,15 +57,8 @@ public static Feature parse(File file) { } public static Feature parse(String path) { - if (FileUtils.isClassPath(path)) { - path = FileUtils.removePrefix(path); - return new FeatureParser(path).feature; - } else { - if (FileUtils.isFilePath(path)) { - path = FileUtils.removePrefix(path); - } - return new FeatureParser(new File(path)).feature; - } + File file = FileUtils.fromRelativeClassPath(path); + return parse(file, path); } public static Feature parse(File file, String relativePath) { @@ -76,10 +69,6 @@ private FeatureParser(File file) { this(file, FileUtils.toRelativeClassPath(file)); } - private FeatureParser(String relativePath) { - this(FileUtils.fromRelativeClassPath(relativePath), relativePath); - } - private FeatureParser(File file, String relativePath) { feature = new Feature(file, relativePath); CharStream stream; diff --git a/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java b/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java index a6219086a..f86e4f722 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java +++ b/karate-core/src/main/java/com/intuit/karate/core/FeatureResult.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; /** @@ -36,16 +37,19 @@ * @author pthomas3 */ public class FeatureResult extends HashMap { - + private final String name; private final List elements = new ArrayList(); private final List tags; private final String packageQualifiedName; - + + private int scenarioCount; + private int failedCount; private List errors; - + private long duration; + private ScriptValueMap resultVars; - + public FeatureResult(Feature feature) { put("elements", elements); put("keyword", "Feature"); @@ -60,26 +64,54 @@ public FeatureResult(Feature feature) { if (feature.getDescription() != null) { temp = temp + "\n" + feature.getDescription(); } - put("description", temp.trim()); + put("description", temp.trim()); List list = feature.getTags(); if (list != null) { tags = new ArrayList(list.size()); put("tags", tags); for (Tag tag : list) { - tags.add(new TagResult(tag)); + tags.add(new TagResult(tag)); } } else { - tags = Collections.EMPTY_LIST; + tags = Collections.EMPTY_LIST; } - } + } + + public String getErrorMessages() { + if (errors == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + Iterator iterator = errors.iterator(); + while (iterator.hasNext()) { + Throwable error = iterator.next(); + sb.append(error.getMessage()); + if (iterator.hasNext()) { + sb.append(','); + } + } + return sb.toString(); + } + + public long getDuration() { + return duration; + } + + public int getFailedCount() { + return failedCount; + } + + public int getScenarioCount() { + return scenarioCount; + } public boolean isFailed() { return errors != null && !errors.isEmpty(); - } + } public List getErrors() { return errors; - } + } public ScriptValueMap getResultVars() { return resultVars; @@ -91,19 +123,31 @@ public void setResultVars(ScriptValueMap resultVars) { public String getPackageQualifiedName() { return packageQualifiedName; - } + } public List getTags() { return tags; } - + + public void addError(Throwable error) { + failedCount++; + if (errors == null) { + errors = new ArrayList(); + } + errors.add(error); + } + public void addResult(ResultElement element) { elements.add(element); - if (element.isFailed()) { - if (errors == null) { - errors = new ArrayList(); + duration += element.getDuration(); + if (element.isFailed()) { + if (element.isBackground()) { + scenarioCount++; // since we will never enter the scenario } - errors.add(element.getError()); + addError(element.getError()); + } + if (!element.isBackground()) { + scenarioCount++; } } @@ -114,5 +158,5 @@ public String getName() { public List getElements() { return elements; } - + } diff --git a/karate-core/src/main/java/com/intuit/karate/core/Result.java b/karate-core/src/main/java/com/intuit/karate/core/Result.java index 974f98a84..737ea91a0 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/Result.java +++ b/karate-core/src/main/java/com/intuit/karate/core/Result.java @@ -84,7 +84,7 @@ public static Result skipped() { } public static Result aborted(long duration) { - return new Result(PASSED, duration, null, true); + return new Result(SKIPPED, duration, null, true); } public String getStatus() { diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioExecutionUnit.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioExecutionUnit.java index c9799dc08..729833b33 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/ScenarioExecutionUnit.java +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioExecutionUnit.java @@ -24,7 +24,9 @@ package com.intuit.karate.core; import com.intuit.karate.LogAppender; +import com.intuit.karate.ScriptEnv; import com.intuit.karate.StepDefs; +import com.intuit.karate.cucumber.ScenarioInfo; import com.intuit.karate.exception.KarateException; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -43,11 +45,11 @@ public class ScenarioExecutionUnit implements ExecutionUnit { private boolean backgroundDone = false; private boolean stopped = false; - public ScenarioExecutionUnit(Scenario scenario, StepDefs stepDefs, FeatureResult featureResult, LogAppender appender) { + public ScenarioExecutionUnit(Scenario scenario, StepDefs stepDefs, ExecutionContext exec) { this.scenario = scenario; this.stepDefs = stepDefs; - this.featureResult = featureResult; - this.appender = appender; + this.featureResult = exec.result; + this.appender = exec.appender; if (scenario.getFeature().getBackground() == null) { backgroundDone = true; } @@ -55,10 +57,21 @@ public ScenarioExecutionUnit(Scenario scenario, StepDefs stepDefs, FeatureResult @Override public void submit(Consumer system, BiConsumer next) { + if (stepDefs.callContext.scenarioHook != null) { + try { + stepDefs.callContext.scenarioHook.beforeScenario(scenario); + } catch (Exception e) { + String message = "scenario hook threw fatal error: " + e.getMessage(); + stepDefs.context.logger.error(message); + featureResult.addError(e); + next.accept(featureResult, new KarateException(message, e)); + return; + } + } if (!backgroundDone) { backgroundDone = true; Background background = scenario.getFeature().getBackground(); - BackgroundResult backgroundResult = new BackgroundResult(background); + BackgroundResult backgroundResult = new BackgroundResult(background); system.accept(() -> { StepListExecutionUnit unit = new StepListExecutionUnit(background.getSteps(), scenario, stepDefs, backgroundResult, appender, stopped); unit.submit(system, (r, e) -> { @@ -66,6 +79,7 @@ public void submit(Consumer system, BiConsumer system, BiConsumer { + ScenarioResult scenarioResult = new ScenarioResult(scenario); + system.accept(() -> { StepListExecutionUnit unit = new StepListExecutionUnit(scenario.getSteps(), scenario, stepDefs, scenarioResult, appender, stopped); unit.submit(system, (r, e) -> { // the timing of this line below is important, it collects errors in the feature-result featureResult.addResult(scenarioResult); + if (e != null) { + stepDefs.context.setScenarioError(e); + } next.accept(featureResult, e); }); }); diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioHook.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioHook.java new file mode 100644 index 000000000..b50b47b51 --- /dev/null +++ b/karate-core/src/main/java/com/intuit/karate/core/ScenarioHook.java @@ -0,0 +1,41 @@ +/* + * The MIT License + * + * Copyright 2018 Intuit Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +/** + * + * @author pthomas3 + */ +public interface ScenarioHook { + + /** + * + * @param scenario + * @return if the scenario should be excluded from the test-run + */ + boolean beforeScenario(Scenario scenario); + + void afterScenario(ScenarioResult result); + +} diff --git a/karate-core/src/main/java/com/intuit/karate/core/SectionExecutionUnit.java b/karate-core/src/main/java/com/intuit/karate/core/SectionExecutionUnit.java index 43667c661..146ed7487 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/SectionExecutionUnit.java +++ b/karate-core/src/main/java/com/intuit/karate/core/SectionExecutionUnit.java @@ -23,10 +23,9 @@ */ package com.intuit.karate.core; -import com.intuit.karate.LogAppender; -import com.intuit.karate.Logger; import com.intuit.karate.ScriptEnv; import com.intuit.karate.StepDefs; +import com.intuit.karate.cucumber.ScenarioInfo; import com.intuit.karate.exception.KarateException; import java.util.ArrayList; import java.util.Collection; @@ -42,9 +41,7 @@ */ public class SectionExecutionUnit implements ExecutionUnit { - private final StepDefs stepDefs; - private final FeatureResult featureResult; - private final LogAppender appender; + private final ExecutionContext exec; private final boolean outline; private final Iterator iterator; @@ -52,10 +49,8 @@ public class SectionExecutionUnit implements ExecutionUnit { private int index = 0; private List errors; - public SectionExecutionUnit(FeatureSection section, StepDefs stepDefs, FeatureResult featureResult, LogAppender appender) { - this.stepDefs = stepDefs; - this.featureResult = featureResult; - this.appender = appender; + public SectionExecutionUnit(FeatureSection section, ExecutionContext exec) { + this.exec = exec; if (section.isOutline()) { outline = true; iterator = section.getScenarioOutline().getScenarios().iterator(); @@ -70,14 +65,20 @@ public void submit(Consumer system, BiConsumer tagsEffective = scenario.getTagsEffective(); if (!Tags.evaluate(env.tagSelector, tagsEffective)) { - Logger logger = stepDefs.context.logger; - logger.trace("skipping scenario at line: {} with tags effective: {}", scenario.getLine(), tagsEffective); + env.logger.trace("skipping scenario at line: {} with tags effective: {}", scenario.getLine(), tagsEffective); SectionExecutionUnit.this.submit(system, next); } else { - ScenarioExecutionUnit unit = new ScenarioExecutionUnit(scenario, stepDefs, featureResult, appender); + // this is where the script-context and vars are inited for a scenario + // karate-config.js will be processed here + exec.callContext.setScenarioInfo(getScenarioInfo(scenario, env)); + StepDefs stepDefs = new StepDefs(env, exec.callContext); + // we hold a reference to the LAST scenario executed + // for cases where the caller needs a result + exec.result.setResultVars(stepDefs.context.getVars()); + ScenarioExecutionUnit unit = new ScenarioExecutionUnit(scenario, stepDefs, exec); system.accept(() -> { unit.submit(system, (r, e) -> { if (outline) { @@ -110,8 +111,18 @@ public void submit(Consumer system, BiConsumer { - private final Step step; - private final StepDefs stepDefs; + private final Step step; private final Scenario scenario; + private final StepDefs stepDefs; - public StepExecutionUnit(Step step, StepDefs stepDefs, Scenario scenario) { - this.step = step; - this.stepDefs = stepDefs; + public StepExecutionUnit(Step step, Scenario scenario, StepDefs stepDefs) { + this.step = step; this.scenario = scenario; + this.stepDefs = stepDefs; } @Override diff --git a/karate-core/src/main/java/com/intuit/karate/core/StepListExecutionUnit.java b/karate-core/src/main/java/com/intuit/karate/core/StepListExecutionUnit.java index cb2cd6752..b8a6cffe9 100644 --- a/karate-core/src/main/java/com/intuit/karate/core/StepListExecutionUnit.java +++ b/karate-core/src/main/java/com/intuit/karate/core/StepListExecutionUnit.java @@ -67,7 +67,7 @@ public void submit(Consumer system, BiConsumer { - StepExecutionUnit unit = new StepExecutionUnit(step, stepDefs, scenario); + StepExecutionUnit unit = new StepExecutionUnit(step, scenario, stepDefs); unit.submit(system, (r, e) -> { // log appender collection for each step happens here StepResult stepResult = new StepResult(step, r); diff --git a/karate-core/src/main/java/com/intuit/karate/cucumber/CucumberRunner.java b/karate-core/src/main/java/com/intuit/karate/cucumber/CucumberRunner.java index 5e46cd11c..86b227b7a 100644 --- a/karate-core/src/main/java/com/intuit/karate/cucumber/CucumberRunner.java +++ b/karate-core/src/main/java/com/intuit/karate/cucumber/CucumberRunner.java @@ -24,28 +24,25 @@ package com.intuit.karate.cucumber; import com.intuit.karate.CallContext; +import com.intuit.karate.FileResource; import com.intuit.karate.FileUtils; -import com.intuit.karate.ScriptValueMap; import com.intuit.karate.core.Engine; import com.intuit.karate.core.Feature; import com.intuit.karate.core.FeatureParser; import com.intuit.karate.core.FeatureResult; -import com.intuit.karate.filter.TagFilter; -import com.intuit.karate.filter.TagFilterException; -import cucumber.runtime.model.CucumberFeature; +import com.intuit.karate.core.ScenarioHook; +import cucumber.api.CucumberOptions; import java.io.File; -import java.net.URL; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.ServiceLoader; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import cucumber.runtime.model.CucumberTagStatement; +import java.util.Arrays; +import java.util.Collections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,83 +54,85 @@ public class CucumberRunner { private static final Logger logger = LoggerFactory.getLogger(CucumberRunner.class); - public static KarateStats parallel(Class clazz, int threadCount) { + public static KarateStats parallel(Class clazz, int threadCount) { return parallel(clazz, threadCount, null); } - public static KarateStats parallel(Class clazz, int threadCount, String reportDir) { - KarateRuntimeOptions kro = new KarateRuntimeOptions(clazz); - List karateFeatures = KarateFeature.loadFeatures(kro); - return parallel(karateFeatures, threadCount, reportDir); - } - - /** - * - * @param tags - use a single list item (comma delimited) for OR, use multiple items for AND - * @param paths - can be paths to feature files or directories that will be scanned - * @param threadCount - number of threads for parallel runner - * @param reportDir - can be null, and defaults to "target/surefire-reports" - * @return stats object - */ + public static KarateStats parallel(Class clazz, int threadCount, String reportDir) { + CucumberOptions co = clazz.getAnnotation(CucumberOptions.class); + List tags; + List features; + if (co == null) { + logger.warn("CucumberOptions annotation not found on class: {}", clazz); + tags = null; + features = null; + } else { + String[] tagsArray = co.tags(); + tags = Arrays.asList(tagsArray); + String[] featuresArray = co.features(); + features = Arrays.asList(featuresArray); + } + if (features == null || features.isEmpty()) { + String relative = FileUtils.toRelativeClassPath(clazz); + features = Collections.singletonList(relative); + } + return parallel(tags, features, null, threadCount, reportDir); + } + public static KarateStats parallel(List tags, List paths, int threadCount, String reportDir) { - KarateRuntimeOptions kro = new KarateRuntimeOptions(tags, paths); - List karateFeatures = KarateFeature.loadFeatures(kro); - return parallel(karateFeatures, threadCount, reportDir); + return parallel(tags, paths, null, threadCount, reportDir); + } + + public static KarateStats parallel(List tags, List paths, ScenarioHook hook, int threadCount, String reportDir) { + String tagSelector = tags == null ? null : Engine.fromCucumberOptionsTags(tags.toArray(new String[]{})); + List files = FileUtils.scanForFeatureFiles(paths); + return parallel(tagSelector, files, hook, threadCount, reportDir); } - - public static KarateStats parallel(List karateFeatures, int threadCount, String userReportDir) { - String reportDir = userReportDir == null ? "target/surefire-reports" : userReportDir; + + public static KarateStats parallel(String tagSelector, List resources, int threadCount, String reportDir) { + return parallel(tagSelector, resources, null, threadCount, reportDir); + } + + public static KarateStats parallel(String tagSelector, List resources, ScenarioHook hook, int threadCount, String reportDir) { + if (reportDir == null) { + reportDir = Engine.getBuildDir() + "/surefire-reports"; + } + final String finalReportDir = reportDir; logger.info("Karate version: {}", FileUtils.getKarateVersion()); KarateStats stats = KarateStats.startTimer(); ExecutorService executor = Executors.newFixedThreadPool(threadCount); + int executedFeatureCount = 0; try { - int count = karateFeatures.size(); - int filteredCount = 0; - List> callables = new ArrayList<>(count); + int count = resources.size(); + List> callables = new ArrayList<>(count); for (int i = 0; i < count; i++) { - KarateFeature karateFeature = karateFeatures.get(i); + FileResource resource = resources.get(i); int index = i + 1; - CucumberFeature feature = karateFeature.getFeature(); - filterOnTags(feature); - if (!feature.getFeatureElements().isEmpty()) { - callables.add(() -> { - // we are now within a separate thread. the reporter filters logs by self thread - String threadName = Thread.currentThread().getName(); - KarateJunitAndJsonReporter reporter = KarateJunitAndJsonReporter.getInstance(feature.getPath(), reportDir); - KarateRuntime runtime = karateFeature.getRuntime(reporter); - try { - feature.run(reporter, reporter, runtime); - runtime.afterFeature(); - logger.info("<<<< feature {} of {} on thread {}: {}", index, count, threadName, feature.getPath()); - } catch (Exception e) { - logger.error("karate xml/json generation failed for: {}", feature.getPath()); - reporter.setFailureReason(e); - } finally { // try our best to close the report file gracefully so that report generation is not broken - reporter.done(); - } - return reporter; - }); - } else { - filteredCount++; - } + Feature feature = FeatureParser.parse(resource.file, resource.relativePath); + // filterOnTags(feature); + callables.add(() -> { + // we are now within a separate thread. the reporter filters logs by self thread + String threadName = Thread.currentThread().getName(); + FeatureResult result = Engine.executeSync(null, feature, tagSelector, new CallContext(hook)); + logger.info("<<<< feature {} of {} on thread {}: {}", index, count, threadName, feature.getRelativePath()); + Engine.saveResultJson(finalReportDir, result); + Engine.saveResultXml(finalReportDir, result); + return result; + }); } - stats.setFeatureCount(count - filteredCount); - - List> futures = executor.invokeAll(callables); + List> futures = executor.invokeAll(callables); stats.stopTimer(); - for (Future future : futures) { - KarateJunitAndJsonReporter reporter = future.get(); // guaranteed to be not-null - KarateJunitFormatter formatter = reporter.getJunitFormatter(); - if (reporter.getFailureReason() != null) { - logger.error("karate xml/json generation failed: {}", formatter.getFeaturePath()); - logger.error("karate xml/json error stack trace", reporter.getFailureReason()); + for (Future future : futures) { + FeatureResult result = future.get(); // guaranteed to be not-null + int scenarioCount = result.getScenarioCount(); + stats.addToTestCount(scenarioCount); + if (scenarioCount != 0) { + executedFeatureCount++; } - stats.addToTestCount(formatter.getTestCount()); - stats.addToFailCount(formatter.getFailCount()); - stats.addToSkipCount(formatter.getSkipCount()); - stats.addToTimeTaken(formatter.getTimeTaken()); - if (formatter.isFail()) { - stats.addToFailedList(formatter.getFeaturePath(), formatter.getFailMessages() + ""); + stats.addToFailCount(result.getFailedCount()); + stats.addToTimeTaken(result.getDuration()); + if (result.isFailed()) { + stats.addToFailedList(result.getPackageQualifiedName(), result.getErrorMessages()); } } } catch (Exception e) { @@ -142,41 +141,20 @@ public static KarateStats parallel(List karateFeatures, int threa } finally { executor.shutdownNow(); } + stats.setFeatureCount(executedFeatureCount); stats.printStats(threadCount); return stats; } - private static void filterOnTags(CucumberFeature feature) throws TagFilterException { - final List featureElements = feature.getFeatureElements(); - ServiceLoader loader = ServiceLoader.load(TagFilter.class); - for (Iterator iterator = featureElements.iterator(); iterator.hasNext();) { - CucumberTagStatement cucumberTagStatement = iterator.next(); - for (TagFilter implClass : loader) { - logger.info("Tag filter found: {}", implClass.getClass().getSimpleName()); - final boolean isFiltered = implClass.filter(feature, cucumberTagStatement); - if (isFiltered) { - logger.info("skipping feature element {} of feature {} due to feature-tag-filter {} ", - cucumberTagStatement.getVisualName(), - feature.getPath(), implClass.getClass().getSimpleName()); - iterator.remove(); - break; - } - } - } - } - - public static Map runFeature(File file, Map vars, boolean evalKarateConfig) { + public static Map runFeature(Feature feature, Map vars, boolean evalKarateConfig) { CallContext callContext = new CallContext(vars, evalKarateConfig); - return runFeature(file, callContext); + FeatureResult result = Engine.executeSync(null, feature, null, callContext); + return result.getResultVars().toPrimitiveMap(); } - public static Map runFeature(File file, CallContext callContext) { - FeatureWrapper featureWrapper = FeatureWrapper.fromFile(file, null, null); - ScriptValueMap scriptValueMap = CucumberUtils.callSync(featureWrapper, callContext); - return scriptValueMap.toPrimitiveMap(); -// Feature feature = FeatureParser.parse(file); -// FeatureResult result = Engine.executeSync(null, feature, null, callContext); -// return result.getResultVars().toPrimitiveMap(); + public static Map runFeature(File file, Map vars, boolean evalKarateConfig) { + Feature feature = FeatureParser.parse(file); + return runFeature(feature, vars, evalKarateConfig); } public static Map runFeature(Class relativeTo, String path, Map vars, boolean evalKarateConfig) { @@ -184,13 +162,9 @@ public static Map runFeature(Class relativeTo, String path, Map< return runFeature(file, vars, evalKarateConfig); } - public static Map runClasspathFeature(String classPath, Map vars, boolean evalKarateConfig) { - URL url = Thread.currentThread().getContextClassLoader().getResource(classPath); - if (url == null) { - throw new RuntimeException("file not found: " + classPath); - } - File file = new File(url.getFile()); - return runFeature(file, vars, evalKarateConfig); + public static Map runFeature(String path, Map vars, boolean evalKarateConfig) { + Feature feature = FeatureParser.parse(path); + return runFeature(feature, vars, evalKarateConfig); } } diff --git a/karate-core/src/main/java/com/intuit/karate/filter/TagFilter.java b/karate-core/src/main/java/com/intuit/karate/filter/TagFilter.java deleted file mode 100644 index 869762521..000000000 --- a/karate-core/src/main/java/com/intuit/karate/filter/TagFilter.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.intuit.karate.filter; - -import cucumber.runtime.model.CucumberFeature; -import cucumber.runtime.model.CucumberTagStatement; - -/** - * Interface that should be implemented to plug in - * any filtering based on cucumber tags - * - * To add a new filter, implement this interface - * and add the fully qualified name of the implementing class - * in resources/META-INF/services/com.intuit.karate.filter.TagFilter - * - * @author ssishtla - */ - -@FunctionalInterface -public interface TagFilter { - - /** - * Method to be over-ridden to evaluate if a feature/scenario - * should be skipped for execution - * - * @param feature - * @param cucumberTagStatement - * @return true - indicates feature should be skipped from - * execution, otherwise false - * @throws TagFilterException - when thrown, stops - * execution of all features - */ - boolean filter(CucumberFeature feature, CucumberTagStatement cucumberTagStatement) throws TagFilterException; -} diff --git a/karate-core/src/main/java/com/intuit/karate/filter/TagFilterException.java b/karate-core/src/main/java/com/intuit/karate/filter/TagFilterException.java deleted file mode 100644 index 58f08650f..000000000 --- a/karate-core/src/main/java/com/intuit/karate/filter/TagFilterException.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.intuit.karate.filter; - -public class TagFilterException extends Exception { - - public TagFilterException() { - super(); - } - - public TagFilterException(String message) { - super(message); - } -} diff --git a/karate-core/src/test/java/com/intuit/karate/FileUtilsTest.java b/karate-core/src/test/java/com/intuit/karate/FileUtilsTest.java index a24ae8a77..4e999109d 100755 --- a/karate-core/src/test/java/com/intuit/karate/FileUtilsTest.java +++ b/karate-core/src/test/java/com/intuit/karate/FileUtilsTest.java @@ -67,7 +67,7 @@ public void testRenameZeroLengthFile() { @Test public void testScanFiles() { - String relativePath = "com/intuit/karate/ui/test.feature"; + String relativePath = "classpath:com/intuit/karate/ui/test.feature"; List files = FileUtils.scanForFeatureFilesOnClassPath(); boolean found = false; for (FileResource file : files) { @@ -83,4 +83,9 @@ public void testScanFiles() { assertTrue(found); } + @Test + public void testRelativePathForClass() { + assertEquals("classpath:com/intuit/karate", FileUtils.toRelativeClassPath(getClass())); + } + } diff --git a/karate-core/src/test/java/com/intuit/karate/core/MandatoryTagHook.java b/karate-core/src/test/java/com/intuit/karate/core/MandatoryTagHook.java new file mode 100644 index 000000000..14511f96f --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/core/MandatoryTagHook.java @@ -0,0 +1,56 @@ +/* + * The MIT License + * + * Copyright 2018 Intuit Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +import java.util.Collection; +import java.util.List; + +/** + * + * @author pthomas3 + */ +public class MandatoryTagHook implements ScenarioHook { + + @Override + public boolean beforeScenario(Scenario scenario) { + Collection tags = scenario.getTagsEffective(); + boolean found = false; + for (Tag tag : tags) { + if ("testId".equals(tag.getName())) { + found = true; + break; + } + } + if (!found) { + throw new RuntimeException("testId tag not present at line: " + scenario.getLine()); + } + return true; + } + + @Override + public void afterScenario(ScenarioResult result) { + + } + +} diff --git a/karate-core/src/test/java/com/intuit/karate/core/ScenarioHookTest.java b/karate-core/src/test/java/com/intuit/karate/core/ScenarioHookTest.java new file mode 100644 index 000000000..c2341ba4c --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/core/ScenarioHookTest.java @@ -0,0 +1,55 @@ +/* + * The MIT License + * + * Copyright 2018 Intuit Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.core; + +import com.intuit.karate.cucumber.CucumberRunner; +import com.intuit.karate.cucumber.KarateStats; +import java.util.Collections; +import static org.junit.Assert.*; +import org.junit.Test; + +/** + * + * @author pthomas3 + */ +public class ScenarioHookTest { + + @Test + public void testStopIfScenarioHasNoTags() { + String path = "classpath:com/intuit/karate/core/test-hook-notags.feature"; + KarateStats stats = CucumberRunner.parallel(null, Collections.singletonList(path), new MandatoryTagHook(), 1, null); + assertEquals(0, stats.getFeatureCount()); + assertEquals(1, stats.getFailCount()); + } + + @Test + public void testHookForExamplesWithTags() { + String path = "classpath:com/intuit/karate/core/test-hook-multiexample.feature"; + KarateStats stats = CucumberRunner.parallel(null, Collections.singletonList(path), new MandatoryTagHook(), 1, null); + assertEquals(1, stats.getFeatureCount()); + assertEquals(7, stats.getTestCount()); + assertEquals(0, stats.getFailCount()); + } + +} diff --git a/karate-core/src/test/java/com/intuit/karate/filter/tag-filter-multiscenario.feature b/karate-core/src/test/java/com/intuit/karate/core/test-hook-multiexample.feature similarity index 100% rename from karate-core/src/test/java/com/intuit/karate/filter/tag-filter-multiscenario.feature rename to karate-core/src/test/java/com/intuit/karate/core/test-hook-multiexample.feature diff --git a/karate-core/src/test/java/com/intuit/karate/core/test-hook-notags.feature b/karate-core/src/test/java/com/intuit/karate/core/test-hook-notags.feature new file mode 100644 index 000000000..aa6ff4f9f --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/core/test-hook-notags.feature @@ -0,0 +1,5 @@ +@ignore +Feature: no tags feature + + Scenario: no tags scenario + diff --git a/karate-core/src/test/java/com/intuit/karate/cucumber/CucumberRunnerTest.java b/karate-core/src/test/java/com/intuit/karate/cucumber/CucumberRunnerTest.java index 07c5e90fe..08199e2ba 100644 --- a/karate-core/src/test/java/com/intuit/karate/cucumber/CucumberRunnerTest.java +++ b/karate-core/src/test/java/com/intuit/karate/cucumber/CucumberRunnerTest.java @@ -24,6 +24,10 @@ package com.intuit.karate.cucumber; import com.intuit.karate.FileUtils; +import com.intuit.karate.core.Engine; +import com.intuit.karate.core.Feature; +import com.intuit.karate.core.FeatureParser; +import com.intuit.karate.core.FeatureResult; import cucumber.api.CucumberOptions; import java.io.File; import java.util.Map; @@ -47,41 +51,35 @@ private boolean contains(String reportPath, String textToFind) { return contents.contains(textToFind); } - public static KarateJunitAndJsonReporter run(File file, String reportPath) throws Exception { - KarateFeature kf = new KarateFeature(file); - KarateJunitAndJsonReporter reporter = new KarateJunitAndJsonReporter(file.getPath(), reportPath); - KarateRuntime runtime = kf.getRuntime(reporter); - kf.getFeature().run(reporter, reporter, runtime); - reporter.done(); - return reporter; - } + private static String resultXml(String name) { + Feature feature = FeatureParser.parse("classpath:com/intuit/karate/cucumber/" + name); + FeatureResult result = Engine.executeSync(null, feature, null, null); + File file = Engine.saveResultXml("target", result); + return FileUtils.toString(file); + } @Test public void testScenario() throws Exception { - String reportPath = "target/scenario.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/scenario.feature"); - run(file, reportPath); - assertTrue(contains(reportPath, "Then match b == { foo: 'bar'}")); + String contents = resultXml("scenario.feature"); + assertTrue(contents.contains("Then match b == { foo: 'bar'}")); } @Test public void testScenarioOutline() throws Exception { - String reportPath = "target/outline.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/outline.feature"); - run(file, reportPath); - assertTrue(contains(reportPath, "When def a = 55")); + String contents = resultXml("outline.feature"); + assertTrue(contents.contains("When def a = 55")); } @Test public void testParallel() { KarateStats stats = CucumberRunner.parallel(getClass(), 1); assertEquals(2, stats.getFailCount()); - String pathBase = "target/surefire-reports/TEST-com.intuit.karate.cucumber."; + String pathBase = "target/surefire-reports/com.intuit.karate.cucumber."; assertTrue(contains(pathBase + "scenario.xml", "Then match b == { foo: 'bar'}")); assertTrue(contains(pathBase + "outline.xml", "Then assert a == 55")); assertTrue(contains(pathBase + "multi-scenario.xml", "Then assert a != 2")); // a scenario failure should not stop other features from running - assertTrue(contains(pathBase + "multi-scenario-fail.xml", "Then assert a != 2..........................................................passed")); + assertTrue(contains(pathBase + "multi-scenario-fail.xml", "Then assert a != 2 ........................................................ passed")); assertEquals(2, stats.getFailedMap().size()); assertTrue(stats.getFailedMap().keySet().contains("com.intuit.karate.cucumber.no-scenario-name")); assertTrue(stats.getFailedMap().keySet().contains("com.intuit.karate.cucumber.multi-scenario-fail")); @@ -98,7 +96,7 @@ public void testRunningFeatureFromJavaApi() { @Test public void testRunningRelativePathFeatureFromJavaApi() { - Map result = CucumberRunner.runClasspathFeature("com/intuit/karate/test-called.feature", null, true); + Map result = CucumberRunner.runFeature("classpath:com/intuit/karate/test-called.feature", null, true); assertEquals(1, result.get("a")); assertEquals(2, result.get("b")); assertEquals("someValue", result.get("someConfig")); @@ -106,11 +104,9 @@ public void testRunningRelativePathFeatureFromJavaApi() { @Test public void testCallerArg() throws Exception { - String reportPath = "target/caller-arg.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/caller-arg.feature"); - run(file, reportPath); - assertFalse(contains(reportPath, "failed")); - assertTrue(contains(reportPath, "* def result = call read('called-arg-null.feature')")); + String contents = resultXml("caller-arg.feature"); + assertFalse(contents.contains("failed")); + assertTrue(contents.contains("* def result = call read('called-arg-null.feature')")); } } diff --git a/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureResultTest.java b/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureResultTest.java index 797780f29..164539cd4 100644 --- a/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureResultTest.java +++ b/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureResultTest.java @@ -24,6 +24,10 @@ package com.intuit.karate.cucumber; import com.intuit.karate.FileUtils; +import com.intuit.karate.core.Engine; +import com.intuit.karate.core.Feature; +import com.intuit.karate.core.FeatureParser; +import com.intuit.karate.core.FeatureResult; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,53 +43,57 @@ public class FeatureResultTest { private static final Logger logger = LoggerFactory.getLogger(FeatureResultTest.class); + + private static FeatureResult result(String name) { + Feature feature = FeatureParser.parse("classpath:com/intuit/karate/cucumber/" + name); + return Engine.executeSync(null, feature, null, null); + } + + private static String xml(FeatureResult result) { + File file = Engine.saveResultXml("target", result); + return FileUtils.toString(file); + } @Test public void testFailureMultiScenarioFeature() throws Exception { - String reportPath = "target/failed-feature-result.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/failed.feature"); - KarateJunitAndJsonReporter reporter = CucumberRunnerTest.run(file, reportPath); - KarateJunitFormatter formatter = reporter.getJunitFormatter(); - assertEquals(2, formatter.getFailCount()); - assertEquals(3, formatter.getTestCount()); - String contents = FileUtils.toString(new File(reportPath)); + FeatureResult result = result("failed.feature"); + assertEquals(2, result.getFailedCount()); + assertEquals(3, result.getScenarioCount()); + String contents = xml(result); assertTrue(contents.contains("assert evaluated to false: a != 1")); assertTrue(contents.contains("assert evaluated to false: a == 3")); // failure1 should have first step as failure, and second step as skipped // TODO: generate the expected content string, below code puts a hard dependency // with KarateJunitFormatter$TestCase.addStepAndResultListing() - assertTrue(contents.contains("Then assert a != 1..........................................................failed")); - assertTrue(contents.contains("And assert a == 2...........................................................skipped")); + assertTrue(contents.contains("Then assert a != 1 ........................................................ failed")); + assertTrue(contents.contains("And assert a == 2 ......................................................... skipped")); // failure2 should have first step as passed, and second step as failed - assertTrue(contents.contains("Then assert a != 2..........................................................passed")); - assertTrue(contents.contains("And assert a == 3...........................................................failed")); + assertTrue(contents.contains("Then assert a != 2 ........................................................ passed")); + assertTrue(contents.contains("And assert a == 3 ......................................................... failed")); // pass1 should have both steps as passed - assertTrue(contents.contains("Then assert a != 4..........................................................passed")); - assertTrue(contents.contains("And assert a != 5...........................................................passed")); + assertTrue(contents.contains("Then assert a != 4 ........................................................ passed")); + assertTrue(contents.contains("And assert a != 5 ......................................................... passed")); } @Test - public void testAbortMultiScenarioFeature() throws Exception { - String reportPath = "target/aborted-feature-result.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/aborted.feature"); - KarateJunitAndJsonReporter reporter = CucumberRunnerTest.run(file, reportPath); - KarateJunitFormatter formatter = reporter.getJunitFormatter(); - assertEquals(0, formatter.getFailCount()); - assertEquals(3, formatter.getTestCount()); - String contents = FileUtils.toString(new File(reportPath)); + public void testAbortMultiScenarioFeature() throws Exception { + FeatureResult result = result("aborted.feature"); + assertEquals(0, result.getFailedCount()); + assertEquals(3, result.getScenarioCount()); + String contents = xml(result); // skip-pass and skip-fail both should have all steps as skipped // TODO: generate the expected content string, below code puts a hard dependency // with KarateJunitFormatter$TestCase.addStepAndResultListing() - assertTrue(contents.contains("* eval karate.abort().......................................................skipped")); - assertTrue(contents.contains("* assert a == 1.............................................................skipped")); - assertTrue(contents.contains("* assert a == 2.............................................................skipped")); + assertTrue(contents.contains("* eval karate.abort() ..................................................... skipped")); + assertTrue(contents.contains("* assert a == 1 ........................................................... skipped")); + assertTrue(contents.contains("* assert a == 2 ........................................................... skipped")); // noskip should have both steps as passed - assertTrue(contents.contains("Then assert a != 3..........................................................passed")); - assertTrue(contents.contains("And assert a != 4...........................................................passed")); + assertTrue(contents.contains("Then assert a != 3 ........................................................ passed")); + assertTrue(contents.contains("And assert a != 4 ......................................................... passed")); } } diff --git a/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureReuseTest.java b/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureReuseTest.java index 0f2d7652e..21cd51008 100644 --- a/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureReuseTest.java +++ b/karate-core/src/test/java/com/intuit/karate/cucumber/FeatureReuseTest.java @@ -42,32 +42,28 @@ public class FeatureReuseTest { private static final Logger logger = LoggerFactory.getLogger(FeatureReuseTest.class); + private static String resultXml(String name) { + Feature feature = FeatureParser.parse("classpath:com/intuit/karate/cucumber/" + name); + FeatureResult result = Engine.executeSync(null, feature, null, null); + File file = Engine.saveResultXml("target", result); + return FileUtils.toString(file); + } + @Test public void testFailureInCalledShouldFailTest() throws Exception { - Feature feature = FeatureParser.parse("classpath:com/intuit/karate/cucumber/caller.feature"); - FeatureResult result = Engine.executeSync(null, feature, null, null); - Engine.saveResultXml("target", result); - String contents = FileUtils.toString(new File("target/" + result.getPackageQualifiedName() + ".xml")); + String contents = resultXml("caller.feature"); assertTrue(contents.contains("assert evaluated to false: input != 4")); } @Test public void testArgumentsPassedForSharedScope() throws Exception { - String reportPath = "target/pass.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/caller-shared.feature"); - KarateJunitAndJsonReporter reporter = CucumberRunnerTest.run(file, reportPath); - assertEquals(0, reporter.getJunitFormatter().getFailCount()); - String contents = FileUtils.toString(new File(reportPath)); + String contents = resultXml("caller-shared.feature"); assertTrue(contents.contains("passed")); } @Test public void testCallerTwo() throws Exception { - String reportPath = "target/pass2.xml"; - File file = new File("src/test/java/com/intuit/karate/cucumber/caller_2.feature"); - KarateJunitAndJsonReporter reporter = CucumberRunnerTest.run(file, reportPath); - assertEquals(0, reporter.getJunitFormatter().getFailCount()); - String contents = FileUtils.toString(new File(reportPath)); + String contents = resultXml("caller_2.feature"); assertTrue(contents.contains("passed")); } diff --git a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterMultiScenarioTest.java b/karate-core/src/test/java/com/intuit/karate/filter/TagFilterMultiScenarioTest.java deleted file mode 100644 index f111fafc6..000000000 --- a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterMultiScenarioTest.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.intuit.karate.filter; - -import com.intuit.karate.cucumber.CucumberRunner; -import com.intuit.karate.cucumber.KarateStats; -import cucumber.api.CucumberOptions; -import org.junit.Test; -import static org.junit.Assert.*; - -@CucumberOptions(tags = {"@testId=5"}) -public class TagFilterMultiScenarioTest { - - @Test - public void testParallel() { - String karateOutputPath = "target/surefire-reports"; - KarateStats stats = CucumberRunner.parallel(getClass(), 5, karateOutputPath); - if (stats != null) { - assertEquals(1, stats.getFeatureCount()); - assertNull(stats.getFailureReason()); - } else { - fail("test was expecting an exception"); - } - } - -} diff --git a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTest.java b/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTest.java deleted file mode 100644 index 2dc747950..000000000 --- a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.intuit.karate.filter; - -import com.intuit.karate.cucumber.CucumberRunner; -import com.intuit.karate.cucumber.KarateStats; -import cucumber.api.CucumberOptions; -import static org.junit.Assert.*; -import org.junit.Test; - -/** - * - * @author ssishtla - */ -@CucumberOptions(features = "classpath:com/intuit/karate/filter/tag-filter.feature") -public class TagFilterTest { - - @Test - public void testParallel() throws Exception { - String karateOutputPath = "target/surefire-reports"; - KarateStats stats = CucumberRunner.parallel(getClass(), 5, karateOutputPath); - if (stats != null) { - assertEquals(0, stats.getFeatureCount()); - assertNotNull(stats.getFailureReason()); - assertTrue(stats.getFailureReason() instanceof TagFilterException); - assertEquals("Feature: tag-filter.feature failed due to tag filtering", stats.getFailureReason().getMessage()); - } else { - fail("Test was expecting an exception"); - } - } - -} diff --git a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTestImpl.java b/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTestImpl.java deleted file mode 100644 index 81b126484..000000000 --- a/karate-core/src/test/java/com/intuit/karate/filter/TagFilterTestImpl.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.intuit.karate.filter; - -import cucumber.runtime.model.CucumberExamples; -import cucumber.runtime.model.CucumberFeature; -import cucumber.runtime.model.CucumberScenarioOutline; -import cucumber.runtime.model.CucumberTagStatement; -import gherkin.formatter.model.Examples; -import gherkin.formatter.model.Scenario; -import gherkin.formatter.model.ScenarioOutline; -import gherkin.formatter.model.Tag; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -public class TagFilterTestImpl implements TagFilter { - - private static final String TAG_FILTER_FEATURE = "tag-filter.feature"; - private static final String TAG_FILTER_MULTISCENARIO_FEATURE = "tag-filter-multiscenario.feature"; - - @Override - public boolean filter(CucumberFeature feature, CucumberTagStatement cucumberTagStatement) throws TagFilterException { - String path = feature.getPath(); - if (path.endsWith(TAG_FILTER_FEATURE)) { - throw new TagFilterException("Feature: " + TAG_FILTER_FEATURE + " failed due to tag filtering"); - } - if (path.endsWith(TAG_FILTER_MULTISCENARIO_FEATURE)) { - validateTags(feature, cucumberTagStatement); - } - return false; - } - - private void validateTags(CucumberFeature cucumberFeature, CucumberTagStatement cucumberTagStatement) throws TagFilterException { - Set effectiveTags = null; - if (cucumberTagStatement.getGherkinModel() instanceof ScenarioOutline) { - final List cucumberExamplesList = ((CucumberScenarioOutline) cucumberTagStatement).getCucumberExamplesList(); - if (cucumberExamplesList.isEmpty()) { - return; // if no examples matched cucumber options, no point checking for tags - } - for (CucumberExamples cucumberExamples : cucumberExamplesList) { - effectiveTags = getAllTagsForScenarioOutline(cucumberExamples.getExamples(), cucumberFeature, cucumberTagStatement); - } - } else if (cucumberTagStatement.getGherkinModel() instanceof Scenario) { - effectiveTags = getAllTagsForScenario(cucumberFeature, cucumberTagStatement); - } - if (null == effectiveTags) { - throw new TagFilterException("Required tags missing for feature " + cucumberFeature.getPath()); - } - } - - private Set getAllTagsForScenario(CucumberFeature cucumberFeature, CucumberTagStatement cucumberTagStatement) { - Set tags = new HashSet<>(); - tags.addAll(cucumberFeature.getGherkinFeature().getTags()); - tags.addAll(cucumberTagStatement.getGherkinModel().getTags()); - return tags; - } - - private Set getAllTagsForScenarioOutline(Examples examples, CucumberFeature cucumberFeature, CucumberTagStatement cucumberTagStatement) { - Set tags = new HashSet<>(); - tags.addAll(getAllTagsForScenario(cucumberFeature, cucumberTagStatement)); - tags.addAll(examples.getTags()); - return tags; - } - -} diff --git a/karate-core/src/test/java/com/intuit/karate/filter/tag-filter.feature b/karate-core/src/test/java/com/intuit/karate/filter/tag-filter.feature deleted file mode 100644 index b841b91cf..000000000 --- a/karate-core/src/test/java/com/intuit/karate/filter/tag-filter.feature +++ /dev/null @@ -1,8 +0,0 @@ -@ignore -Feature: Tag Filter feature - - Scenario: Tag Filter scenario - - - - diff --git a/karate-demo/src/test/java/demo/java/JavaApiTest.java b/karate-demo/src/test/java/demo/java/JavaApiTest.java index 83ce36774..d511a3642 100644 --- a/karate-demo/src/test/java/demo/java/JavaApiTest.java +++ b/karate-demo/src/test/java/demo/java/JavaApiTest.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; +import org.junit.BeforeClass; import org.junit.Test; /** @@ -12,6 +13,12 @@ */ public class JavaApiTest { + @BeforeClass + public static void beforeClass() { + // skip 'callSingle' in karate-config.js + System.setProperty("karate.env", "mock"); + } + @Test public void testCallingFeatureFromJava() { Map args = new HashMap(); @@ -24,7 +31,7 @@ public void testCallingFeatureFromJava() { public void testCallingClasspathFeatureFromJava() { Map args = new HashMap(); args.put("name", "World"); - Map result = CucumberRunner.runClasspathFeature("demo/java/from-java.feature", args, true); + Map result = CucumberRunner.runFeature("classpath:demo/java/from-java.feature", args, true); assertEquals("Hello World", result.get("greeting")); } diff --git a/karate-gatling/src/main/scala/com/intuit/karate/gatling/KarateAction.scala b/karate-gatling/src/main/scala/com/intuit/karate/gatling/KarateAction.scala index 47a3b12c6..4398493af 100644 --- a/karate-gatling/src/main/scala/com/intuit/karate/gatling/KarateAction.scala +++ b/karate-gatling/src/main/scala/com/intuit/karate/gatling/KarateAction.scala @@ -98,7 +98,7 @@ class KarateAction(val name: String, val callTag: String, val protocol: KaratePr val asyncSystem: Consumer[Runnable] = r => getActor() ! r val asyncNext: Runnable = () => next ! session - val callContext = new CallContext(null, 0, null, -1, false, true, null, asyncSystem, asyncNext, stepInterceptor) + val callContext = new CallContext(null, 0, null, -1, false, true, null, asyncSystem, asyncNext, stepInterceptor, null) CucumberUtils.callAsync(name, callTag, callContext)