diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs index 07f1680eb..8f9fe49af 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs @@ -157,7 +157,7 @@ public void AddNuGetPackage(string nugetPackage, string nugetVersion) _solutionDriver.DefaultProject.AddNuGetPackage(nugetPackage, nugetVersion); } - public void AddFailingStepBinding(string scenarioBlock, string stepRegex) + public void AddFailingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") { AddStepBinding(scenarioBlock, stepRegex, @"throw new System.Exception(""simulated failure"");", @"Throw New System.Exception(""simulated failure"")"); } diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index f76a3a41a..570b20e17 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -1,10 +1,27 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using FluentAssertions; +using Reqnroll.TestProjectGenerator.Driver; +using Reqnroll.TestProjectGenerator; +using System.Collections.Generic; +using System.IO; +using System.Linq; namespace Reqnroll.SystemTests.Generation; [TestCategory("Generation")] public class GenerationTestBase : SystemTestBase { + private HooksDriver _hookDriver = null!; + private TestProjectGenerator.Driver.ConfigurationDriver _configDriver = null!; + + protected override void TestInitialize() + { + base.TestInitialize(); + _hookDriver = _testContainer.Resolve(); + _configDriver = _testContainer.Resolve(); + } + [TestMethod] public void GeneratorAllIn_sample_can_be_handled() { @@ -30,18 +47,239 @@ When something happens with | who | when | | me | today | | someone else | tomorrow | + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + AddPassingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosPass(); + } + + //test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) + [TestMethod] + public void FailingScenariosAreCountedAsFailures() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + AddFailingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosFail(); + } + + [TestMethod] + public void PendingScenariosAreCountedAsPending() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + AddPendingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosPend(); + } + + [TestMethod] + public void IgnoredScenariosAreCountedAsIgnored() + { + AddScenario( + """ + @ignore + Scenario: Sample Scenario + When something happens + + @ignore + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + AddPassingStepBinding(); + _configDriver.SetIsRowTestsAllowed( false); //This is necessary as MSTest and Xunit count the number of physical Test methods. + ExecuteTests(); + + ShouldAllScenariosBeIgnored(3); + } + + [TestMethod] + public void UndefinedScenariosAreNotExecuted() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + //AddPassingStepBinding(); + + ExecuteTests(); + + ShouldAllUndefinedScenariosNotBeExecuted(); + } + + + //test async steps (async steps are executed in order) + [TestMethod] + public void AsyncStepsAreExecutedInOrder() + { + AddScenario( + """ + Scenario: Async Scenario Steps + Given a list to hold step numbers + When Async Step '1' is called + When Async Step '2' is called + When Async Step '3' is called + Then async step order should be '1,2,3' + """); + + var _asyncBindingClassContent = + @"namespace AsyncSequence.StepDefinitions +{ + [Binding] + public class AsyncsequenceStepDefinitions + { + private ScenarioContext _scenarioContext; + + public AsyncsequenceStepDefinitions(ScenarioContext scenarioContext) { + _scenarioContext = scenarioContext; + } + [Given(""a list to hold step numbers"")] + public async Task GivenAPlaceholder() + { + await Task.Run(() => global::Log.LogStep() ); + } + + [When(""Async Step {string} is called"")] + public async Task WhenStepIsTaken(string p0) + { + await Task.Run(() => global::Log.LogStep() ); + } + + [Then(""async step order should be {string}"")] + public async Task ThenStepSequenceIs( string p0) + { + await Task.Run( () => + { + global::Log.LogStep(); + } + ); + } + } +} +"; + AddBindingClass(_asyncBindingClassContent); + + ExecuteTests(); + CheckAreStepsExecutedInOrder(new string[] { "GivenAPlaceholder", "WhenStepIsTaken", "ThenStepSequenceIs" }); + + ShouldAllScenariosPass(); + } + //test hooks: before/after test run (require special handling by test frameworks) + //TODO: Consider adding a AddHookBinding method to SystemTestBase + [TestMethod] + public void BeforeAndAfterTestHooksRun() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | """); - _projectsDriver.AddPassingStepBinding(); + + + AddPassingStepBinding(); + + _projectsDriver.AddHookBinding("BeforeTestRun", "BeforeTestRun", null, null, null, "global::Log.LogHook();"); + _projectsDriver.AddHookBinding("AfterTestRun", "AfterTestRun", null, null, null, "global::Log.LogHook();"); ExecuteTests(); + _hookDriver?.CheckIsHookExecutedInOrder(new string[] { "BeforeTestRun", "AfterTestRun" }); ShouldAllScenariosPass(); + + } + + //test hooks: before/after test feature & scenario (require special handling by test frameworks) + [TestMethod] + public void BeforeAndAfterFeatureAndScenarioHooksRun() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + | me | + | you | + """); + + + AddPassingStepBinding(); + + _projectsDriver.AddHookBinding("BeforeFeature", "BeforeFeatureRun", null, null, null, "global::Log.LogHook();"); + _projectsDriver.AddHookBinding("AfterFeature", "AfterFeatureRun", null, null, null, "global::Log.LogHook();"); + _projectsDriver.AddHookBinding("BeforeScenario", "BeforeSenarioRun", null, null, null, "global::Log.LogHook();"); + _projectsDriver.AddHookBinding("AfterScenario", "AfterScenarioRun", null, null, null, "global::Log.LogHook();"); + + ExecuteTests(); + + _hookDriver.CheckIsHookExecutedInOrder(new string[] { "BeforeFeatureRun", "BeforeSenarioRun", "AfterScenarioRun", "BeforeSenarioRun", "AfterScenarioRun", "BeforeSenarioRun", "AfterScenarioRun", "AfterFeatureRun" }); + ShouldAllScenariosPass(); + } - //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) - //TODO: test async steps (async steps are executed in order) - //TODO: test hooks: before/after test run (require special handling by test frameworks) - //TODO: test hooks: before/after test feature & scenario (require special handling by test frameworks) //TODO: test scenario outlines (nr of examples, params are available in ScenarioContext, allowRowTests=false, examples tags) //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class + + } diff --git a/Tests/Reqnroll.SystemTests/SystemTestBase.cs b/Tests/Reqnroll.SystemTests/SystemTestBase.cs index 61dd12c7b..09d9b29e3 100644 --- a/Tests/Reqnroll.SystemTests/SystemTestBase.cs +++ b/Tests/Reqnroll.SystemTests/SystemTestBase.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Text.RegularExpressions; using FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -23,6 +26,7 @@ public abstract class SystemTestBase protected CompilationDriver _compilationDriver = null!; protected int _preparedTests = 0; + private TestProjectFolders _testProjectFolders = null!; public TestContext TestContext { get; set; } = null!; @@ -55,6 +59,8 @@ protected virtual void TestInitialize() _executionDriver = _testContainer.Resolve(); _vsTestExecutionDriver = _testContainer.Resolve(); _compilationDriver = _testContainer.Resolve(); + _testProjectFolders = _testContainer.Resolve(); + } protected void AddFeatureFileFromResource(string fileName, int? preparedTests = null) @@ -137,13 +143,89 @@ protected void ExecuteTests() protected void ShouldAllScenariosPass(int? expectedNrOfTestsSpec = null) { - if (expectedNrOfTestsSpec == null && _preparedTests == 0) + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass"); + _folderCleaner.CleanSolutionFolder(); + } + + protected void ShouldAllScenariosFail(int? expectedNrOfTestsSpec = null) + { + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Failed.Should().Be(expectedNrOfTests, "all tests should fail"); + _folderCleaner.CleanSolutionFolder(); + + } + + protected void ShouldAllScenariosPend(int? expectedNrOfTestsSpec = null) + { + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Pending.Should().Be(expectedNrOfTests, "all tests should Pend"); + _folderCleaner.CleanSolutionFolder(); + } + + protected void ShouldAllScenariosBeIgnored(int? expectedNrOfTestsSpec = null) + { + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Ignored.Should().Be(expectedNrOfTests, "all tests should be Ignored"); + _folderCleaner.CleanSolutionFolder(); + } + + protected void ShouldAllUndefinedScenariosNotBeExecuted(int? expectedNrOfTestsSpec = null) + { + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(0, "none of the tests should pass"); + _vsTestExecutionDriver.LastTestExecutionResult.Failed.Should().Be(0, "none of the tests should fail"); + + // MSTest treats tests with undefined steps as Ignored, the other two frameworks treat them as Pending + int pendingExpected = _testRunConfiguration.UnitTestProvider == UnitTestProvider.MSTest ? 0 : expectedNrOfTests; + _vsTestExecutionDriver.LastTestExecutionResult.Pending.Should().Be(pendingExpected, "All of the tests should Pend"); + + int IgnoredExpected = _testRunConfiguration.UnitTestProvider == UnitTestProvider.MSTest ? expectedNrOfTests : 0; + _vsTestExecutionDriver.LastTestExecutionResult.Ignored.Should().Be(IgnoredExpected, "None of the tests should be Ignored"); + + + _folderCleaner.CleanSolutionFolder(); + } + + protected int ConfirmAllTestsRan(int? expectedNrOfTestsSpec) + { + if (expectedNrOfTestsSpec == null && _preparedTests == 0) throw new ArgumentException($"If {nameof(_preparedTests)} is not set, the {nameof(expectedNrOfTestsSpec)} is mandatory.", nameof(expectedNrOfTestsSpec)); var expectedNrOfTests = expectedNrOfTestsSpec ?? _preparedTests; _vsTestExecutionDriver.LastTestExecutionResult.Should().NotBeNull(); _vsTestExecutionDriver.LastTestExecutionResult.Total.Should().Be(expectedNrOfTests, $"the run should contain {expectedNrOfTests} tests"); - _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass"); - _folderCleaner.CleanSolutionFolder(); + return expectedNrOfTests; + } + + protected void AddPassingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") + { + _projectsDriver.AddPassingStepBinding(scenarioBlock, stepRegex); + } + + protected void AddFailingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") + { + _projectsDriver.AddFailingStepBinding(scenarioBlock, stepRegex); + } + + protected void AddPendingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") + { + _projectsDriver.AddStepBinding(scenarioBlock, stepRegex, "throw new PendingStepException();", "ScenarioContext.Current.Pending()"); + } + + protected void AddBindingClass(string content) + { + _projectsDriver.AddBindingClass(content); + } + + //TODO: Consider moving this to the TestProjectGenerator as a driver method + public void CheckAreStepsExecutedInOrder(IEnumerable methodNames) + { + _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); + + var pathToLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); + var lines = File.ReadAllLines(pathToLogFile); + var methodNameLines = methodNames.Select(m => $"-> step: {m}"); + lines.Should().ContainInOrder(methodNameLines); } }