From 77bea548459bb87b3b7c32082b41022f5dee5a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1sp=C3=A1r=20Nagy?= Date: Wed, 17 Apr 2024 07:33:15 +0200 Subject: [PATCH] Fleshing out Generation System Tests (2) (#99) * Commit to flesh out Generation System Test. * Added test to verify that async step bindings are executed in the order specified in the Scenario. * Tests that verify that Before/After Hooks are run. * cleanup; promoted some code from GenerationTestBAse to SystemTestBase. * Adjusted the expectations of the Undefined Step test to account for different handling by MSTest. Adjusted Ignored test to disable row tests; this provides for consistent handling by all three test frameworks. * fix build * code cleanup * refactor common scenarios * Improve hook / step assertions * Merge outcome related tests * Fixes Adding @ignore to an Examples block generates invalid code for NUnit v3+ issue (#103) * Add fix to CHANGELOG * cleanup * Test scenario outlines (nr of examples, params are available in ScenarioContext, examples tags) * Add portability tests for before/after test run hooks (.NET Framework version of Reqnroll is subscribed to assembly unload) --------- Co-authored-by: Chris Rudolphi <1702962+clrudolphi@users.noreply.github.com> --- CHANGELOG.md | 1 + .../NUnit3TestGeneratorProvider.cs | 2 +- .../AppConfigGeneratorTests.cs | 1 - .../FeatureFileGeneratorTests.cs | 1 - .../JsonConfigGeneratorTests.cs | 1 - .../ConfigurationModel/Configuration.cs | 1 - .../Data/AppConfigSection.cs | 2 +- .../Data/ProjectFile.cs | 3 +- .../Data/SolutionFile.cs | 28 +- .../{HooksDriver.cs => BindingsDriver.cs} | 78 ++++-- ...onDriver.cs => ConfigurationFileDriver.cs} | 4 +- .../Driver/JsonConfigurationLoaderDriver.cs | 8 +- .../Driver/ProjectsDriver.cs | 32 ++- .../Driver/TestSuiteSetupDriver.cs | 8 +- .../BaseBindingsGenerator.cs | 2 +- .../CSharp10BindingsGenerator.cs | 5 + .../CSharpBindingsGenerator.cs | 5 + .../AppConfigGenerator.cs | 1 - .../Factories/ProjectBuilderFactory.cs | 1 - .../FeatureFileGenerator.cs | 18 +- .../FilesystemWriter/FileWriter.cs | 2 + .../ProjectBuilder.cs | 14 +- .../TRXParser.cs | 2 + .../TestExecutionResult.cs | 2 +- .../TestProjectFolders.cs | 1 + .../VSTestExecutionDriver.cs | 5 +- ...8b0de49-90e4-4313-9c7d-e1fe55c4c33d.csproj | 125 +++++++++ .../Drivers/ConfigurationLoaderDriver.cs | 12 +- .../StepDefinitions/ConfigurationSteps.cs | 8 +- .../StepDefinitions/ExecutionResultSteps.cs | 10 +- .../ReqnrollConfigurationSteps.cs | 19 +- .../Generation/GenerationTestBase.cs | 262 +++++++++++++++++- .../Generation/NUnitGenerationTest.cs | 8 + .../Generation/XUnitGenerationTest.cs | 3 + .../Portability/PortabilityTestBase.cs | 21 +- Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs | 6 +- Tests/Reqnroll.SystemTests/SystemTestBase.cs | 79 +++++- 37 files changed, 644 insertions(+), 137 deletions(-) rename Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/{HooksDriver.cs => BindingsDriver.cs} (57%) rename Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/{ConfigurationDriver.cs => ConfigurationFileDriver.cs} (97%) create mode 100644 Tests/Reqnroll.GeneratorTests/nCrunchTemp_38b0de49-90e4-4313-9c7d-e1fe55c4c33d.csproj diff --git a/CHANGELOG.md b/CHANGELOG.md index 11ae4b901..0ada8ba44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Support for PriorityAttribute in MsTest adapter * Support for Scenario Outline / DataRowAttribute in MsTest adapter * Fix for #81 in which Cucumber Expressions fail when two enums with the same short name (differing namespaces) are used as parameters +* Fix: Adding @ignore to an Examples block generates invalid code for NUnit v3+ (#103) # v1.0.1 - 2024-02-16 diff --git a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs index 898bb8035..66fc9568f 100644 --- a/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs +++ b/Reqnroll.Generator/UnitTestProvider/NUnit3TestGeneratorProvider.cs @@ -140,7 +140,7 @@ public void SetRow(TestClassGenerationContext generationContext, CodeMemberMetho } if (isIgnored) - args.Add(new CodeAttributeArgument("Ignored", new CodePrimitiveExpression(true))); + args.Add(new CodeAttributeArgument("IgnoreReason", new CodePrimitiveExpression("Ignored by @ignore tag"))); CodeDomHelper.AddAttribute(testMethod, ROW_ATTR, args.ToArray()); } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/AppConfigGeneratorTests.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/AppConfigGeneratorTests.cs index 8d7afcc27..730e35230 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/AppConfigGeneratorTests.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/AppConfigGeneratorTests.cs @@ -2,7 +2,6 @@ using Reqnroll.TestProjectGenerator.ConfigurationModel; using Reqnroll.TestProjectGenerator.Data; using Reqnroll.TestProjectGenerator.Factories.ConfigurationGenerator; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; using Xunit; namespace Reqnroll.TestProjectGenerator.Tests diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/FeatureFileGeneratorTests.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/FeatureFileGeneratorTests.cs index 4692c1fac..f4a23adf1 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/FeatureFileGeneratorTests.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/FeatureFileGeneratorTests.cs @@ -1,5 +1,4 @@ using FluentAssertions; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; using Xunit; namespace Reqnroll.TestProjectGenerator.Tests diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/JsonConfigGeneratorTests.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/JsonConfigGeneratorTests.cs index 5137bb094..57d2d6239 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/JsonConfigGeneratorTests.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator.Tests/JsonConfigGeneratorTests.cs @@ -2,7 +2,6 @@ using Reqnroll.TestProjectGenerator.ConfigurationModel; using Reqnroll.TestProjectGenerator.Data; using Reqnroll.TestProjectGenerator.Factories.ConfigurationGenerator; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; using Xunit; namespace Reqnroll.TestProjectGenerator.Tests diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationModel/Configuration.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationModel/Configuration.cs index 9e3414abd..ff316838f 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationModel/Configuration.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ConfigurationModel/Configuration.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Globalization; using Reqnroll.TestProjectGenerator.Data; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; namespace Reqnroll.TestProjectGenerator.ConfigurationModel { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/AppConfigSection.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/AppConfigSection.cs index b1f01a9ff..0c53ec045 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/AppConfigSection.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/AppConfigSection.cs @@ -1,4 +1,4 @@ -namespace Reqnroll.TestProjectGenerator.NewApi._1_Memory +namespace Reqnroll.TestProjectGenerator.Data { public class AppConfigSection { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/ProjectFile.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/ProjectFile.cs index 312a06e7b..a8d37693a 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/ProjectFile.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/ProjectFile.cs @@ -4,12 +4,11 @@ namespace Reqnroll.TestProjectGenerator.Data { [DebuggerDisplay("{" + nameof(Path) + ("} [{" + nameof(BuildAction) + "}]"))] - public class ProjectFile :SolutionFile //FeatureFiles, Code, App.Config, NuGet.Config, packages.config, + public class ProjectFile: SolutionFile //FeatureFiles, Code, App.Config, NuGet.Config, packages.config, { public ProjectFile(string path, string buildAction, string content, CopyToOutputDirectory copyToOutputDirectory = CopyToOutputDirectory.DoNotCopy) : this(path, buildAction, content, copyToOutputDirectory, new Dictionary()) { - } public ProjectFile(string path, string buildAction, string content, CopyToOutputDirectory copyToOutputDirectory, IReadOnlyDictionary additionalMsBuildProperties) : base(path, content) diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/SolutionFile.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/SolutionFile.cs index 3be064212..e5bf1615a 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/SolutionFile.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Data/SolutionFile.cs @@ -1,14 +1,30 @@ +using System; + namespace Reqnroll.TestProjectGenerator.Data { - public class SolutionFile + public class SolutionFile(string path, string content) { - public SolutionFile(string path, string content) + private bool _isFrozen = false; + + public string Path { get; } = path; //relative from project + public string Content { get; private set; } = content; + + internal void Freeze() { - Path = path; - Content = content; + _isFrozen = true; } - public string Path { get; } //relative from project - public string Content { get; } + public void Append(string addedContent) + { + if (_isFrozen) + { + throw new InvalidOperationException("Cannot append to frozen file"); + } + + if (!Content.EndsWith(Environment.NewLine)) + Content += Environment.NewLine; + + Content += addedContent; + } } } \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/HooksDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/BindingsDriver.cs similarity index 57% rename from Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/HooksDriver.cs rename to Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/BindingsDriver.cs index 749b3df18..fb9bb5ad0 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/HooksDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/BindingsDriver.cs @@ -8,15 +8,8 @@ namespace Reqnroll.TestProjectGenerator.Driver { - public class HooksDriver + public class BindingsDriver(TestProjectFolders _testProjectFolders) { - private readonly TestProjectFolders _testProjectFolders; - - public HooksDriver(TestProjectFolders testProjectFolders) - { - _testProjectFolders = testProjectFolders; - } - public void CheckIsHookExecuted(string methodName, int expectedTimesExecuted) { int hookExecutionCount = GetHookExecutionCount(methodName); @@ -27,13 +20,12 @@ public int GetHookExecutionCount(string methodName) { _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); - string pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); - if (!File.Exists(pathToHookLogFile)) + if (!File.Exists(_testProjectFolders.LogFilePath)) { return 0; } - string content = File.ReadAllText(pathToHookLogFile); + string content = File.ReadAllText(_testProjectFolders.LogFilePath); content.Should().NotBeNull(); var regex = new Regex($@"-> hook: {methodName}"); @@ -41,13 +33,15 @@ public int GetHookExecutionCount(string methodName) return regex.Matches(content).Count; } + private string GetLogFileLockPath() => _testProjectFolders.LogFilePath + ".lock"; + public void AcquireHookLock() { _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); - var pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log.lock"); + var pathToHookLogFile = GetLogFileLockPath(); - Directory.CreateDirectory(Path.GetDirectoryName(pathToHookLogFile)); + Directory.CreateDirectory(Path.GetDirectoryName(pathToHookLogFile)!); using (File.Open(pathToHookLogFile, FileMode.CreateNew)) { } @@ -56,10 +50,7 @@ public void AcquireHookLock() public void ReleaseHookLock() { _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); - - var pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log.lock"); - - File.Delete(pathToHookLogFile); + File.Delete(GetLogFileLockPath()); } public async Task WaitForIsWaitingForHookLockAsync(string methodName) @@ -81,14 +72,12 @@ private bool CheckIsWaitingForHookLock(string methodName) { _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); - var pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); - - if (!File.Exists(pathToHookLogFile)) + if (!File.Exists(_testProjectFolders.LogFilePath)) { return false; } - var content = File.ReadAllText(pathToHookLogFile); + var content = File.ReadAllText(_testProjectFolders.LogFilePath); content.Should().NotBeNull(); var regex = new Regex($@"-> waiting for hook lock: {methodName}"); @@ -96,12 +85,19 @@ private bool CheckIsWaitingForHookLock(string methodName) return regex.Matches(content).Count == 1; } + public IEnumerable GetActualLogLines(string category) + { + _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); + + var lines = File.ReadAllLines(_testProjectFolders.LogFilePath); + return lines.Where(l => l.StartsWith($"-> {category}:")); + } + public void CheckIsNotHookExecuted(string methodName, int timesExecuted) { _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); - var pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); - var content = File.ReadAllText(pathToHookLogFile); + var content = File.ReadAllText(_testProjectFolders.LogFilePath); content.Should().NotBeNull(); var regex = new Regex($@"-> hook: {methodName}"); @@ -109,14 +105,40 @@ public void CheckIsNotHookExecuted(string methodName, int timesExecuted) regex.Matches(content).Count.Should().NotBe(timesExecuted); } - public void CheckIsHookExecutedInOrder(IEnumerable methodNames) + private IEnumerable GetActualHookLines() => GetActualLogLines("hook"); + + public void AssertHooksExecutedInOrder(params string[] methodNames) { - _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace(); + var hookLines = GetActualHookLines(); - var pathToHookLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); - var lines = File.ReadAllLines(pathToHookLogFile); var methodNameLines = methodNames.Select(m => $"-> hook: {m}"); - lines.Should().ContainInOrder(methodNameLines); + hookLines.Should().ContainInOrder(methodNameLines); + } + + public void AssertExecutedHooksEqual(params string[] methodNames) + { + var hookLines = GetActualHookLines(); + + var methodNameLines = methodNames.Select(m => $"-> hook: {m}"); + hookLines.Should().Equal(methodNameLines); + } + + private IEnumerable GetActualStepLines() => GetActualLogLines("step"); + + public void AssertStepsExecutedInOrder(params string[] methodNames) + { + var stepLines = GetActualStepLines(); + + var methodNameLines = methodNames.Select(m => $"-> step: {m}"); + stepLines.Should().ContainInOrder(methodNameLines); + } + + public void AssertExecutedStepsEqual(params string[] methodNames) + { + var stepLines = GetActualStepLines(); + + var methodNameLines = methodNames.Select(m => $"-> step: {m}"); + stepLines.Should().Equal(methodNameLines); } } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationFileDriver.cs similarity index 97% rename from Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationDriver.cs rename to Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationFileDriver.cs index 74115448d..9272f1ccc 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ConfigurationFileDriver.cs @@ -4,11 +4,11 @@ namespace Reqnroll.TestProjectGenerator.Driver { - public class ConfigurationDriver + public class ConfigurationFileDriver { private readonly SolutionDriver _solutionDriver; - public ConfigurationDriver(SolutionDriver solutionDriver) + public ConfigurationFileDriver(SolutionDriver solutionDriver) { _solutionDriver = solutionDriver; } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/JsonConfigurationLoaderDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/JsonConfigurationLoaderDriver.cs index 5b97b7b85..f8b6799c6 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/JsonConfigurationLoaderDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/JsonConfigurationLoaderDriver.cs @@ -5,17 +5,17 @@ namespace Reqnroll.TestProjectGenerator.Driver public class JsonConfigurationLoaderDriver { private readonly ProjectsDriver _projectsDriver; - private readonly ConfigurationDriver _configurationDriver; + private readonly ConfigurationFileDriver _configurationFileDriver; - public JsonConfigurationLoaderDriver(ProjectsDriver projectsDriver, ConfigurationDriver configurationDriver) + public JsonConfigurationLoaderDriver(ProjectsDriver projectsDriver, ConfigurationFileDriver configurationFileDriver) { _projectsDriver = projectsDriver; - _configurationDriver = configurationDriver; + _configurationFileDriver = configurationFileDriver; } public void AddReqnrollJson(string reqnrollJson) { - _configurationDriver.SetConfigurationFormat(ConfigurationFormat.None); + _configurationFileDriver.SetConfigurationFormat(ConfigurationFormat.None); _projectsDriver.AddFile("reqnroll.json", reqnrollJson); } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs index 07f1680eb..9eee89bd3 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs @@ -12,6 +12,7 @@ public class ProjectsDriver private readonly SolutionDriver _solutionDriver; private readonly ProjectBuilderFactory _projectBuilderFactory; private readonly TestProjectFolders _testProjectFolders; + public ProjectFile LastFeatureFile { get; private set; } public ProjectsDriver(SolutionDriver solutionDriver, ProjectBuilderFactory projectBuilderFactory, TestProjectFolders testProjectFolders) { @@ -59,9 +60,9 @@ public void AddHookBinding(string eventType, string name, string hookTypeAttribu AddHookBinding(_solutionDriver.DefaultProject, eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); } - public void AddHookBinding(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) + public void AddHookBinding(string eventType, string name = null, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) { - AddHookBinding(_solutionDriver.DefaultProject, eventType, name, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); + AddHookBinding(_solutionDriver.DefaultProject, eventType, name ?? eventType, code, order, hookTypeAttributeTags, methodScopeAttributeTags, classScopeAttributeTags); } private void AddHookBinding(ProjectBuilder project, string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, IList classScopeAttributeTags = null) @@ -81,27 +82,34 @@ private void AddAsyncHookBindingIncludingLocking(ProjectBuilder project, string public void AddFeatureFile(string featureFileContent) { - _solutionDriver.DefaultProject.AddFeatureFile(featureFileContent); + LastFeatureFile = _solutionDriver.DefaultProject.AddFeatureFile(featureFileContent); } public void AddScenario(string scenarioContent) { - AddFeatureFile( - $$""" - Feature: Sample Feature - - {{scenarioContent}} - """); + if (LastFeatureFile != null) + { + LastFeatureFile.Append(scenarioContent); + } + else + { + AddFeatureFile( + $$""" + Feature: Sample Feature + + {{scenarioContent}} + """); + } } - public void AddStepBinding(string attributeName, string regex, string csharpcode, string vbnetcode) + public void AddStepBinding(string attributeName, string regex, string csharpcode, string vbnetcode = null) { _solutionDriver.DefaultProject.AddStepBinding(attributeName, regex, csharpcode, vbnetcode); } public void AddLoggingStepBinding(string attributeName, string methodName, string regex) { - _solutionDriver.DefaultProject.AddLoggingStepBinding(attributeName, methodName, Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"), regex); + _solutionDriver.DefaultProject.AddLoggingStepBinding(attributeName, methodName, regex); } public void AddStepBinding(string projectName, string bindingCode) => AddStepBinding(_solutionDriver.Projects[projectName], bindingCode); @@ -157,7 +165,7 @@ public void AddNuGetPackage(string nugetPackage, string nugetVersion) _solutionDriver.DefaultProject.AddNuGetPackage(nugetPackage, nugetVersion); } - public void AddFailingStepBinding(string scenarioBlock, string stepRegex) + public void AddFailingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") { AddStepBinding(scenarioBlock, stepRegex, @"throw new System.Exception(""simulated failure"");", @"Throw New System.Exception(""simulated failure"")"); } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/TestSuiteSetupDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/TestSuiteSetupDriver.cs index 63f60f2f8..6ea0185af 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/TestSuiteSetupDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/TestSuiteSetupDriver.cs @@ -9,16 +9,16 @@ public class TestSuiteSetupDriver private readonly ProjectsDriver _projectsDriver; private readonly TestSuiteInitializationDriver _testSuiteInitializationDriver; private readonly JsonConfigurationLoaderDriver _jsonConfigurationLoaderDriver; - private readonly ConfigurationDriver _configurationDriver; + private readonly ConfigurationFileDriver _configurationFileDriver; private bool _isProjectCreated; public TestSuiteSetupDriver(ProjectsDriver projectsDriver, TestSuiteInitializationDriver testSuiteInitializationDriver, JsonConfigurationLoaderDriver jsonConfigurationLoaderDriver, - ConfigurationDriver configurationDriver) + ConfigurationFileDriver configurationFileDriver) { _projectsDriver = projectsDriver; _testSuiteInitializationDriver = testSuiteInitializationDriver; _jsonConfigurationLoaderDriver = jsonConfigurationLoaderDriver; - _configurationDriver = configurationDriver; + _configurationFileDriver = configurationFileDriver; } public void AddGenericWhenStepBinding() @@ -136,7 +136,7 @@ public void AddScenarioWithGivenStep(string step, string tags = "") public void AddAppConfigFromString(string appConfigContent) { - _configurationDriver.SetConfigurationFormat(ConfigurationFormat.None); + _configurationFileDriver.SetConfigurationFormat(ConfigurationFormat.None); _projectsDriver.AddFile("app.config", appConfigContent); } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs index 646145720..d799d3426 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/BaseBindingsGenerator.cs @@ -16,7 +16,7 @@ public ProjectFile GenerateStepDefinition(string methodName, string methodImplem return GenerateStepDefinition(method); } - public ProjectFile GenerateLoggingStepDefinition(string methodName, string pathToLogFile, string attributeName, string regex, ParameterType parameterType = ParameterType.Normal, string argumentName = null) + public ProjectFile GenerateLoggingStepDefinition(string methodName, string attributeName, string regex, ParameterType parameterType = ParameterType.Normal, string argumentName = null) { string method = GetLoggingStepDefinitionCode(methodName, attributeName, regex, parameterType, argumentName); return GenerateStepDefinition(method); diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs index f8cfd23dd..2f7809437 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharp10BindingsGenerator.cs @@ -44,6 +44,11 @@ internal static void LogHook([CallerMemberName] string stepName = null!) Retry(5, () => WriteToFile($@"-> hook: {stepName}{Environment.NewLine}")); } + internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) + { + Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); + } + static void WriteToFile(string line) { using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs index fd0a17861..40c0229a6 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/BindingsGenerator/CSharpBindingsGenerator.cs @@ -109,6 +109,11 @@ internal static async Task LogHookIncludingLockingAsync([CallerMemberName] strin WriteToFile($@"-> hook: {stepName}{Environment.NewLine}"); } + internal static void LogCustom(string category, string value, [CallerMemberName] string memberName = null) + { + Retry(5, () => WriteToFile($@"-> {category}: {value}:{memberName}{Environment.NewLine}")); + } + static void WriteToFile(string line) { using (FileStream fs = File.Open(LogFileLocation, FileMode.Append, FileAccess.Write, FileShare.None)) diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ConfigurationGenerator/AppConfigGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ConfigurationGenerator/AppConfigGenerator.cs index f7a0329eb..1e790efba 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ConfigurationGenerator/AppConfigGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ConfigurationGenerator/AppConfigGenerator.cs @@ -9,7 +9,6 @@ using Reqnroll.TestProjectGenerator.Data; using Reqnroll.TestProjectGenerator.Extensions; using Reqnroll.TestProjectGenerator.Helpers; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; namespace Reqnroll.TestProjectGenerator.Factories.ConfigurationGenerator { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs index 54437d9ec..500ac8098 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Factories/ProjectBuilderFactory.cs @@ -3,7 +3,6 @@ using Reqnroll.TestProjectGenerator.Factories.BindingsGenerator; using Reqnroll.TestProjectGenerator.Factories.ConfigurationGenerator; using Reqnroll.TestProjectGenerator.Helpers; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; namespace Reqnroll.TestProjectGenerator.Factories { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs index 8e17c551f..984f99e52 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FeatureFileGenerator.cs @@ -1,17 +1,15 @@ using System; using Reqnroll.TestProjectGenerator.Data; -namespace Reqnroll.TestProjectGenerator.NewApi._1_Memory +namespace Reqnroll.TestProjectGenerator; + +public class FeatureFileGenerator { - public class FeatureFileGenerator + public ProjectFile Generate(string featureFileContent, string featureFileName = null) { - public ProjectFile Generate(string featureFileContent, string featureFileName = null) - { - featureFileName = featureFileName ?? $"FeatureFile{Guid.NewGuid():N}.feature"; - + featureFileName = featureFileName ?? $"FeatureFile{Guid.NewGuid():N}.feature"; - string fileContent = featureFileContent.Replace("'''", "\"\"\""); - return new ProjectFile(featureFileName, "None", fileContent); - } + string fileContent = featureFileContent.Replace("'''", "\"\"\""); + return new ProjectFile(featureFileName, "None", fileContent); } -} +} \ No newline at end of file diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/FileWriter.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/FileWriter.cs index b2ea4c261..bcdab71ec 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/FileWriter.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/FilesystemWriter/FileWriter.cs @@ -26,6 +26,8 @@ public void Write(SolutionFile projectFile, string projectRootPath) { File.WriteAllText(absolutePath, projectFile.Content); } + + projectFile.Freeze(); } } } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs index 48c1e3ea7..ef8329902 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/ProjectBuilder.cs @@ -6,7 +6,6 @@ using Reqnroll.TestProjectGenerator.Driver; using Reqnroll.TestProjectGenerator.Factories.BindingsGenerator; using Reqnroll.TestProjectGenerator.Factories.ConfigurationGenerator; -using Reqnroll.TestProjectGenerator.NewApi._1_Memory; namespace Reqnroll.TestProjectGenerator { @@ -79,13 +78,14 @@ public void AddFile(ProjectFile projectFile) _project.AddFile(projectFile ?? throw new ArgumentNullException(nameof(projectFile))); } - public void AddFeatureFile(string featureFileContent) + public ProjectFile AddFeatureFile(string featureFileContent) { EnsureProjectExists(); var featureFile = _featureFileGenerator.Generate(featureFileContent); _project.AddFile(featureFile); + return featureFile; } public void AddStepBinding(string attributeName, string regex, string csharpcode, string vbnetcode) @@ -100,15 +100,15 @@ public void AddStepBinding(string attributeName, string regex, string csharpcode _project.AddFile(bindingsGenerator.GenerateStepDefinition("StepBinding", methodImplementation, attributeName, regex, ParameterType.DocString, "docStringArg")); } - public void AddLoggingStepBinding(string attributeName, string methodName, string pathToLogFile, string regex) + public void AddLoggingStepBinding(string attributeName, string methodName, string regex) { EnsureProjectExists(); var bindingsGenerator = _bindingsGeneratorFactory.FromLanguage(_project.ProgrammingLanguage); - _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, pathToLogFile, attributeName, regex)); - _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, pathToLogFile, attributeName, regex, ParameterType.Table, "tableArg")); - _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, pathToLogFile, attributeName, regex, ParameterType.DocString, "docStringArg")); + _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, attributeName, regex)); + _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, attributeName, regex, ParameterType.Table, "tableArg")); + _project.AddFile(bindingsGenerator.GenerateLoggingStepDefinition(methodName, attributeName, regex, ParameterType.DocString, "docStringArg")); } public void AddHookBinding(string eventType, string name, string code = "", int? order = null, IList hookTypeAttributeTags = null, IList methodScopeAttributeTags = null, @@ -245,7 +245,7 @@ private void EnsureProjectExists() _project.AddNuGetPackage("Microsoft.Bcl.AsyncInterfaces", "6.0.0", new NuGetPackageAssembly("Microsoft.Bcl.AsyncInterfaces", "netstandard2.0\\Microsoft.Bcl.AsyncInterfaces.dll")); var generator = _bindingsGeneratorFactory.FromLanguage(_project.ProgrammingLanguage); - _project.AddFile(generator.GenerateLoggerClass(Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"))); + _project.AddFile(generator.GenerateLoggerClass(_testProjectFolders.LogFilePath)); switch (_project.ProgrammingLanguage) { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TRXParser.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TRXParser.cs index d4f19e435..44a7648f1 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TRXParser.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TRXParser.cs @@ -155,6 +155,7 @@ private List GetTestResultsInternal(XElement testRunResultsElement, var testResults = from unitTestResultElement in testRunResultsElement?.Elements(_unitTestResultElementName) ?? Enumerable.Empty() let outputElement = unitTestResultElement.Element(_unitTestResultOutputElementName) let idAttribute = unitTestResultElement.Attribute("executionId") + let testNameAttribute = unitTestResultElement.Attribute("testName") let outcomeAttribute = unitTestResultElement.Attribute("outcome") let stdOutElement = outputElement?.Element(_unitTestResultStdOutElementName) let errorInfoElement = outputElement?.Element(xmlns + "ErrorInfo") @@ -165,6 +166,7 @@ private List GetTestResultsInternal(XElement testRunResultsElement, select new TestResult { Id = idAttribute.Value, + TestName = testNameAttribute.Value, Outcome = outcomeAttribute.Value, StdOut = stdOutElement?.Value, ErrorMessage = errorMessage?.Value, diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestExecutionResult.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestExecutionResult.cs index 4385aa41c..e8e8522f0 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestExecutionResult.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestExecutionResult.cs @@ -32,7 +32,7 @@ public class TestResult public string Id { get; set; } public string Outcome { get; set; } public string StdOut { get; set; } - public string Title { get; set; } + public string TestName { get; set; } public string Feature { get; set; } public int ScheduleOrder { get; set; } public List Steps { get; set; } diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestProjectFolders.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestProjectFolders.cs index 61f07fae6..addc8d75f 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestProjectFolders.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/TestProjectFolders.cs @@ -12,6 +12,7 @@ public class TestProjectFolders public string CompiledAssemblyPath { get; set; } public string PathToSolutionDirectory => Path.GetDirectoryName(PathToSolutionFile); public string TestAssemblyFileName { get; set; } + public string LogFilePath => Path.Combine(PathToSolutionDirectory, "steps.log"); public string PathToSolutionFile { diff --git a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/VSTestExecutionDriver.cs b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/VSTestExecutionDriver.cs index f53a95208..c1edcc62b 100644 --- a/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/VSTestExecutionDriver.cs +++ b/Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/VSTestExecutionDriver.cs @@ -53,10 +53,9 @@ public VSTestExecutionDriver( public void CheckIsBindingMethodExecuted(string methodName, int timesExecuted) { - string pathToLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log"); - string logFileContent = File.ReadAllText(pathToLogFile, Encoding.UTF8); + string logFileContent = File.ReadAllText(_testProjectFolders.LogFilePath, Encoding.UTF8); - var regex = new Regex($@"-> step: {methodName}"); + var regex = new Regex($"-> step: {methodName}"); regex.Match(logFileContent).Success.Should().BeTrue($"method {methodName} was not executed."); regex.Matches(logFileContent).Count.Should().Be(timesExecuted); diff --git a/Tests/Reqnroll.GeneratorTests/nCrunchTemp_38b0de49-90e4-4313-9c7d-e1fe55c4c33d.csproj b/Tests/Reqnroll.GeneratorTests/nCrunchTemp_38b0de49-90e4-4313-9c7d-e1fe55c4c33d.csproj new file mode 100644 index 000000000..5018cc575 --- /dev/null +++ b/Tests/Reqnroll.GeneratorTests/nCrunchTemp_38b0de49-90e4-4313-9c7d-e1fe55c4c33d.csproj @@ -0,0 +1,125 @@ + + + $(Reqnroll_Test_TFM) + Reqnroll.GeneratorTests + $(Reqnroll_KeyFile) + $(Reqnroll_EnableStrongNameSigning) + $(Reqnroll_PublicSign) + Reqnroll.GeneratorTests + true + false + + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tests/Reqnroll.Specs/Drivers/ConfigurationLoaderDriver.cs b/Tests/Reqnroll.Specs/Drivers/ConfigurationLoaderDriver.cs index 94824f804..9b6033f5a 100644 --- a/Tests/Reqnroll.Specs/Drivers/ConfigurationLoaderDriver.cs +++ b/Tests/Reqnroll.Specs/Drivers/ConfigurationLoaderDriver.cs @@ -6,12 +6,12 @@ namespace Reqnroll.Specs.Drivers { public class ConfigurationLoaderDriver { - private readonly ConfigurationDriver _configurationDriver; + private readonly ConfigurationFileDriver _configurationFileDriver; private readonly SolutionDriver _solutionDriver; - public ConfigurationLoaderDriver(ConfigurationDriver configurationDriver, SolutionDriver solutionDriver) + public ConfigurationLoaderDriver(ConfigurationFileDriver configurationFileDriver, SolutionDriver solutionDriver) { - _configurationDriver = configurationDriver; + _configurationFileDriver = configurationFileDriver; _solutionDriver = solutionDriver; } @@ -21,11 +21,11 @@ public void SetFromReqnrollConfiguration(ReqnrollConfiguration reqnrollConfigura foreach (string stepAssemblyName in reqnrollConfiguration.AdditionalStepAssemblies) { - _configurationDriver.AddStepAssembly(project, new BindingAssembly(stepAssemblyName)); + _configurationFileDriver.AddStepAssembly(project, new BindingAssembly(stepAssemblyName)); } - _configurationDriver.SetBindingCulture(project, reqnrollConfiguration.BindingCulture); - _configurationDriver.SetFeatureLanguage(project, reqnrollConfiguration.FeatureLanguage); + _configurationFileDriver.SetBindingCulture(project, reqnrollConfiguration.BindingCulture); + _configurationFileDriver.SetFeatureLanguage(project, reqnrollConfiguration.FeatureLanguage); } } } diff --git a/Tests/Reqnroll.Specs/StepDefinitions/ConfigurationSteps.cs b/Tests/Reqnroll.Specs/StepDefinitions/ConfigurationSteps.cs index 8c133a16c..921b41ee8 100644 --- a/Tests/Reqnroll.Specs/StepDefinitions/ConfigurationSteps.cs +++ b/Tests/Reqnroll.Specs/StepDefinitions/ConfigurationSteps.cs @@ -5,12 +5,12 @@ namespace Reqnroll.Specs.StepDefinitions [Binding] public class ConfigurationSteps { - private readonly ConfigurationDriver _configurationDriver; + private readonly ConfigurationFileDriver _configurationFileDriver; private readonly CompilationResultDriver _compilationResultDriver; - public ConfigurationSteps(ConfigurationDriver configurationDriver, CompilationResultDriver compilationResultDriver) + public ConfigurationSteps(ConfigurationFileDriver configurationFileDriver, CompilationResultDriver compilationResultDriver) { - _configurationDriver = configurationDriver; + _configurationFileDriver = configurationFileDriver; _compilationResultDriver = compilationResultDriver; } @@ -29,7 +29,7 @@ public void ThenTheReqnroll_JsonIsUsedForConfiguration() [Given(@"the feature language is '(.*)'")] public void GivenTheFeatureLanguageIs(string featureLanguage) { - _configurationDriver.SetFeatureLanguage(featureLanguage); + _configurationFileDriver.SetFeatureLanguage(featureLanguage); } } } diff --git a/Tests/Reqnroll.Specs/StepDefinitions/ExecutionResultSteps.cs b/Tests/Reqnroll.Specs/StepDefinitions/ExecutionResultSteps.cs index b55537bda..9c0ee1b43 100644 --- a/Tests/Reqnroll.Specs/StepDefinitions/ExecutionResultSteps.cs +++ b/Tests/Reqnroll.Specs/StepDefinitions/ExecutionResultSteps.cs @@ -13,14 +13,14 @@ namespace Reqnroll.Specs.StepDefinitions [Binding] public class ExecutionResultSteps { - private readonly HooksDriver _hooksDriver; + private readonly BindingsDriver _bindingsDriver; private readonly VSTestExecutionDriver _vsTestExecutionDriver; private readonly TestProjectFolders _testProjectFolders; private readonly TestRunLogDriver _testRunLogDriver; - public ExecutionResultSteps(HooksDriver hooksDriver, VSTestExecutionDriver vsTestExecutionDriver, TestProjectFolders testProjectFolders, TestRunLogDriver testRunLogDriver) + public ExecutionResultSteps(BindingsDriver bindingsDriver, VSTestExecutionDriver vsTestExecutionDriver, TestProjectFolders testProjectFolders, TestRunLogDriver testRunLogDriver) { - _hooksDriver = hooksDriver; + _bindingsDriver = bindingsDriver; _vsTestExecutionDriver = vsTestExecutionDriver; _testProjectFolders = testProjectFolders; _testRunLogDriver = testRunLogDriver; @@ -66,13 +66,13 @@ public void ThenTheBindingMethodIsExecuted(string methodName, int times) [Then(@"the hook '(.*)' is executed (\d+) times")] public void ThenTheHookIsExecuted(string methodName, int times) { - _hooksDriver.CheckIsHookExecuted(methodName, times); + _bindingsDriver.CheckIsHookExecuted(methodName, times); } [Then(@"the hooks are executed in the order")] public void ThenTheHooksAreExecutedInTheOrder(Table table) { - _hooksDriver.CheckIsHookExecutedInOrder(table.Rows.Select(r => r[0])); + _bindingsDriver.AssertHooksExecutedInOrder(table.Rows.Select(r => r[0]).ToArray()); } [Then(@"the execution log should contain text '(.*)'")] diff --git a/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs b/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs index 1ecab808b..f40a732ee 100644 --- a/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs +++ b/Tests/Reqnroll.Specs/StepDefinitions/ReqnrollConfigurationSteps.cs @@ -1,25 +1,24 @@ using Reqnroll.Specs.Drivers; using Reqnroll.TestProjectGenerator; using Reqnroll.TestProjectGenerator.Driver; -using ConfigurationDriver = Reqnroll.TestProjectGenerator.Driver.ConfigurationDriver; namespace Reqnroll.Specs.StepDefinitions { [Binding] public class ReqnrollConfigurationSteps { - private readonly ConfigurationDriver _configurationDriver; + private readonly ConfigurationFileDriver _configurationFileDriver; private readonly JsonConfigurationParserDriver _jsonConfigurationParserDriver; private readonly ConfigurationLoaderDriver _configurationLoaderDriver; private readonly TestSuiteSetupDriver _testSuiteSetupDriver; public ReqnrollConfigurationSteps( - ConfigurationDriver configurationDriver, + ConfigurationFileDriver configurationFileDriver, JsonConfigurationParserDriver jsonConfigurationParserDriver, ConfigurationLoaderDriver configurationLoaderDriver, TestSuiteSetupDriver testSuiteSetupDriver) { - _configurationDriver = configurationDriver; + _configurationFileDriver = configurationFileDriver; _jsonConfigurationParserDriver = jsonConfigurationParserDriver; _configurationLoaderDriver = configurationLoaderDriver; _testSuiteSetupDriver = testSuiteSetupDriver; @@ -28,7 +27,7 @@ public ReqnrollConfigurationSteps( [Given(@"the project has no reqnroll\.json configuration")] public void GivenTheProjectHasNoReqnroll_JsonConfiguration() { - _configurationDriver.SetConfigurationFormat(ConfigurationFormat.None); + _configurationFileDriver.SetConfigurationFormat(ConfigurationFormat.None); } [Given(@"there is a project with this reqnroll\.json configuration")] @@ -47,31 +46,31 @@ public void GivenTheReqnrollConfigurationIs(string reqnrollJsonConfig) [Given(@"the project is configured to use the (.+) provider")] public void GivenTheProjectIsConfiguredToUseTheUnitTestProvider(string providerName) { - _configurationDriver.SetUnitTestProvider(providerName); + _configurationFileDriver.SetUnitTestProvider(providerName); } [Given(@"Reqnroll is configured in the reqnroll\.json")] public void GivenReqnrollIsConfiguredInTheReqnrollJson() { - _configurationDriver.SetConfigurationFormat(ConfigurationFormat.Json); + _configurationFileDriver.SetConfigurationFormat(ConfigurationFormat.Json); } [Given(@"obsoleteBehavior configuration value is set to (.*)")] public void GivenObsoleteBehaviorConfigurationValueIsSetTo(string obsoleteBehaviorValue) { - _configurationDriver.SetRuntimeObsoleteBehavior(obsoleteBehaviorValue); + _configurationFileDriver.SetRuntimeObsoleteBehavior(obsoleteBehaviorValue); } [Given(@"row testing is (.+)")] public void GivenRowTestingIsRowTest(bool enabled) { - _configurationDriver.SetIsRowTestsAllowed(enabled); + _configurationFileDriver.SetIsRowTestsAllowed(enabled); } [Given(@"the type '(.*)' is registered as '(.*)' in Reqnroll runtime configuration")] public void GivenTheTypeIsRegisteredAsInReqnrollRuntimeConfiguration(string typeName, string interfaceName) { - _configurationDriver.AddRuntimeRegisterDependency(typeName, interfaceName); + _configurationFileDriver.AddRuntimeRegisterDependency(typeName, interfaceName); } [Given(@"there is a scenario")] diff --git a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs index f76a3a41a..561f42714 100644 --- a/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs @@ -1,9 +1,11 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using FluentAssertions; namespace Reqnroll.SystemTests.Generation; [TestCategory("Generation")] -public class GenerationTestBase : SystemTestBase +public abstract class GenerationTestBase : SystemTestBase { [TestMethod] public void GeneratorAllIn_sample_can_be_handled() @@ -19,29 +21,265 @@ public void GeneratorAllIn_sample_can_be_handled() public void Handles_simple_scenarios_without_namespace_collisions() { _projectsDriver.CreateProject("CollidingNamespace.Reqnroll", "C#"); - + AddSimpleScenarioAndOutline(); AddScenario( """ - Scenario: Sample Scenario - When something happens - Scenario: Scenario with DataTable When something happens with | who | when | | me | today | | someone else | tomorrow | """); - _projectsDriver.AddPassingStepBinding(); + AddPassingStepBinding(); + + ExecuteTests(); + + ShouldAllScenariosPass(); + } + + #region Test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) + protected virtual string GetExpectedPendingOutcome() => "NotExecuted"; + protected virtual string GetExpectedUndefinedOutcome() => "NotExecuted"; + protected virtual string GetExpectedIgnoredOutcome() => "NotExecuted"; + + [TestMethod] + public void Handles_different_scenario_and_scenario_outline_outcomes() + { + AddFeatureFile( + """ + Feature: Sample Feature + + Scenario: Passing scenario + When the step passes + + Scenario: Failing scenario + When the step fails + + Scenario: Pending scenario + When the step is pending + + Scenario: Undefined scenario + When the step is undefined + + @ignore + Scenario: Ignored scenario + When the step fails + + Scenario Outline: SO + When the step + Examples: + | result | + | passes | + | fails | + | is pending | + | is undefined | + @ignore + Examples: + | result | + | ignored | + + @ignore + Scenario Outline: ExampleIgnored + When the step + Examples: + | result | + | fails | + """); + _projectsDriver.AddPassingStepBinding(stepRegex: "the step passes"); + _projectsDriver.AddFailingStepBinding(stepRegex: "the step fails"); + _projectsDriver.AddStepBinding("StepDefinition", regex: "the step is pending", "throw new PendingStepException();", "Throw New PendingStepException()"); + ExecuteTests(); + + // handles PASSED + var expectedPassedOutcome = "Passed"; + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("Passing")) + .Which.Outcome.Should().Be(expectedPassedOutcome); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("passes")) + .Which.Outcome.Should().Be(expectedPassedOutcome); + + // handles FAILED + var expectedFailedOutcome = "Failed"; + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("Failing")) + .Which.Outcome.Should().Be(expectedFailedOutcome); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("fails")) + .Which.Outcome.Should().Be(expectedFailedOutcome); + + // handles PENDING + var expectedPendingOutcome = GetExpectedPendingOutcome(); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("Pending")) + .Which.Outcome.Should().Be(expectedPendingOutcome); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("pending")) + .Which.Outcome.Should().Be(expectedPendingOutcome); + + // handles UNDEFINED + var expectedUndefinedOutcome = GetExpectedUndefinedOutcome(); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("Undefined")) + .Which.Outcome.Should().Be(expectedUndefinedOutcome); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("undefined")) + .Which.Outcome.Should().Be(expectedUndefinedOutcome); + + // handles IGNORED + var expectedIgnoredOutcome = GetExpectedIgnoredOutcome(); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("Ignored")) + .Which.Outcome.Should().Be(expectedIgnoredOutcome); + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("ExampleIgnored")) + .Which.Outcome.Should().Be(expectedIgnoredOutcome); + AssertIgnoredScenarioOutlineExampleHandled(); + } + + protected virtual void AssertIgnoredScenarioOutlineExampleHandled() + { + // the scenario outline examples ignored by a tag on the examples block are not generated + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().NotContain(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("ignored")) + .And.Subject.Where(tr => tr.TestName.StartsWith("SO")).Should().HaveCount(4); + + } + #endregion + + #region Test async steps (async steps are executed in order) + [TestMethod] + public void Async_steps_are_executed_in_order() + { + AddScenario( + """ + Scenario: Async Scenario Steps + When Async step 1 is called + When Async step 2 is called + When Async step 3 is called + """); + + AddBindingClass( + """ + namespace AsyncSequence.StepDefinitions + { + [Binding] + public class AsyncSequenceStepDefinitions + { + [When("Async step 1 is called")] + public async Task WhenStep1IsTaken() + { + await Task.Run(() => global::Log.LogStep() ); + } + [When("Async step 2 is called")] + public async Task WhenStep2IsTaken() + { + await Task.Run(() => global::Log.LogStep() ); + } + [When("Async step 3 is called")] + public async Task WhenStep3IsTaken() + { + await Task.Run(() => global::Log.LogStep() ); + } + } + } + """); + + ExecuteTests(); + _bindingDriver.AssertExecutedStepsEqual("WhenStep1IsTaken", "WhenStep2IsTaken", "WhenStep3IsTaken"); + + ShouldAllScenariosPass(); + } + #endregion + + #region Test hooks: before/after run, feature & scenario hook (require special handling by test frameworks) + [TestMethod] + public void TestRun_Feature_and_Scenario_hooks_are_executed_in_right_order() + { + var testsInFeatureFile = 3; + AddSimpleScenario(); + AddSimpleScenarioOutline(testsInFeatureFile - 1); + AddPassingStepBinding(); + AddHookBinding("BeforeTestRun"); + AddHookBinding("AfterTestRun"); + AddHookBinding("BeforeFeature"); + AddHookBinding("AfterFeature"); + AddHookBinding("BeforeScenario"); + AddHookBinding("AfterScenario"); ExecuteTests(); + _bindingDriver.AssertExecutedHooksEqual( + "BeforeTestRun", + "BeforeFeature", + "BeforeScenario", + "AfterScenario", + "BeforeScenario", + "AfterScenario", + "BeforeScenario", + "AfterScenario", + "AfterFeature", + "AfterTestRun"); ShouldAllScenariosPass(); } + #endregion + + #region Test scenario outlines (nr of examples, params are available in ScenarioContext, allowRowTests=false, examples tags) + + [TestMethod] + public void Scenario_outline_examples_gather_tags_and_parameters() + { + AddFeatureFile( + """ + @feature_tag + Feature: Sample Feature + + @rule_tag + Rule: Sample Rule + + @so1_tag + Scenario Outline: SO1 + When happens to + Examples: + | what1 | person | + | something | me | + | nothing | you | + + @so2_tag + Scenario Outline: SO2 + When happens to + Examples: + | what2 | person | + | something | me | + @e3_tag + Examples: E3 + | what2 | person | + | nothing | you | + """); + _projectsDriver.AddStepBinding("StepDefinition", regex: ".*", + """ + global::Log.LogCustom("tags", string.Join(",", _scenarioContext.ScenarioInfo.CombinedTags)); + global::Log.LogCustom("parameters", string.Join(",", _scenarioContext.ScenarioInfo.Arguments.OfType().Select(kv => $"{kv.Key}={kv.Value}"))); + """); + + ExecuteTests(); + + ShouldAllScenariosPass(4); + + _bindingDriver.GetActualLogLines("tags").Should().BeEquivalentTo( + "-> tags: so1_tag,feature_tag,rule_tag:StepBinding", + "-> tags: so1_tag,feature_tag,rule_tag:StepBinding", + "-> tags: so2_tag,feature_tag,rule_tag:StepBinding", + "-> tags: so2_tag,e3_tag,feature_tag,rule_tag:StepBinding"); + + _bindingDriver.GetActualLogLines("parameters").Should().BeEquivalentTo( + "-> parameters: what1=something,person=me:StepBinding", + "-> parameters: what1=nothing,person=you:StepBinding", + "-> parameters: what2=something,person=me:StepBinding", + "-> parameters: what2=nothing,person=you:StepBinding"); + } + + #endregion - //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) - //TODO: test async steps (async steps are executed in order) - //TODO: test hooks: before/after test run (require special handling by test frameworks) - //TODO: test hooks: before/after test feature & scenario (require special handling by test frameworks) - //TODO: test scenario outlines (nr of examples, params are available in ScenarioContext, allowRowTests=false, examples tags) //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class } diff --git a/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs index 9d4b15b4a..f5f8238ef 100644 --- a/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs +++ b/Tests/Reqnroll.SystemTests/Generation/NUnitGenerationTest.cs @@ -1,3 +1,4 @@ +using FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using Reqnroll.TestProjectGenerator; @@ -16,4 +17,11 @@ protected override void TestInitialize() base.TestInitialize(); _testRunConfiguration.UnitTestProvider = UnitTestProvider.NUnit3; } + + protected override void AssertIgnoredScenarioOutlineExampleHandled() + { + _vsTestExecutionDriver.LastTestExecutionResult.LeafTestResults + .Should().ContainSingle(tr => tr.TestName.StartsWith("SO") && tr.TestName.Contains("ignored")) + .Which.Outcome.Should().Be(GetExpectedIgnoredOutcome()); + } } \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs index adf2a569f..086f2ab37 100644 --- a/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs +++ b/Tests/Reqnroll.SystemTests/Generation/XUnitGenerationTest.cs @@ -16,4 +16,7 @@ protected override void TestInitialize() base.TestInitialize(); _testRunConfiguration.UnitTestProvider = UnitTestProvider.xUnit; } + + protected override string GetExpectedPendingOutcome() => "Failed"; + protected override string GetExpectedUndefinedOutcome() => "Failed"; } \ No newline at end of file diff --git a/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs b/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs index 7ff9c6efd..cfb178601 100644 --- a/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs +++ b/Tests/Reqnroll.SystemTests/Portability/PortabilityTestBase.cs @@ -1,6 +1,5 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Reqnroll.TestProjectGenerator.Driver; -using System.Runtime.InteropServices; namespace Reqnroll.SystemTests.Portability; @@ -40,7 +39,21 @@ public void GeneratorAllIn_sample_can_be_compiled_with_DotnetMSBuild() _compilationDriver.CompileSolution(); } - //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline) - //TODO: test async steps (async steps are executed in order) - //TODO: test before/after test run hooks (.NET Framework version of Reqnroll is subscribed to assembly unload) + #region Test before/after test run hooks (.NET Framework version of Reqnroll is subscribed to assembly unload) + [TestMethod] + public void TestRun_hooks_are_executed() + { + AddSimpleScenario(); + AddPassingStepBinding(); + AddHookBinding("BeforeTestRun"); + AddHookBinding("AfterTestRun"); + + ExecuteTests(); + + _bindingDriver.AssertExecutedHooksEqual( + "BeforeTestRun", + "AfterTestRun"); + ShouldAllScenariosPass(); + } + #endregion } diff --git a/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs b/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs index d64f4d6ab..5f098744f 100644 --- a/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs +++ b/Tests/Reqnroll.SystemTests/Smoke/SmokeTest.cs @@ -9,11 +9,7 @@ public class SmokeTest : SystemTestBase [TestMethod] public void Handles_the_simplest_scenario() { - AddScenario( - """ - Scenario: Sample Scenario - When something happens - """); + AddSimpleScenario(); _projectsDriver.AddPassingStepBinding(); ExecuteTests(); diff --git a/Tests/Reqnroll.SystemTests/SystemTestBase.cs b/Tests/Reqnroll.SystemTests/SystemTestBase.cs index e785c36fa..e0b8d9013 100644 --- a/Tests/Reqnroll.SystemTests/SystemTestBase.cs +++ b/Tests/Reqnroll.SystemTests/SystemTestBase.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Text.RegularExpressions; using FluentAssertions; using Microsoft.Extensions.DependencyInjection; @@ -22,6 +23,8 @@ public abstract class SystemTestBase protected TestRunConfiguration _testRunConfiguration = null!; protected CurrentVersionDriver _currentVersionDriver = null!; protected CompilationDriver _compilationDriver = null!; + protected BindingsDriver _bindingDriver = null!; + protected TestProjectFolders _testProjectFolders = null!; protected int _preparedTests = 0; @@ -79,6 +82,21 @@ protected virtual void TestInitialize() _executionDriver = _testContainer.GetService(); _vsTestExecutionDriver = _testContainer.GetService(); _compilationDriver = _testContainer.GetService(); + _testProjectFolders = _testContainer.GetService(); + _bindingDriver = _testContainer.GetService(); + } + + + [TestCleanup] + public void TestCleanupMethod() + { + TestCleanup(); + } + + protected virtual void TestCleanup() + { + if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + _folderCleaner.CleanSolutionFolder(); } protected void AddFeatureFileFromResource(string fileName, int? preparedTests = null) @@ -134,6 +152,41 @@ protected void AddScenario(string scenarioContent, int? preparedTests = null) UpdatePreparedTests(scenarioContent, preparedTests); } + protected void AddSimpleScenario() + { + AddScenario( + """ + Scenario: Sample Scenario + When something happens + """); + } + + protected void AddSimpleScenarioOutline(int numberOfExamples = 2) + { + var examples = numberOfExamples == 2 + ? """ + | me | + | you | + """ + : string.Join( + Environment.NewLine, + Enumerable.Range(1, numberOfExamples).Select(i => $" | example {i} |")); + AddScenario( + $""" + Scenario Outline: Scenario outline with examples + When something happens to + Examples: + | person | + {examples} + """); + } + + protected void AddSimpleScenarioAndOutline() + { + AddSimpleScenario(); + AddSimpleScenarioOutline(); + } + private void UpdatePreparedTests(string gherkinContent, int? preparedTests) { if (preparedTests == null) @@ -161,13 +214,33 @@ protected void ExecuteTests() protected void ShouldAllScenariosPass(int? expectedNrOfTestsSpec = null) { - if (expectedNrOfTestsSpec == null && _preparedTests == 0) + int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec); + _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass"); + } + + protected int ConfirmAllTestsRan(int? expectedNrOfTestsSpec) + { + if (expectedNrOfTestsSpec == null && _preparedTests == 0) throw new ArgumentException($"If {nameof(_preparedTests)} is not set, the {nameof(expectedNrOfTestsSpec)} is mandatory.", nameof(expectedNrOfTestsSpec)); var expectedNrOfTests = expectedNrOfTestsSpec ?? _preparedTests; _vsTestExecutionDriver.LastTestExecutionResult.Should().NotBeNull(); _vsTestExecutionDriver.LastTestExecutionResult.Total.Should().Be(expectedNrOfTests, $"the run should contain {expectedNrOfTests} tests"); - _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass"); - _folderCleaner.CleanSolutionFolder(); + return expectedNrOfTests; + } + + protected void AddHookBinding(string eventType, string? name = null, string code = "") + { + _projectsDriver.AddHookBinding(eventType, name, code: code); + } + + protected void AddPassingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*") + { + _projectsDriver.AddPassingStepBinding(scenarioBlock, stepRegex); + } + + protected void AddBindingClass(string content) + { + _projectsDriver.AddBindingClass(content); } }