diff --git a/TestFx.sln b/TestFx.sln index 33d02154c5..701090895f 100644 --- a/TestFx.sln +++ b/TestFx.sln @@ -21,6 +21,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTest.Core", "src\TestFram EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PlatformServices.Desktop", "src\Adapter\PlatformServices.Desktop\PlatformServices.Desktop.csproj", "{B0FCE474-14BC-449A-91EA-A433342C0D63}" EndProject +Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "PlatformServices.Shared", "src\Adapter\PlatformServices.Shared\PlatformServices.Shared.shproj", "{2177C273-AE07-43B3-B87A-443E47A23C5A}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Extension.Desktop", "src\TestFramework\Extension.Desktop\Extension.Desktop.csproj", "{A7EA583B-A2B0-47DA-A058-458F247C7575}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Extension.Core", "src\TestFramework\Extension.Core\Extension.Core.csproj", "{6C9FE494-8315-4667-B3F6-75DC62A62319}" @@ -142,8 +144,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "targets", "targets", "{F60B ProjectSection(SolutionItems) = preProject scripts\build\TestFx.Loc.targets = scripts\build\TestFx.Loc.targets scripts\build\TestFx.Settings.targets = scripts\build\TestFx.Settings.targets - scripts\build\TestFx.Versions.targets = scripts\build\TestFx.Versions.targets scripts\build\TestFx.targets = scripts\build\TestFx.targets + scripts\build\TestFx.Versions.targets = scripts\build\TestFx.Versions.targets EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{BCF525B1-E67F-486D-B091-06A8BB8A2793}" @@ -186,10 +188,25 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeploymentTestProjectNetCor EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProjectNetCore", "test\E2ETests\TestAssets\TimeoutTestProjectNetCore\TimeoutTestProjectNetCore.csproj", "{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DiscoveryAndExecutionTests", "test\E2ETests\DiscoveryAndExecutionTests\DiscoveryAndExecutionTests.csproj", "{EEE57613-6424-4A1C-9635-B73768F2146D}" + ProjectSection(ProjectDependencies) = postProject + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} + {98BA6D2C-1F3D-4636-8E1D-D4932B7A253D} = {98BA6D2C-1F3D-4636-8E1D-D4932B7A253D} + {A7EA583B-A2B0-47DA-A058-458F247C7575} = {A7EA583B-A2B0-47DA-A058-458F247C7575} + {BBC99A6B-4490-49DD-9C12-AF2C1E95576E} = {BBC99A6B-4490-49DD-9C12-AF2C1E95576E} + {B0FCE474-14BC-449A-91EA-A433342C0D63} = {B0FCE474-14BC-449A-91EA-A433342C0D63} + {4004757A-0080-4410-B90A-6169B20F151B} = {4004757A-0080-4410-B90A-6169B20F151B} + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} + {5A4967CD-B527-4D43-81C2-4CA90EE10222} = {5A4967CD-B527-4D43-81C2-4CA90EE10222} + {9C1219E0-E775-47F9-9236-63F03F774801} = {9C1219E0-E775-47F9-9236-63F03F774801} + {7252D9E3-267D-442C-96BC-C73AEF3241D6} = {7252D9E3-267D-442C-96BC-C73AEF3241D6} + EndProjectSection EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\Adapter\PlatformServices.Shared\PlatformServices.Shared.projitems*{2177c273-ae07-43b3-b87a-443e47a23c5a}*SharedItemsImports = 13 src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13 src\TestFramework\Extension.Shared\Extension.Shared.projitems*{df131865-84ee-4540-8112-e88acebdea09}*SharedItemsImports = 4 EndGlobalSection @@ -346,12 +363,12 @@ Global {B0FCE474-14BC-449A-91EA-A433342C0D63}.Debug|x86.Build.0 = Debug|Any CPU {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|Any CPU.ActiveCfg = Release|Any CPU {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|Any CPU.Build.0 = Release|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.Build.0 = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.Build.0 = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.Build.0 = Debug|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.Build.0 = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.Build.0 = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.Build.0 = Release|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU @@ -1168,6 +1185,30 @@ Global {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.Build.0 = Release|Any CPU {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.ActiveCfg = Release|Any CPU {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|ARM.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|ARM.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x64.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x64.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x86.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x86.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|Any CPU.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|ARM.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|ARM.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x64.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x64.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x86.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1181,6 +1222,7 @@ Global {E48AC786-E150-4F41-9A16-32F02E4493D8} = {FF8B1B72-55A1-4FFE-809E-7B79323ED8D0} {7252D9E3-267D-442C-96BC-C73AEF3241D6} = {E48AC786-E150-4F41-9A16-32F02E4493D8} {B0FCE474-14BC-449A-91EA-A433342C0D63} = {24088844-2107-4DB2-8F3F-CBCA94FC4B28} + {2177C273-AE07-43B3-B87A-443E47A23C5A} = {24088844-2107-4DB2-8F3F-CBCA94FC4B28} {A7EA583B-A2B0-47DA-A058-458F247C7575} = {E48AC786-E150-4F41-9A16-32F02E4493D8} {6C9FE494-8315-4667-B3F6-75DC62A62319} = {E48AC786-E150-4F41-9A16-32F02E4493D8} {F2D0BF2C-38F2-4244-80E3-4AAD1C3F4C89} = {A9596292-7E67-4566-9096-143DDAA4E8D8} @@ -1232,6 +1274,7 @@ Global {26F0B8EF-92D4-4A23-ACB4-D1B662F0EEBE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} + {EEE57613-6424-4A1C-9635-B73768F2146D} = {F1A27537-78D1-4BBD-8E76-ADB31BC0C2B4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9} diff --git a/scripts/Build.ps1 b/scripts/Build.ps1 index 618ab1f4d3..562e272f16 100644 --- a/scripts/Build.ps1 +++ b/scripts/Build.ps1 @@ -48,6 +48,9 @@ Param( [Alias("tpv")] [string] $TestPlatformVersion = $null, + + [Alias("np")] + [Switch] $DisallowPrereleaseMSBuild, [Alias("f")] [Switch] $Force, @@ -111,20 +114,23 @@ function Print-Help { Write-Host -object "" Write-Host -object "********* MSTest Adapter Build Script *********" Write-Host -object "" - Write-Host -object " Help (-h) - [Switch] - Prints this help message." - Write-Host -object " Clean (-cl) - [Switch] - Indicates that this should be a clean build." - Write-Host -object " SkipRestore (-sr) - [Switch] - Indicates nuget package restoration should be skipped." - Write-Host -object " ClearPackageCache (-cache) - [Switch] - Indicates local package cache should be cleared before restore." - Write-Host -object " Updatexlf (-uxlf) - [Switch] - Indicates that there are resource changes and that these need to be copied to other languages as well." - Write-Host -object " IsLocalizedBuild (-loc) - [Switch] - Indicates that the build needs to generate resource assemblies as well." - Write-Host -object " Official - [Switch] - Indicates that this is an official build. Only used in CI builds." - Write-Host -object " Full - [Switch] - Indicates to perform a full build which includes Adapter, Framework" + Write-Host -object " Help (-h) - [switch] - Prints this help message." + Write-Host -object " Clean (-cl) - [switch] - Indicates that this should be a clean build." + Write-Host -object " SkipRestore (-sr) - [switch] - Indicates nuget package restoration should be skipped." + Write-Host -object " ClearPackageCache (-cache) - [switch] - Indicates local package cache should be cleared before restore." + Write-Host -object " Updatexlf (-uxlf) - [switch] - Indicates that there are resource changes and that these need to be copied to other languages as well." + Write-Host -object " IsLocalizedBuild (-loc) - [switch] - Indicates that the build needs to generate resource assemblies as well." + Write-Host -object " Official - [switch] - Indicates that this is an official build. Only used in CI builds." + Write-Host -object " Full - [switch] - Indicates to perform a full build which includes Adapter, Framework" + Write-Host -object " DisallowPrereleaseMSBuild (-np) - [switch] - Uses an RTM version of MSBuild to build the projects" + Write-Host -object "" + Write-Host -object " Configuration (-c) - [string] - Specifies the build configuration. Defaults to 'Debug'." + Write-Host -object " FrameworkVersion (-fv) - [string] - Specifies the version of the Test Framework nuget package." + Write-Host -object " AdapterVersion (-av) - [string] - Specifies the version of the Test Adapter nuget package." + Write-Host -object " VersionSuffix (-vs) - [string] - Specifies the version suffix for the nuget packages." + Write-Host -object " Target - [string] - Specifies the build target. Defaults to 'Build'." Write-Host -object "" - Write-Host -object " Configuration (-c) - [String] - Specifies the build configuration. Defaults to 'Debug'." - Write-Host -object " FrameworkVersion (-fv) - [String] - Specifies the version of the Test Framework nuget package." - Write-Host -object " AdapterVersion (-av) - [String] - Specifies the version of the Test Adapter nuget package." - Write-Host -object " VersionSuffix (-vs) - [String] - Specifies the version suffix for the nuget packages." - Write-Host -object " Target - [String] - Specifies the build target. Defaults to 'Build'." + Write-Host -object " Steps (-s) - [string[]] - List of build steps to run, valid steps: `"UpdateTPVersion`", `"Restore`", `"Build`", `"Publish`"" Write-Host -object "" Exit 0 @@ -316,7 +322,7 @@ function Sync-PackageVersions { Replace-InFile -File $_ -RegEx $packageRegex -ReplaceWith (' ..\..\ - ..\..\ + $(TestFxRoot) + true - + + diff --git a/scripts/build/TestFx.Versions.targets b/scripts/build/TestFx.Versions.targets index 4e3fe47b70..b009a9c11a 100644 --- a/scripts/build/TestFx.Versions.targets +++ b/scripts/build/TestFx.Versions.targets @@ -1,7 +1,7 @@  - 16.9.1 + 16.10.0-preview-20210304-04 11.0 \ No newline at end of file diff --git a/scripts/build/TestFx.targets b/scripts/build/TestFx.targets index 3eeed84813..b4c443c849 100644 --- a/scripts/build/TestFx.targets +++ b/scripts/build/TestFx.targets @@ -2,7 +2,7 @@ - + ..\..\ @@ -84,7 +84,7 @@ - + diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 213eae1fe1..7d948ba4dd 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -136,7 +136,7 @@ function Locate-VsInstallPath($hasVsixExtension = "false") { Write-Verbose "$vswhere -latest -products * -requires $requiredPackageIds -property installationPath" try { - if ($Official) { + if ($Official -or $DisallowPrereleaseMSBuild) { $vsInstallPath = & $vswhere -latest -products * -requires $requiredPackageIds -property installationPath } else { diff --git a/scripts/test.ps1 b/scripts/test.ps1 index cdf2239303..7a153cc590 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -39,7 +39,7 @@ $CurrentScriptDir = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)) $env:TF_ROOT_DIR = $CurrentScriptDir.Parent.FullName $env:TF_TOOLS_DIR = Join-Path $env:TF_ROOT_DIR "tools" $env:DOTNET_CLI_VERSION = "6.0.100-alpha.1.21067.8" -$env:TF_TESTS_OUTDIR_PATTERN = "*.Tests" +$env:TF_TESTS_OUTDIR_PATTERN = "*Tests" $env:TF_UNITTEST_FILES_PATTERN = "*.UnitTests*.dll" $env:TF_COMPONENTTEST_FILES_PATTERN = "*.ComponentTests*.dll" $env:TF_E2ETEST_FILES_PATTERN = "*.E2ETests*.dll" @@ -95,7 +95,7 @@ function Invoke-Test # Get test assemblies from these folders that match the pattern specified. foreach($container in $testFolders) { - $testContainer = Get-ChildItem $container\* -Recurse -Include $env:TF_UNITTEST_FILES_PATTERN, $env:TF_COMPONENTTEST_FILES_PATTERN, $env:TF_E2ETEST_FILES_PATTERN + $testContainer = Get-ChildItem $container\* -Recurse -Include $env:TF_UNITTEST_FILES_PATTERN, $env:TF_COMPONENTTEST_FILES_PATTERN, $env:TF_E2ETEST_FILES_PATTERN, "DiscoveryAndExecutionTests.dll" $testContainerName = $testContainer.Name $testContainerPath = $testContainer.FullName diff --git a/src/Adapter/MSTest.CoreAdapter/Constants.cs b/src/Adapter/MSTest.CoreAdapter/Constants.cs index 80f4f4637b..6d930643ae 100644 --- a/src/Adapter/MSTest.CoreAdapter/Constants.cs +++ b/src/Adapter/MSTest.CoreAdapter/Constants.cs @@ -29,7 +29,6 @@ internal static class Constants internal static readonly Uri ExecutorUri = new Uri(ExecutorUriString); #region Test Property registration - internal static readonly TestProperty DescriptionProperty = TestProperty.Register("Description", DescriptionLabel, typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); internal static readonly TestProperty WorkItemIdsProperty = TestProperty.Register("WorkItemIds", WorkItemIdsLabel, typeof(string[]), TestPropertyAttributes.Hidden, typeof(TestCase)); @@ -90,6 +89,9 @@ internal static class Constants internal static readonly TestProperty TfsTeamProjectProperty = TestProperty.Register(TfsTeamProject, TfsTeamProject, typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); + internal static readonly TestProperty TestDynamicDataTypeProperty = TestProperty.Register("MSTest.DynamicDataType", "DynamicDataType", typeof(int), TestPropertyAttributes.Hidden, typeof(TestCase)); + + internal static readonly TestProperty TestDynamicDataProperty = TestProperty.Register("MSTest.DynamicData", "DynamicData", typeof(string[]), TestPropertyAttributes.Hidden, typeof(TestCase)); #endregion #region Private Constants diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs index bbb1e42701..f10a5d420b 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs @@ -6,21 +6,34 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Discovery using System; using System.Collections.Generic; using System.Diagnostics; - using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Reflection; using System.Security; using System.Text; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using ITestContext = Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ITestContext; + using UTF = Microsoft.VisualStudio.TestTools.UnitTesting; + /// /// Enumerates through all types in the assembly in search of valid test methods. /// internal class AssemblyEnumerator : MarshalByRefObject { + /// + /// Helper for reflection API's. + /// + private static readonly ReflectHelper ReflectHelper = new ReflectHelper(); + + /// + /// Type cache + /// + private readonly TypeCache typeCache = new TypeCache(ReflectHelper); + /// /// Initializes a new instance of the class. /// @@ -40,6 +53,11 @@ public AssemblyEnumerator(MSTestSettings settings) MSTestSettings.PopulateSettings(settings); } + /// + /// Sets run settings to use for current discovery session. + /// + public string RunSettingsXml { private get; set; } + /// /// Returns object to be used for controlling lifetime, null means infinite lifetime. /// @@ -55,14 +73,15 @@ public override object InitializeLifetimeService() /// /// Enumerates through all types in the assembly in search of valid test methods. /// - /// The assembly file name. - /// Contains warnings if any, that need to be passed back to the caller. - /// A collection of Test Elements. - [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Catching a generic exception since it is a requirement to not abort discovery in case of any errors.")] + /// The assembly file name. + /// Contains warnings if any, that need to be passed back to the caller. + /// A collection of Test Elements. internal ICollection EnumerateAssembly(string assemblyFileName, out ICollection warnings) { Debug.Assert(!string.IsNullOrWhiteSpace(assemblyFileName), "Invalid assembly file name."); + string runSettingsXml = this.RunSettingsXml; + var warningMessages = new List(); var tests = new List(); @@ -73,9 +92,7 @@ internal ICollection EnumerateAssembly(string assemblyFileName, // For normal test assemblies continue loading it in the default context since: // 1. There isn't much benefit in terms of Performance loading the assembly in a Reflection Only context during discovery. // 2. Loading it in Reflection only context entails a bunch of custom logic to identify custom attributes which is over-kill for normal desktop users. - assembly = PlatformServiceProvider.Instance.FileOperations.LoadAssembly( - assemblyFileName, - isReflectionOnly: true); + assembly = PlatformServiceProvider.Instance.FileOperations.LoadAssembly(assemblyFileName, isReflectionOnly: true); } else { @@ -91,42 +108,8 @@ internal ICollection EnumerateAssembly(string assemblyFileName, continue; } - string typeFullName = null; - - try - { - ICollection warningsFromTypeEnumerator; - - typeFullName = type.FullName; - var unitTestCases = this.GetTypeEnumerator(type, assemblyFileName).Enumerate(out warningsFromTypeEnumerator); - - if (warningsFromTypeEnumerator != null) - { - warningMessages.AddRange(warningsFromTypeEnumerator); - } - - if (unitTestCases != null) - { - tests.AddRange(unitTestCases); - } - } - catch (Exception exception) - { - // If we fail to discover type from a class, then don't abort the discovery - // Move to the next type. - string message = string.Format( - CultureInfo.CurrentCulture, - Resource.CouldNotInspectTypeDuringDiscovery, - typeFullName, - assemblyFileName, - exception.Message); - warningMessages.Add(message); - - PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( - "AssemblyEnumerator: Exception occurred while enumerating type {0}. {1}", - typeFullName, - exception); - } + var testsInType = this.DiscoverTestsInType(assemblyFileName, runSettingsXml, assembly, type, warningMessages); + tests.AddRange(testsInType); } warnings = warningMessages; @@ -149,20 +132,13 @@ internal Type[] GetTypes(Assembly assembly, string assemblyFileName, ICollection } catch (ReflectionTypeLoadException ex) { - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "MSTestExecutor.TryGetTests: Failed to discover tests from {0}. Reason:{1}", - assemblyFileName, - ex); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("Exceptions thrown from the Loader :"); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"MSTestExecutor.TryGetTests: {Resource.TestAssembly_AssemblyDiscoveryFailure}", assemblyFileName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.ExceptionsThrown); if (ex.LoaderExceptions != null) { // If not able to load all type, log a warning and continue with loaded types. - var message = string.Format( - CultureInfo.CurrentCulture, - Resource.TypeLoadFailed, - assemblyFileName, - this.GetLoadExceptionDetails(ex)); + var message = string.Format(CultureInfo.CurrentCulture, Resource.TypeLoadFailed, assemblyFileName, this.GetLoadExceptionDetails(ex)); warningMessages?.Add(message); @@ -220,11 +196,191 @@ internal string GetLoadExceptionDetails(ReflectionTypeLoadException ex) /// a TypeEnumerator instance. internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName) { - var reflectHelper = new ReflectHelper(); - var typevalidator = new TypeValidator(reflectHelper); - var testMethodValidator = new TestMethodValidator(reflectHelper); + var typeValidator = new TypeValidator(ReflectHelper); + var testMethodValidator = new TestMethodValidator(ReflectHelper); + + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); + } + + private IEnumerable DiscoverTestsInType(string assemblyFileName, string runSettingsXml, Assembly assembly, Type type, List warningMessages) + { + var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(assemblyFileName); + sourceLevelParameters = RunSettingsUtilities.GetTestRunParameters(runSettingsXml)?.ConcatWithOverwrites(sourceLevelParameters) + ?? sourceLevelParameters + ?? new Dictionary(); + + string typeFullName = null; + var tests = new List(); + + try + { + typeFullName = type.FullName; + var unitTestCases = this.GetTypeEnumerator(type, assemblyFileName).Enumerate(out var warningsFromTypeEnumerator); + var typeIgnored = ReflectHelper.IsAttributeDefined(type, typeof(UTF.IgnoreAttribute), false); + + if (warningsFromTypeEnumerator != null) + { + warningMessages.AddRange(warningsFromTypeEnumerator); + } + + if (unitTestCases != null) + { + foreach (var test in unitTestCases) + { + test.Ignored = typeIgnored || test.Ignored; + if (test.Ignored) + { + tests.Add(test); + continue; + } + + if (this.DynamicDataAttached(sourceLevelParameters, assembly, test, tests)) + { + continue; + } + + tests.Add(test); + } + } + } + catch (Exception exception) + { + // If we fail to discover type from a class, then don't abort the discovery + // Move to the next type. + string message = string.Format(CultureInfo.CurrentCulture, Resource.CouldNotInspectTypeDuringDiscovery, typeFullName, assemblyFileName, exception.Message); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"AssemblyEnumerator: {message}"); + warningMessages.Add(message); + } + + return tests; + } + + private bool DynamicDataAttached(IDictionary sourceLevelParameters, Assembly assembly, UnitTestElement test, List tests) + { + if (test.TestMethod.HasManagedMethodAndTypeProperties == false) + { + return false; + } + + using (var writer = new ThreadSafeStringWriter(CultureInfo.InvariantCulture)) + { + var testMethod = test.TestMethod; + var testContext = PlatformServiceProvider.Instance.GetTestContext(testMethod, writer, sourceLevelParameters); + var testMethodInfo = this.typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces); + if (testMethodInfo == null) + { + return false; + } + + return false /* DataSourceAttribute discovery is disabled for now, since we cannot serialize DataRow values. + || this.TryProcessDataSource(test, testMethodInfo, testContext, tests) */ + || this.TryProcessTestDataSourceTests(test, testMethodInfo, tests); + } + } + + private bool TryProcessDataSource(UnitTestElement test, TestMethodInfo testMethodInfo, ITestContext testContext, List tests) + { + UTF.DataSourceAttribute[] dataSourceAttributes = ReflectHelper.GetAttributes(testMethodInfo.MethodInfo, false); + if (dataSourceAttributes != null && dataSourceAttributes.Length == 1) + { + try + { + return this.ProcessDataSourceTests(test, testMethodInfo, testContext, tests); + } + catch (Exception ex) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumarateDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + return false; + } + } + else if (dataSourceAttributes != null && dataSourceAttributes.Length > 1) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumarateDataSourceAttribute_MoreThenOneDefined, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, dataSourceAttributes.Length); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + throw new InvalidOperationException(message); + } + + return false; + } + + private bool ProcessDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, ITestContext testContext, List tests) + { + var dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(testMethodInfo, testContext); + if (dataRows == null || !dataRows.Any()) + { + return false; + } + + try + { + int rowIndex = 0; + + foreach (var dataRow in dataRows) + { + // TODO: Test serialization + rowIndex++; + + var displayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, rowIndex); + var discoveredTest = test.Clone(); + discoveredTest.DisplayName = displayName; + discoveredTest.TestMethod.DataType = DynamicDataType.DataSourceAttribute; + discoveredTest.TestMethod.Data = new[] { (object)rowIndex }; + tests.Add(discoveredTest); + } + + return true; + } + catch + { + testContext.SetDataConnection(null); + testContext.SetDataRow(null); + } + + return false; + } + + private bool TryProcessTestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List tests) + { + var methodInfo = testMethodInfo.MethodInfo; + + UTF.ITestDataSource[] testDataSources = ReflectHelper.GetAttributes(methodInfo, false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + try + { + return this.ProcessTestDataSourceTests(test, (MethodInfo)methodInfo, testDataSources, tests); + } + catch (Exception ex) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumarateIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + return false; + } + } + + private bool ProcessTestDataSourceTests(UnitTestElement test, MethodInfo methodInfo, UTF.ITestDataSource[] testDataSources, List tests) + { + if (testDataSources == null || testDataSources.Length == 0) + { + return false; + } + + foreach (var dataSource in testDataSources) + { + var data = dataSource.GetData(methodInfo); + + foreach (var d in data) + { + var discoveredTest = test.Clone(); + discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, d); + + discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; + discoveredTest.TestMethod.Data = d; + + tests.Add(discoveredTest); + } + } - return new TypeEnumerator(type, assemblyFileName, reflectHelper, typevalidator, testMethodValidator); + return true; } } } diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs index fb5683726b..8cda44b9e5 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs @@ -5,7 +5,6 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Discovery { using System; using System.Collections.Generic; - using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Reflection; @@ -22,8 +21,7 @@ internal class AssemblyEnumeratorWrapper /// /// Assembly name for UTF /// - private static readonly AssemblyName UnitTestFrameworkAssemblyName = - typeof(TestMethodAttribute).GetTypeInfo().Assembly.GetName(); + private static readonly AssemblyName UnitTestFrameworkAssemblyName = typeof(TestMethodAttribute).GetTypeInfo().Assembly.GetName(); /// /// Gets test elements from an assembly. @@ -32,12 +30,7 @@ internal class AssemblyEnumeratorWrapper /// The run Settings. /// Contains warnings if any, that need to be passed back to the caller. /// A collection of test elements. - [SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Catching a generic exception since it is a requirement to not abort discovery in case of any errors.")] - [SuppressMessage("Microsoft.Design", "CA1021:AvoidOutParameters", MessageId = "3#", Justification = "This is only for internal use.")] - internal ICollection GetTests( - string assemblyFileName, - IRunSettings runSettings, - out ICollection warnings) + internal ICollection GetTests(string assemblyFileName, IRunSettings runSettings, out ICollection warnings) { warnings = new List(); @@ -52,16 +45,11 @@ internal ICollection GetTests( { if (!PlatformServiceProvider.Instance.FileOperations.DoesFileExist(fullFilePath)) { - var message = string.Format( - CultureInfo.CurrentCulture, - Resource.TestAssembly_FileDoesNotExist, - fullFilePath); + var message = string.Format(CultureInfo.CurrentCulture, Resource.TestAssembly_FileDoesNotExist, fullFilePath); throw new FileNotFoundException(message); } - if (!PlatformServiceProvider.Instance.TestSource.IsAssemblyReferenced( - UnitTestFrameworkAssemblyName, - fullFilePath)) + if (!PlatformServiceProvider.Instance.TestSource.IsAssemblyReferenced(UnitTestFrameworkAssemblyName, fullFilePath)) { return null; } @@ -71,33 +59,18 @@ internal ICollection GetTests( } catch (FileNotFoundException ex) { - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "MSTestDiscoverer.TryGetTests: Failed to discover tests from {0}. Reason:{1}", - fullFilePath, - ex); - - var message = string.Format( - CultureInfo.CurrentCulture, - Resource.TestAssembly_AssemblyDiscoveryFailure, - fullFilePath, - ex.Message); + var message = string.Format(CultureInfo.CurrentCulture, Resource.TestAssembly_AssemblyDiscoveryFailure, fullFilePath, ex.Message); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"MSTestDiscoverer.TryGetTests: {Resource.TestAssembly_AssemblyDiscoveryFailure}", fullFilePath, ex); warnings.Add(message); return null; } catch (ReflectionTypeLoadException ex) { - var message = string.Format( - CultureInfo.CurrentCulture, - Resource.TestAssembly_AssemblyDiscoveryFailure, - fullFilePath, - ex.Message); + var message = string.Format(CultureInfo.CurrentCulture, Resource.TestAssembly_AssemblyDiscoveryFailure, fullFilePath, ex.Message); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"MSTestDiscoverer.TryGetTests: {Resource.TestAssembly_AssemblyDiscoveryFailure}", fullFilePath, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.ExceptionsThrown); warnings.Add(message); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "MSPhoneTestDiscoverer.TryGetTests: Failed to discover tests from {0}. Reason:{1}", - assemblyFileName, - ex); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("Exceptions thrown from the Loader :"); if (ex.LoaderExceptions != null) { @@ -120,17 +93,9 @@ internal ICollection GetTests( // Discover test doesn't work if there is a managed C++ project in solution // Assembly.Load() fails to load the managed cpp executable, with FileLoadException. It can load the dll // successfully though. This is known CLR issue. - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "MSTestDiscoverer.TryGetTests: Failed to discover tests from {0}. Reason:{1}", - assemblyFileName, - ex); - var message = ex is FileNotFoundException fileNotFoundEx - ? fileNotFoundEx.Message - : string.Format( - CultureInfo.CurrentCulture, - Resource.TestAssembly_AssemblyDiscoveryFailure, - fullFilePath, - ex.Message); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"MSTestDiscoverer.TryGetTests: {Resource.TestAssembly_AssemblyDiscoveryFailure}", fullFilePath, ex); + var message = ex is FileNotFoundException fileNotFoundEx ? fileNotFoundEx.Message : string.Format(CultureInfo.CurrentCulture, Resource.TestAssembly_AssemblyDiscoveryFailure, fullFilePath, ex.Message); + warnings.Add(message); return null; } @@ -141,8 +106,17 @@ private ICollection GetTestsInIsolation(string fullFilePath, IR using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(fullFilePath, runSettings, frameworkHandle: null)) { // Create an instance of a type defined in adapter so that adapter gets loaded in the child app domain - var assemblyEnumerator = isolationHost.CreateInstanceForType( - typeof(AssemblyEnumerator), new object[] { MSTestSettings.CurrentSettings }) as AssemblyEnumerator; + var assemblyEnumerator = isolationHost.CreateInstanceForType(typeof(AssemblyEnumerator), new object[] { MSTestSettings.CurrentSettings }) as AssemblyEnumerator; + + // This might not be supported if an older version of "PlatformServices + try + { + assemblyEnumerator.RunSettingsXml = runSettings?.SettingsXml; + } + catch + { + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.OlderTFMVersionFound); + } return assemblyEnumerator.EnumerateAssembly(fullFilePath, out warnings); } diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs index f670606148..ffb48f3a9d 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs @@ -49,7 +49,6 @@ internal TypeEnumerator(Type type, string assemblyName, ReflectHelper reflectHel /// /// Contains warnings if any, that need to be passed back to the caller. /// list of test cases. - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1021:AvoidOutParameters", MessageId = "0#", Justification = "This is only for internal use.")] internal virtual ICollection Enumerate(out ICollection warnings) { warnings = new Collection(); @@ -199,6 +198,8 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT testElement.WorkItemIds = workItemAttributes.Select(x => x.Id.ToString()).ToArray(); } + testElement.Ignored = this.reflectHelper.IsAttributeDefined(method, typeof(IgnoreAttribute), false); + // Get Deployment items if any. testElement.DeploymentItems = PlatformServiceProvider.Instance.TestDeployment.GetDeploymentItems( method, diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs index fd918bd17d..d89d2bc597 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs @@ -11,10 +11,12 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution using System.Reflection; using System.Text; using System.Threading; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestTools.UnitTesting; + using UnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; using UTF = Microsoft.VisualStudio.TestTools.UnitTesting; @@ -91,31 +93,7 @@ public Attribute[] GetAllAttributes(bool inherit) } public TAttributeType[] GetAttributes(bool inherit) - where TAttributeType : Attribute - { - Attribute[] attributeArray = ReflectHelper.GetCustomAttributes(this.TestMethod, typeof(TAttributeType), inherit); - - TAttributeType[] tAttributeArray = attributeArray as TAttributeType[]; - if (tAttributeArray != null) - { - return tAttributeArray; - } - - List tAttributeList = new List(); - if (attributeArray != null) - { - foreach (Attribute attribute in attributeArray) - { - TAttributeType tAttribute = attribute as TAttributeType; - if (tAttribute != null) - { - tAttributeList.Add(tAttribute); - } - } - } - - return tAttributeList.ToArray(); - } + where TAttributeType : Attribute => ReflectHelper.GetAttributes(this.TestMethod, inherit); /// /// Execute test method. Capture failures, handle async and return result. diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index 30338d7c42..d8727d1dda 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -10,10 +10,13 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution using System.Globalization; using System.Linq; using System.Reflection; + using Extensions; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; + using UTF = Microsoft.VisualStudio.TestTools.UnitTesting; /// @@ -217,136 +220,29 @@ internal UnitTestResult[] RunTestMethod() List results = new List(); var isDataDriven = false; - // Parent result. Added in properties bag only when results are greater than 1. - var parentResultWatch = new Stopwatch(); - parentResultWatch.Start(); - var parentResult = new UTF.TestResult - { - Outcome = UTF.UnitTestOutcome.InProgress, - ExecutionId = Guid.NewGuid() - }; - if (this.testMethodInfo.TestMethodOptions.Executor != null) { - UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); - if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) + if (this.test.DataType == DynamicDataType.ITestDataSource) + { + var testResults = this.ExecuteTestWithDataSource(null, this.test.Data); + results.AddRange(testResults); + } + else if (this.ExecuteDataSourceBasedTests(results)) { isDataDriven = true; - Stopwatch watch = new Stopwatch(); - watch.Start(); - - try - { - IEnumerable dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); - - if (dataRows == null) - { - watch.Stop(); - var inconclusiveResult = new UTF.TestResult(); - inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; - inconclusiveResult.Duration = watch.Elapsed; - results.Add(inconclusiveResult); - } - else - { - try - { - int rowIndex = 0; - foreach (object dataRow in dataRows) - { - watch.Reset(); - watch.Start(); - - this.testContext.SetDataRow(dataRow); - UTF.TestResult[] testResults; - - try - { - testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); - } - catch (Exception ex) - { - testResults = new[] - { - new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } - }; - } - - watch.Stop(); - foreach (var testResult in testResults) - { - testResult.DatarowIndex = rowIndex; - testResult.Duration = watch.Elapsed; - } - - rowIndex++; - - results.AddRange(testResults); - } - } - finally - { - this.testContext.SetDataConnection(null); - this.testContext.SetDataRow(null); - } - } - } - catch (Exception ex) - { - watch.Stop(); - var failedResult = new UTF.TestResult(); - failedResult.Outcome = UTF.UnitTestOutcome.Error; - failedResult.TestFailureException = ex; - failedResult.Duration = watch.Elapsed; - results.Add(failedResult); - } } else { - UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); - - if (testDataSources != null && testDataSources.Length > 0) + var testResults = this.ExecuteTest(); + foreach (var testResult in testResults) { - isDataDriven = true; - foreach (var testDataSource in testDataSources) + if (string.IsNullOrWhiteSpace(testResult.DisplayName)) { - foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) - { - this.testMethodInfo.SetArguments(data); - UTF.TestResult[] testResults; - try - { - testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); - } - catch (Exception ex) - { - testResults = new[] - { - new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } - }; - } - - foreach (var testResult in testResults) - { - testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); - } - - results.AddRange(testResults); - this.testMethodInfo.SetArguments(null); - } - } - } - else - { - try - { - results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); - } - catch (Exception ex) - { - results.Add(new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) }); + testResult.DisplayName = this.test.DisplayName; } } + + results.AddRange(testResults); } } else @@ -357,9 +253,6 @@ internal UnitTestResult[] RunTestMethod() this.testMethodInfo.TestMethodName); } - parentResultWatch.Stop(); - parentResult.Duration = parentResultWatch.Elapsed; - // Get aggregate outcome. var aggregateOutcome = this.GetAggregateOutcome(results); this.testContext.SetOutcome(aggregateOutcome); @@ -373,13 +266,160 @@ internal UnitTestResult[] RunTestMethod() // In case of data driven, set parent info in results. if (isDataDriven) { - parentResult.Outcome = aggregateOutcome; - results = this.UpdateResultsWithParentInfo(results, parentResult); + results = this.UpdateResultsWithParentInfo(results, Guid.NewGuid()); } return results.ToArray().ToUnitTestResults(); } + private bool ExecuteDataSourceBasedTests(List results) + { + var isDataDriven = false; + + UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); + if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) + { + isDataDriven = true; + Stopwatch watch = new Stopwatch(); + watch.Start(); + + try + { + IEnumerable dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); + + if (dataRows == null) + { + watch.Stop(); + var inconclusiveResult = new UTF.TestResult(); + inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; + inconclusiveResult.Duration = watch.Elapsed; + results.Add(inconclusiveResult); + } + else + { + try + { + int rowIndex = 0; + + foreach (object dataRow in dataRows) + { + UTF.TestResult[] testResults = this.ExecuteTestWithDataRow(dataRow, rowIndex++); + results.AddRange(testResults); + } + } + finally + { + this.testContext.SetDataConnection(null); + this.testContext.SetDataRow(null); + } + } + } + catch (Exception ex) + { + watch.Stop(); + var failedResult = new UTF.TestResult(); + failedResult.Outcome = UTF.UnitTestOutcome.Error; + failedResult.TestFailureException = ex; + failedResult.Duration = watch.Elapsed; + results.Add(failedResult); + } + } + else + { + UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + + if (testDataSources != null && testDataSources.Length > 0) + { + isDataDriven = true; + foreach (var testDataSource in testDataSources) + { + foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) + { + try + { + var testResults = this.ExecuteTestWithDataSource(testDataSource, data); + + results.AddRange(testResults); + } + finally + { + this.testMethodInfo.SetArguments(null); + } + } + } + } + } + + return isDataDriven; + } + + private UTF.TestResult[] ExecuteTestWithDataSource(UTF.ITestDataSource testDataSource, object[] data) + { + var stopwatch = Stopwatch.StartNew(); + + this.testMethodInfo.SetArguments(data); + var testResults = this.ExecuteTest(); + stopwatch.Stop(); + + var hasDisplayName = !string.IsNullOrWhiteSpace(this.test.DisplayName); + foreach (var testResult in testResults) + { + if (testResult.Duration == TimeSpan.Zero) + { + testResult.Duration = stopwatch.Elapsed; + } + + var displayName = this.test.Name; + if (testDataSource != null) + { + displayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); + } + else if (hasDisplayName) + { + displayName = this.test.DisplayName; + } + + testResult.DisplayName = displayName; + } + + return testResults; + } + + private UTF.TestResult[] ExecuteTestWithDataRow(object dataRow, int rowIndex) + { + var stopwatch = Stopwatch.StartNew(); + + this.testContext.SetDataRow(dataRow); + var testResults = this.ExecuteTest(); + stopwatch.Stop(); + + var displayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, this.test.DisplayName, rowIndex); + + foreach (var testResult in testResults) + { + testResult.DisplayName = displayName; + testResult.DatarowIndex = rowIndex; + testResult.Duration = stopwatch.Elapsed; + } + + return testResults; + } + + private UTF.TestResult[] ExecuteTest() + { + try + { + return this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); + } + catch (Exception ex) + { + return new[] + { + new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex.Message), ex) } + }; + } + } + /// /// Gets aggregate outcome. /// @@ -408,9 +448,9 @@ private UTF.UnitTestOutcome GetAggregateOutcome(List results) /// Add parent results as first result in updated result. /// /// Results. - /// Parent results. + /// Current execution id. /// Updated results which contains parent result as first result. All other results contains parent result info. - private List UpdateResultsWithParentInfo(List results, UTF.TestResult parentResult) + private List UpdateResultsWithParentInfo(List results, Guid executionId) { // Return results in case there are no results. if (!results.Any()) @@ -420,13 +460,11 @@ private UTF.UnitTestOutcome GetAggregateOutcome(List results) // UpdatedResults contain parent result at first position and remaining results has parent info updated. var updatedResults = new List(); - updatedResults.Add(parentResult); foreach (var result in results) { result.ExecutionId = Guid.NewGuid(); - result.ParentExecId = parentResult.ExecutionId; - parentResult.InnerResultsCount++; + result.ParentExecId = executionId; updatedResults.Add(result); } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs index f5d4564733..074845f5fc 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs @@ -323,7 +323,6 @@ private PropertyInfo ResolveTestContext(Type classType) /// /// The type. /// The instance. - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Discoverer should continue with remaining sources.")] private TestAssemblyInfo GetAssemblyInfo(Type type) { var assembly = type.GetTypeInfo().Assembly; diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs index 2a11160ac4..3443b01454 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs @@ -3,8 +3,10 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions { + using System.Collections.Generic; + using System.Linq; + using Microsoft.TestPlatform.AdapterUtilities; - using Microsoft.TestPlatform.AdapterUtilities.ManagedNameUtilities; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -16,8 +18,8 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions internal static class TestCaseExtensions { internal static readonly TestProperty ManagedTypeProperty = TestProperty.Register( - id: Contants.ManagedTypePropertyId, - label: Contants.ManagedTypeLabel, + id: ManagedNameConstants.ManagedTypePropertyId, + label: ManagedNameConstants.ManagedTypeLabel, category: string.Empty, description: string.Empty, valueType: typeof(string), @@ -26,8 +28,8 @@ internal static class TestCaseExtensions owner: typeof(TestCase)); internal static readonly TestProperty ManagedMethodProperty = TestProperty.Register( - id: Contants.ManagedMethodPropertyId, - label: Contants.ManagedMethodLabel, + id: ManagedNameConstants.ManagedMethodPropertyId, + label: ManagedNameConstants.ManagedMethodLabel, category: string.Empty, description: string.Empty, valueType: typeof(string), @@ -64,6 +66,17 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string testMethod = new TestMethod(name, testClassName, source, isAsync); } + var dataType = (DynamicDataType)testCase.GetPropertyValue(Constants.TestDynamicDataTypeProperty, (int)DynamicDataType.None); + if (dataType != DynamicDataType.None) + { + var data = testCase.GetPropertyValue(Constants.TestDynamicDataProperty, null); + + testMethod.DataType = dataType; + testMethod.Data = Helpers.DataSerializationHelper.Deserialze(data); + } + + testMethod.DisplayName = testCase.DisplayName; + if (declaringClassName != null && declaringClassName != testClassName) { testMethod.DeclaringClassFullName = declaringClassName; @@ -77,6 +90,43 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string DisplayName = testCase.DisplayName }; + if (testCase.Traits.Any()) + { + testElement.Traits = testCase.Traits.ToArray(); + } + + var cssIteration = testCase.GetPropertyValue(Constants.CssIterationProperty, null); + if (!string.IsNullOrWhiteSpace(cssIteration)) + { + testElement.CssIteration = cssIteration; + } + + var cssProjectStructure = testCase.GetPropertyValue(Constants.CssProjectStructureProperty, null); + if (!string.IsNullOrWhiteSpace(cssIteration)) + { + testElement.CssProjectStructure = cssProjectStructure; + } + + var description = testCase.GetPropertyValue(Constants.DescriptionProperty, null); + if (!string.IsNullOrWhiteSpace(description)) + { + testElement.Description = description; + } + + var workItemIds = testCase.GetPropertyValue(Constants.WorkItemIdsProperty, null); + if (workItemIds != null && workItemIds.Length > 0) + { + testElement.WorkItemIds = workItemIds; + } + + var deploymentItems = testCase.GetPropertyValue[]>(Constants.DeploymentItemsProperty, null); + if (deploymentItems != null && deploymentItems.Length > 0) + { + testElement.DeploymentItems = deploymentItems; + } + + testElement.DoNotParallelize = testCase.GetPropertyValue(Constants.DoNotParallelizeProperty, false); + return testElement; } diff --git a/src/Adapter/MSTest.CoreAdapter/Friends.cs b/src/Adapter/MSTest.CoreAdapter/Friends.cs index fede71cd63..fd34b61095 100644 --- a/src/Adapter/MSTest.CoreAdapter/Friends.cs +++ b/src/Adapter/MSTest.CoreAdapter/Friends.cs @@ -4,6 +4,7 @@ // Friend assemblies using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] -[assembly: InternalsVisibleTo("MSTestAdapter.Smoke.E2ETests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] \ No newline at end of file +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo(assemblyName: "DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] +[assembly: InternalsVisibleTo(assemblyName: "MSTestAdapter.Smoke.E2ETests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo(assemblyName: "DiscoveryAndExecutionTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] \ No newline at end of file diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs new file mode 100644 index 0000000000..2f8aad98bb --- /dev/null +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers +{ + using System.IO; + using System.Runtime.Serialization.Json; + using System.Text; + + internal static class DataSerializationHelper + { + /// + /// Serializes the date in such a way that won't throw exceptions during deserialization in Test Platform. + /// The result can be deserialized using method. + /// + /// Data array to serialize. + /// Serialzed array. + public static string[] Serialize(object[] data) + { + if (data == null) + { + return null; + } + + var serializer = GetSerializer(); + var serializedData = new string[data.Length]; + + for (int i = 0; i < data.Length; i++) + { + if (data[i] == null) + { + serializedData[i] = null; + continue; + } + + using (var memoryStream = new MemoryStream()) + { + serializer.WriteObject(memoryStream, data[i]); + var serializerData = memoryStream.ToArray(); + + serializedData[i] = Encoding.UTF8.GetString(serializerData, 0, serializerData.Length); + } + } + + return serializedData; + } + + /// + /// Deserialzes the data serialzed by method. + /// + /// Serialized data array to deserialize. + /// Deserialized array. + public static object[] Deserialze(string[] serializedData) + { + if (serializedData == null) + { + return null; + } + + var serializer = GetSerializer(); + var data = new object[serializedData.Length]; + + for (int i = 0; i < serializedData.Length; i++) + { + if (serializedData[i] == null) + { + data[i] = null; + continue; + } + + var serialzedDataBytes = Encoding.UTF8.GetBytes(serializedData[i]); + using (var memoryStream = new MemoryStream(serialzedDataBytes)) + { + data[i] = serializer.ReadObject(memoryStream); + } + } + + return data; + } + + private static DataContractJsonSerializer GetSerializer() + { + var settings = new DataContractJsonSerializerSettings() + { + UseSimpleDictionaryFormat = true, + EmitTypeInformation = System.Runtime.Serialization.EmitTypeInformation.Always + }; + + var serializer = new DataContractJsonSerializer(typeof(object), settings); + + return serializer; + } + } +} diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs index 22224a4058..a2c4493e39 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs @@ -113,7 +113,7 @@ public bool HasAttributeDerivedFrom(MemberInfo memberInfo, Type baseAttributeTyp Dictionary attributes = this.GetAttributes(memberInfo, inherit); if (attributes == null) { - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("ReflectHelper.HasAttributeDerivedFrom: Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data."); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"ReflectHelper.HasAttributeDerivedFrom: {Resource.FailedFetchAttributeCache}"); return this.IsAttributeDefined(memberInfo, baseAttributeType, inherit); } @@ -200,6 +200,20 @@ public override object InitializeLifetimeService() return null; } + internal static T[] GetAttributes(MethodBase methodBase, bool inherit) + where T : Attribute + { + Attribute[] attributeArray = GetCustomAttributes(methodBase, typeof(T), inherit); + + var attributes = attributeArray as T[]; + if (attributes != null) + { + return attributes; + } + + return attributeArray?.Select(a => a as T)?.Where(a => a != null)?.ToArray(); + } + /// /// Match return type of method. /// @@ -629,7 +643,6 @@ private IEnumerable GetTestPropertyAttributes(MemberInfo propertyAttr /// The member to inspect. /// Look at inheritance chain. /// attributes defined. - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] private Dictionary GetAttributes(MemberInfo memberInfo, bool inherit) { // If the information is cached, then use it otherwise populate the cache using @@ -658,16 +671,10 @@ private Dictionary GetAttributes(MemberInfo memberInfo, bool inh } catch (Exception ex2) { - description = - ex.GetType().FullName + - ": (Failed to get exception description due to an exception of type " + - ex2.GetType().FullName + ')'; + description = string.Format(CultureInfo.CurrentCulture, Resource.ExceptionOccuredWhileGettingTheExceptionDescription, ex.GetType().FullName, ex2.GetType().FullName); // ex.GetType().FullName + } - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1}", - memberInfo.GetType().FullName, - description); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.FailedToGetCustomAttribute, memberInfo.GetType().FullName, description); // Since we cannot check by attribute names, do it in reflection way. // Note 1: this will not work for different version of assembly but it is better than nothing. diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/RunSettingsUtilities.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/RunSettingsUtilities.cs index 646effd838..d5ebb0db98 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/RunSettingsUtilities.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/RunSettingsUtilities.cs @@ -5,11 +5,12 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers { using System; using System.Collections.Generic; - using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Xml; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + using TestPlatform.ObjectModel; internal class RunSettingsUtilities @@ -36,7 +37,7 @@ internal static XmlReaderSettings ReaderSettings /// If there is no test run parameters section defined in the settingsxml a blank dictionary is returned. internal static Dictionary GetTestRunParameters(string settingsXml) { - var nodeValue = GetNodeValue>(settingsXml, TestAdapter.Constants.TestRunParametersName, TestRunParameters.FromXml); + var nodeValue = GetNodeValue(settingsXml, TestAdapter.Constants.TestRunParametersName, TestRunParameters.FromXml); if (nodeValue == default(Dictionary)) { // Return default. @@ -65,37 +66,38 @@ internal static void ThrowOnHasAttributes(XmlReader reader) } } - [SuppressMessage("Microsoft.Security.Xml", "CA3053:UseXmlSecureResolver", Justification = "XmlReaderSettings.XmlResolver is not available in portable code.")] private static T GetNodeValue(string settingsXml, string nodeName, Func nodeParser) { + if (string.IsNullOrWhiteSpace(settingsXml)) + { + return default(T); + } + // use XmlReader to avoid loading of the plugins in client code (mainly from VS). - if (!string.IsNullOrWhiteSpace(settingsXml)) + using (StringReader stringReader = new StringReader(settingsXml)) { - using (StringReader stringReader = new StringReader(settingsXml)) - { - XmlReader reader = XmlReader.Create(stringReader, ReaderSettings); + XmlReader reader = XmlReader.Create(stringReader, ReaderSettings); - // read to the fist child - XmlReaderUtilities.ReadToRootNode(reader); - reader.ReadToNextElement(); + // read to the fist child + XmlReaderUtilities.ReadToRootNode(reader); + reader.ReadToNextElement(); - // Read till we reach nodeName element or reach EOF - while (!string.Equals(reader.Name, nodeName, StringComparison.OrdinalIgnoreCase) - && - !reader.EOF) - { - reader.SkipToNextElement(); - } + // Read till we reach nodeName element or reach EOF + while (!string.Equals(reader.Name, nodeName, StringComparison.OrdinalIgnoreCase) + && + !reader.EOF) + { + reader.SkipToNextElement(); + } - if (!reader.EOF) - { - // read nodeName element. - return nodeParser(reader); - } + if (!reader.EOF) + { + // read nodeName element. + return nodeParser(reader); } } return default(T); } } -} +} \ No newline at end of file diff --git a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj index f020861623..66c6afc6c0 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj +++ b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj @@ -38,6 +38,7 @@ + @@ -54,6 +55,7 @@ + diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index c99d9e7343..351491782b 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -167,6 +167,11 @@ private set /// The existing settings object. public static void PopulateSettings(MSTestSettings settings) { + if (settings == null) + { + return; + } + CurrentSettings.CaptureDebugTraces = settings.CaptureDebugTraces; CurrentSettings.ForcedLegacyMode = settings.ForcedLegacyMode; CurrentSettings.TestSettingsFile = settings.TestSettingsFile; @@ -240,12 +245,17 @@ public static bool IsLegacyScenario(IMessageLogger logger) /// /// Gets the adapter specific settings from the xml. /// - /// The xml with the settings passed from the test platform. + /// The xml with the settings passed from the test platform. /// The name of the adapter settings to fetch - Its either MSTest or MSTestV2 /// The settings if found. Null otherwise. - internal static MSTestSettings GetSettings(string runsettingsXml, string settingName) + internal static MSTestSettings GetSettings(string runSettingsXml, string settingName) { - using (var stringReader = new StringReader(runsettingsXml)) + if (string.IsNullOrWhiteSpace(runSettingsXml)) + { + return null; + } + + using (var stringReader = new StringReader(runSettingsXml)) { XmlReader reader = XmlReader.Create(stringReader, XmlRunSettingsUtilities.ReaderSettings); diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs new file mode 100644 index 0000000000..b1b82712c0 --- /dev/null +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel +{ + internal enum DynamicDataType : int + { + None = 0, + DataSourceAttribute = 1, + ITestDataSource = 2 + } +} diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs index 9251564a4d..4190a5ba22 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs @@ -56,14 +56,8 @@ internal TestMethod(MethodBase method, string name, string fullClassName, string } ManagedNameHelper.GetManagedName(method, out var managedType, out var managedMethod); - - // ManagedNameHelpers currently does not support spaces in method names. - // If there are spaces in the method name, we'll use the legacy way to find the method. - if (!managedMethod.Contains(" ")) - { - this.ManagedTypeName = managedType; - this.ManagedMethodName = managedMethod; - } + this.ManagedTypeName = managedType; + this.ManagedMethodName = managedMethod; } internal TestMethod(string managedTypeName, string managedMethodName, string name, string fullClassName, string assemblyName, bool isAsync) @@ -73,18 +67,14 @@ internal TestMethod(string managedTypeName, string managedMethodName, string nam this.ManagedMethodName = managedMethodName; } - /// - /// Gets the name of the test method - /// + /// public string Name { get; private set; } - /// - /// Gets the full classname of the test method - /// + /// public string FullClassName { get; private set; } /// - /// Gets or sets the declaring class full name. This will be used while getting navigation data. + /// Gets or sets the declaring assembly full name. This will be used while getting navigation data. /// This will be null if AssemblyName is same as DeclaringAssemblyName. /// Reason to set to null in the above case is to minimize the transfer of data across appdomains and not have a performance hit. /// @@ -122,14 +112,10 @@ public string DeclaringClassFullName } } - /// - /// Gets the name of the test assembly - /// + /// public string AssemblyName { get; private set; } - /// - /// Gets a value indicating whether specifies test method is async - /// + /// public bool IsAsync { get; private set; } /// @@ -140,5 +126,22 @@ public string DeclaringClassFullName /// public bool HasManagedMethodAndTypeProperties => !string.IsNullOrWhiteSpace(this.ManagedTypeName) && !string.IsNullOrWhiteSpace(this.ManagedMethodName); + + /// + /// Gets or sets type of dynamic data if any + /// + internal DynamicDataType DataType { get; set; } + + /// + /// Gets or sets indices of dynamic data + /// + internal object[] Data { get; set; } + + /// + /// Gets or sets the display name set during discovery + /// + internal string DisplayName { get; set; } + + internal TestMethod Clone() => this.MemberwiseClone() as TestMethod; } } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs index 3946c7026e..f4309ba974 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs @@ -7,14 +7,17 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel using System.Collections.Generic; using System.Diagnostics; using System.Globalization; + using System.IO; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; /// /// The unit test element. /// [Serializable] + [DebuggerDisplay("{GetDisplayName()} ({TestMethod.ManagedTypeName})")] internal class UnitTestElement { /// @@ -103,15 +106,28 @@ public UnitTestElement(TestMethod testMethod) /// internal string[] WorkItemIds { get; set; } + internal UnitTestElement Clone() + { + var clone = this.MemberwiseClone() as UnitTestElement; + if (this.TestMethod != null) + { + clone.TestMethod = this.TestMethod.Clone(); + } + + return clone; + } + /// /// Convert the UnitTestElement instance to an Object Model testCase instance. /// /// An instance of . internal TestCase ToTestCase() { - string fullName = this.TestMethod.HasManagedMethodAndTypeProperties - ? string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.TestMethod.ManagedTypeName, this.TestMethod.ManagedMethodName) - : string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.TestMethod.FullClassName, this.TestMethod.Name); + // This causes compatibility problems with older runners. + // string fullName = this.TestMethod.HasManagedMethodAndTypeProperties + // ? string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.TestMethod.ManagedTypeName, this.TestMethod.ManagedMethodName) + // : string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.TestMethod.FullClassName, this.TestMethod.Name); + var fullName = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.TestMethod.FullClassName, this.TestMethod.Name); TestCase testCase = new TestCase(fullName, TestAdapter.Constants.ExecutorUri, this.TestMethod.AssemblyName); testCase.DisplayName = this.GetDisplayName(); @@ -188,6 +204,41 @@ internal TestCase ToTestCase() testCase.SetPropertyValue(TestAdapter.Constants.DoNotParallelizeProperty, this.DoNotParallelize); } + // Store resolved data if any + if (this.TestMethod.DataType != DynamicDataType.None) + { + var data = Helpers.DataSerializationHelper.Serialize(this.TestMethod.Data); + + testCase.SetPropertyValue(TestAdapter.Constants.TestDynamicDataTypeProperty, (int)this.TestMethod.DataType); + testCase.SetPropertyValue(TestAdapter.Constants.TestDynamicDataProperty, data); + } + + string fileName = testCase.Source; + try + { + fileName = Path.GetFileName(fileName); + } + catch + { + } + + var testFullId = testCase.ExecutorUri?.ToString() + fileName; + if (this.TestMethod.HasManagedMethodAndTypeProperties) + { + testFullId += $"{this.TestMethod.ManagedTypeName}{this.TestMethod.ManagedMethodName}"; + } + else + { + testFullId += testCase.FullyQualifiedName; + } + + if (this.TestMethod.DataType != DynamicDataType.None) + { + testFullId += testCase.DisplayName; + } + + testCase.Id = EqtHash.GuidFromString(testFullId); + return testCase; } @@ -195,9 +246,12 @@ private string GetDisplayName() { if (string.IsNullOrWhiteSpace(this.DisplayName)) { - return string.IsNullOrWhiteSpace(this.TestMethod.ManagedMethodName) - ? this.TestMethod.Name - : this.TestMethod.ManagedMethodName; + return this.TestMethod.Name; + + // This causes compatibility problems with older runners. + // return string.IsNullOrWhiteSpace(this.TestMethod.ManagedMethodName) + // ? this.TestMethod.Name + // : this.TestMethod.ManagedMethodName; } else { diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs index 24482a8b5a..c2a4f51a32 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestResult.cs @@ -14,6 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel using Constants = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Constants; [Serializable] + [DebuggerDisplay("{DisplayName} ({Outcome})")] public class UnitTestResult { /// @@ -154,14 +155,14 @@ internal TestResult ToTestResult(TestCase testCase, DateTimeOffset startTime, Da Debug.Assert(testCase != null, "testCase"); var testResult = new TestResult(testCase) - { - DisplayName = this.DisplayName, - Duration = this.Duration, - ErrorMessage = this.ErrorMessage, - ErrorStackTrace = this.ErrorStackTrace, - Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings), - StartTime = startTime, - EndTime = endTime + { + DisplayName = this.DisplayName, + Duration = this.Duration, + ErrorMessage = this.ErrorMessage, + ErrorStackTrace = this.ErrorStackTrace, + Outcome = UnitTestOutcomeHelper.ToTestOutcome(this.Outcome, currentSettings), + StartTime = startTime, + EndTime = endTime }; testResult.SetPropertyValue(Constants.ExecutionIdProperty, this.ExecutionId); diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs index 4724a2c692..4a240891e7 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs @@ -20,7 +20,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resource { @@ -70,6 +70,33 @@ internal static string AttachmentSetDisplayName { } } + /// + /// Looks up a localized string similar to Exception occurred while enumarating DataSourceAttribute on "{0}.{1}": {2}. + /// + internal static string CannotEnumarateDataSourceAttribute { + get { + return ResourceManager.GetString("CannotEnumarateDataSourceAttribute", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to A test method can only contain one DataSourceAttribute, but found {2} on "{0}.{1}".. + /// + internal static string CannotEnumarateDataSourceAttribute_MoreThenOneDefined { + get { + return ResourceManager.GetString("CannotEnumarateDataSourceAttribute_MoreThenOneDefined", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Exception occurred while enumarating IDataSource attribute on "{0}.{1}": {2}. + /// + internal static string CannotEnumarateIDataSourceAttribute { + get { + return ResourceManager.GetString("CannotEnumarateIDataSourceAttribute", resourceCulture); + } + } + /// /// Looks up a localized string similar to The parameter should not be null or empty.. /// @@ -142,6 +169,24 @@ internal static string EnumeratorLoadTypeErrorFormat { } } + /// + /// Looks up a localized string similar to "{0}": (Failed to get exception description due to an exception of type "{1}".. + /// + internal static string ExceptionOccuredWhileGettingTheExceptionDescription { + get { + return ResourceManager.GetString("ExceptionOccuredWhileGettingTheExceptionDescription", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Exceptions thrown:. + /// + internal static string ExceptionsThrown { + get { + return ResourceManager.GetString("ExceptionsThrown", resourceCulture); + } + } + /// /// Looks up a localized string similar to Test '{0}' execution has been aborted.. /// @@ -160,6 +205,24 @@ internal static string Execution_Test_Timeout { } } + /// + /// Looks up a localized string similar to Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data.. + /// + internal static string FailedFetchAttributeCache { + get { + return ResourceManager.GetString("FailedFetchAttributeCache", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1}. + /// + internal static string FailedToGetCustomAttribute { + get { + return ResourceManager.GetString("FailedToGetCustomAttribute", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid value '{0}' specified for 'Scope'. Supported scopes are {1}.. /// @@ -205,6 +268,15 @@ internal static string LegacyScenariosNotSupportedWarning { } } + /// + /// Looks up a localized string similar to An older version of MSTestV2 package is loaded in asssembly, test discovery might fail to discover all data tests if they depend on `.runsettings` file.. + /// + internal static string OlderTFMVersionFound { + get { + return ResourceManager.GetString("OlderTFMVersionFound", resourceCulture); + } + } + /// /// Looks up a localized string similar to Running tests in any of the provided sources is not supported for the selected platform. /// diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx index 474f4ed5d0..176efd759d 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx @@ -320,4 +320,42 @@ Error: {1} Test '{0}' execution has been aborted. + + Exception occurred while enumarating DataSourceAttribute on "{0}.{1}": {2} + {0}: TypeName with namespace, +{1}: Method name, +{2}: Exception details + + + A test method can only contain one DataSourceAttribute, but found {2} on "{0}.{1}". + {0}: TypeName with namespace, +{1}: Method name, +{2}: Number of attributed defined. + + + Exception occurred while enumarating IDataSource attribute on "{0}.{1}": {2} + {0}: TypeName with namespace, +{1}: Method name, +{2}: Exception details + + + "{0}": (Failed to get exception description due to an exception of type "{1}". + {0}: Type of the original exception that we're trying to get the desciption of. +{1}: Thrown exception + + + Exceptions thrown: + This is usually preceeds by TestAssembly_AssemblyDiscoveryFailure message, and precceded by list of exceptions thrown in a test discovery session. + + + Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data. + + + Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1} + {0}: Attribute full type name. +{1}: Exception description + + + An older version of MSTestV2 package is loaded in asssembly, test discovery might fail to discover all data tests if they depend on `.runsettings` file. + \ No newline at end of file diff --git a/src/Adapter/PlatformServices.Desktop/packages.config b/src/Adapter/PlatformServices.Desktop/packages.config index fc97748e33..a0cd2b7c55 100644 --- a/src/Adapter/PlatformServices.Desktop/packages.config +++ b/src/Adapter/PlatformServices.Desktop/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs b/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs index 47e10c8a1d..56a7f01433 100644 --- a/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs +++ b/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs @@ -19,8 +19,9 @@ public interface ITestMethod string FullClassName { get; } /// - /// Gets the declaring class full name. - /// This will be used for resolving overloads and while getting navigation data. + /// Gets the declaring class full name. This will be used while getting navigation data. + /// This will be null if AssemblyName is same as DeclaringAssemblyName. + /// Reason to set to null in the above case is to minimize the transfer of data across appdomains and not have a performance hit. /// string DeclaringClassFullName { get; } diff --git a/src/Adapter/PlatformServices.Interface/packages.config b/src/Adapter/PlatformServices.Interface/packages.config index 477f795ab9..455e5c4a11 100644 --- a/src/Adapter/PlatformServices.Interface/packages.config +++ b/src/Adapter/PlatformServices.Interface/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj b/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj index fe1ba4baa9..cedc665c7c 100644 --- a/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj +++ b/src/Adapter/PlatformServices.NetCore/PlatformServices.NetCore.csproj @@ -78,8 +78,5 @@ - - - \ No newline at end of file diff --git a/src/Adapter/PlatformServices.Portable/packages.config b/src/Adapter/PlatformServices.Portable/packages.config index 477f795ab9..455e5c4a11 100644 --- a/src/Adapter/PlatformServices.Portable/packages.config +++ b/src/Adapter/PlatformServices.Portable/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems new file mode 100644 index 0000000000..ca1cde6378 --- /dev/null +++ b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems @@ -0,0 +1,14 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + true + 2177c273-ae07-43b3-b87a-443e47a23c5a + + + UnitTestFramework.Extension.Shared + + + + + \ No newline at end of file diff --git a/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.shproj b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.shproj new file mode 100644 index 0000000000..92805076c8 --- /dev/null +++ b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.shproj @@ -0,0 +1,13 @@ + + + + 2177c273-ae07-43b3-b87a-443e47a23c5a + 14.0 + + + + + + + + diff --git a/src/TestFramework/Extension.Shared/Extension.Shared.shproj b/src/TestFramework/Extension.Shared/Extension.Shared.shproj index 9f7ebd4c48..da99638394 100644 --- a/src/TestFramework/Extension.Shared/Extension.Shared.shproj +++ b/src/TestFramework/Extension.Shared/Extension.Shared.shproj @@ -7,7 +7,7 @@ - + diff --git a/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config b/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config index e778835012..5930306458 100644 --- a/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config +++ b/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/ComponentTests/TestAssets/Directory.Build.targets b/test/ComponentTests/TestAssets/Directory.Build.targets new file mode 100644 index 0000000000..87043c6abb --- /dev/null +++ b/test/ComponentTests/TestAssets/Directory.Build.targets @@ -0,0 +1,12 @@ + + + + + false + false + + + + + + diff --git a/test/E2ETests/Automation.CLI/Automation.CLI.csproj b/test/E2ETests/Automation.CLI/Automation.CLI.csproj index 1a11e80d67..83ee2a0787 100644 --- a/test/E2ETests/Automation.CLI/Automation.CLI.csproj +++ b/test/E2ETests/Automation.CLI/Automation.CLI.csproj @@ -33,7 +33,7 @@ - ..\..\..\packages\Microsoft.TestPlatform.AdapterUtilities.$(TestPlatformVersion)\lib\net45\Microsoft.TestPlatform.AdapterUtilities.dll + ..\..\..\packages\Microsoft.TestPlatform.AdapterUtilities.$(TestPlatformVersion)\lib\netstandard2.0\Microsoft.TestPlatform.AdapterUtilities.dll ..\..\..\packages\Microsoft.TestPlatform.TranslationLayer.$(TestPlatformVersion)\lib\net451\Microsoft.TestPlatform.CommunicationUtilities.dll @@ -88,10 +88,11 @@ + + - diff --git a/test/E2ETests/Automation.CLI/CLITestBase.common.cs b/test/E2ETests/Automation.CLI/CLITestBase.common.cs new file mode 100644 index 0000000000..50a9775e60 --- /dev/null +++ b/test/E2ETests/Automation.CLI/CLITestBase.common.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.CLIAutomation +{ + using System; + using System.IO; + using System.Linq; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + public partial class CLITestBase + { + private const string E2ETestsRelativePath = @"..\..\..\"; + private const string TestAssetsFolder = "TestAssets"; + private const string ArtifactsFolder = "artifacts"; + private const string PackagesFolder = "packages"; + + // This value is automatically updated by "build.ps1" script. + private const string TestPlatformCLIPackage = @"Microsoft.TestPlatform.16.10.0-preview-20210304-04"; + private const string VstestConsoleRelativePath = @"tools\net451\Common7\IDE\Extensions\TestPlatform\vstest.console.exe"; + + /// + /// Gets the relative path of repository root from start-up directory. + /// + /// Relative path of the repository root + protected virtual string GetRelativeRepositoryRootPath() => E2ETestsRelativePath; + + /// + /// Gets the full path to a test asset. + /// + /// Name of the asset with extension. E.g. SimpleUnitTest.dll + /// Full path to the test asset. + /// + /// Test assets follow several conventions: + /// (a) They are built for provided build configuration. + /// (b) Name of the test asset matches the parent directory name. E.g. TestAssets\SimpleUnitTest\SimpleUnitTest.xproj must + /// produce TestAssets\SimpleUnitTest\bin\Debug\SimpleUnitTest.dll + /// (c) TestAssets are copied over to a central location i.e. "TestAssets\artifacts\*.*" + /// + protected string GetAssetFullPath(string assetName) + { + var assetPath = Path.Combine( + Environment.CurrentDirectory, + this.GetRelativeRepositoryRootPath(), + ArtifactsFolder, + TestAssetsFolder, + assetName); + + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + + return assetPath; + } + + protected string GetTestAdapterPath() + { + var testAdapterPath = Path.Combine( + Environment.CurrentDirectory, + this.GetRelativeRepositoryRootPath(), + ArtifactsFolder, + TestAssetsFolder); + + return testAdapterPath; + } + + /// + /// Gets the RunSettingXml having testadapterpath filled in specified by arguement. + /// Inserts testAdapterPath in existing runSetting if not present already, + /// or generates new runSettings with testAdapterPath if runSettings is Empty. + /// + /// RunSettings provided for discovery/execution + /// Full path to TestAdapter. + /// RunSettingXml as string + protected string GetRunSettingXml(string settingsXml, string testAdapterPath) + { + if (string.IsNullOrEmpty(settingsXml)) + { + settingsXml = XmlRunSettingsUtilities.CreateDefaultRunSettings(); + } + + XmlDocument doc = new XmlDocument(); + using (var xmlReader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true })) + { + doc.Load(xmlReader); + } + + XmlElement root = doc.DocumentElement; + RunConfiguration runConfiguration = new RunConfiguration(testAdapterPath); + XmlElement runConfigElement = runConfiguration.ToXml(); + if (root[runConfiguration.SettingsName] == null) + { + XmlNode newNode = doc.ImportNode(runConfigElement, true); + root.AppendChild(newNode); + } + else + { + XmlNode newNode = doc.ImportNode(runConfigElement.FirstChild, true); + root[runConfiguration.SettingsName].AppendChild(newNode); + } + + return doc.OuterXml; + } + } +} diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.e2e.cs similarity index 77% rename from test/E2ETests/Automation.CLI/CLITestBase.cs rename to test/E2ETests/Automation.CLI/CLITestBase.e2e.cs index 9c6f5b85ef..5e70630a7c 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.e2e.cs @@ -12,17 +12,8 @@ namespace Microsoft.MSTestV2.CLIAutomation using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; - public class CLITestBase + public partial class CLITestBase { - private const string E2ETestsRelativePath = @"..\..\..\"; - private const string TestAssetsFolder = "TestAssets"; - private const string ArtifactsFolder = "artifacts"; - private const string PackagesFolder = "packages"; - - // This value is automatically updated by "build.ps1" script. - private const string TestPlatformCLIPackage = @"Microsoft.TestPlatform.16.9.1"; - private const string VstestConsoleRelativePath = @"tools\net451\Common7\IDE\Extensions\TestPlatform\vstest.console.exe"; - private static VsTestConsoleWrapper vsTestConsoleWrapper; private DiscoveryEventsHandler discoveryEventsHandler; private RunEventsHandler runEventsHandler; @@ -90,7 +81,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "", /// Full path to vstest.console.exe public string GetConsoleRunnerPath() { - var packagesFolder = Path.Combine(Environment.CurrentDirectory, E2ETestsRelativePath, PackagesFolder); + var packagesFolder = Path.Combine(Environment.CurrentDirectory, this.GetRelativeRepositoryRootPath(), PackagesFolder); var vstestConsolePath = Path.Combine(packagesFolder, TestPlatformCLIPackage, VstestConsoleRelativePath); Assert.IsTrue(File.Exists(vstestConsolePath), "GetConsoleRunnerPath: Path not found: {0}", vstestConsolePath); @@ -258,81 +249,6 @@ public void ValidateTestRunTime(int thresholdTime) $"Test Run was expected to not exceed {thresholdTime} but it took {this.runEventsHandler.ElapsedTimeInRunningTests}"); } - /// - /// Gets the full path to a test asset. - /// - /// Name of the asset with extension. E.g. SimpleUnitTest.dll - /// Full path to the test asset. - /// - /// Test assets follow several conventions: - /// (a) They are built for provided build configuration. - /// (b) Name of the test asset matches the parent directory name. E.g. TestAssets\SimpleUnitTest\SimpleUnitTest.xproj must - /// produce TestAssets\SimpleUnitTest\bin\Debug\SimpleUnitTest.dll - /// (c) TestAssets are copied over to a central location i.e. "TestAssets\artifacts\*.*" - /// - protected string GetAssetFullPath(string assetName) - { - var assetPath = Path.Combine( - Environment.CurrentDirectory, - E2ETestsRelativePath, - ArtifactsFolder, - TestAssetsFolder, - assetName); - - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); - - return assetPath; - } - - protected string GetTestAdapterPath() - { - var testAdapterPath = Path.Combine( - Environment.CurrentDirectory, - E2ETestsRelativePath, - ArtifactsFolder, - TestAssetsFolder); - - return testAdapterPath; - } - - /// - /// Gets the RunSettingXml having testadapterpath filled in specified by arguement. - /// Inserts testAdapterPath in existing runSetting if not present already, - /// or generates new runSettings with testAdapterPath if runSettings is Empty. - /// - /// RunSettings provided for discovery/execution - /// Full path to TestAdapter. - /// RunSettingXml as string - protected string GetRunSettingXml(string settingsXml, string testAdapterPath) - { - if (string.IsNullOrEmpty(settingsXml)) - { - settingsXml = XmlRunSettingsUtilities.CreateDefaultRunSettings(); - } - - XmlDocument doc = new XmlDocument(); - using (var xmlReader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true })) - { - doc.Load(xmlReader); - } - - XmlElement root = doc.DocumentElement; - RunConfiguration runConfiguration = new RunConfiguration(testAdapterPath); - XmlElement runConfigElement = runConfiguration.ToXml(); - if (root[runConfiguration.SettingsName] == null) - { - XmlNode newNode = doc.ImportNode(runConfigElement, true); - root.AppendChild(newNode); - } - else - { - XmlNode newNode = doc.ImportNode(runConfigElement.FirstChild, true); - root[runConfiguration.SettingsName].AppendChild(newNode); - } - - return doc.OuterXml; - } - /// /// Gets the test method name from full name. /// diff --git a/test/E2ETests/Automation.CLI/packages.config b/test/E2ETests/Automation.CLI/packages.config index cfe2887a1b..ffb32bb024 100644 --- a/test/E2ETests/Automation.CLI/packages.config +++ b/test/E2ETests/Automation.CLI/packages.config @@ -1,9 +1,9 @@  - - - - + + + + diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs new file mode 100644 index 0000000000..04bad79ef3 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + + + [TestClass] + public class DataRowTests : CLITestBase + { + private const string TestAssembly = "DataRowTestProject.dll"; + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRows_SimpleDataRows() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethod (BaseString1)", + "DataRowTestMethod (BaseString2)", + "DataRowTestMethod (BaseString3)", + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)" + ); + + // 4 tests of BaseClass.DataRowTestMethod - 3 data row results and 1 parent result + // 3 tests of DerivedClass.DataRowTestMethod - 2 data row results and 1 parent result + // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests + Assert.That.PassedTestCount(testResults, 7 - 2); + } + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)" + ); + + // 3 tests of DerivedClass.DataRowTestMethod - 2 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 3 - 1); + } + + [TestMethod] + public void DataRowsExecuteWithRequiredAndOptionalParameters() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodWithSomeOptionalParameters (123)", + "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString1)", + "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString2,DerivedOptionalString3)" + ); + + // 4 tests of DerivedClass.DataRowTestMethodWithSomeOptionalParameters - 3 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 4 - 1); + } + + [TestMethod] + public void DataRowsExecuteWithParamsArrayParameter() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodWithParamsParameters (2)", + "DataRowTestMethodWithParamsParameters (2,DerivedSingleParamsArg)", + "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2)", + "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2,DerivedParamsArg3)" + ); + + // 5 tests of DerivedClass.DataRowTestMethodWithParamsParameters - 4 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 5 - 1); + } + + [TestMethod] + public void DataRowsFailWhenInvalidArgumentsProvided() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowOptionalInvalidArguments"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodFailsWithInvalidArguments ()", + "DataRowTestMethodFailsWithInvalidArguments (2)", + "DataRowTestMethodFailsWithInvalidArguments (2,DerivedRequiredArgument,DerivedOptionalArgument,DerivedExtraArgument)" + ); + + // 4 tests of DerivedClass.DataRowTestMethodFailsWithInvalidArguments - 3 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 4 - 1); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs new file mode 100644 index 0000000000..5d8e4b11c8 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + using System.Linq; + + [TestClass] + public class DataSourceTests : CLITestBase + { + private const string TestAssembly = "DataSourceTestProject.dll"; + + [TestMethod] + public void ExecuteCsvTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "CsvTestMethod"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "CsvTestMethod (Data Row 0)", + "CsvTestMethod (Data Row 2)" + ); + + Assert.That.ContainsTestsFailed(testResults, + "CsvTestMethod (Data Row 1)", + "CsvTestMethod (Data Row 3)" + ); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj b/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj new file mode 100644 index 0000000000..c12fa7e130 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj @@ -0,0 +1,58 @@ + + + ..\..\..\ + false + $(TestFxRoot)artifacts\$(Configuration)\ + + + + + net452 + v4.5.2 + false + + false + 1685 + true + false + + + + + + + + + + + + + + + + $(SourcePath)MSTest.CoreAdapter\Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll + + + + $(SourcePath)MSTest.CoreAdapter\System.Collections.Concurrent.dll + + + + $(SourcePath)PlatformServices.Desktop\Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + + + + $(SourcePath)PlatformServices.Interface\Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll + + + + $(SourcePath)MSTest.Core\Microsoft.VisualStudio.TestPlatform.TestFramework.dll + + + + $(SourcePath)Extension.Desktop\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll + + + + + \ No newline at end of file diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs b/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs new file mode 100644 index 0000000000..7c51e42d53 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + + using System.Collections.Generic; + using System.Linq; + + using Assert = Microsoft.VisualStudio.TestTools.UnitTesting.Assert; + + public static class AssertionExtensions + { + public static void PassedTestCount(this Assert _, IEnumerable actual, int expectedCount) + => AssertOutcomeCount(actual, TestOutcome.Passed, expectedCount); + + public static void FailedTestCount(this Assert _, IEnumerable actual, int expectedCount) + => AssertOutcomeCount(actual, TestOutcome.Failed, expectedCount); + + public static void TestsPassed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Passed, expectedTests, true, settings); + + public static void TestsPassed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Passed, expectedTests, true); + + public static void TestsFailed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Failed, expectedTests, true, settings); + + public static void TestsFailed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Failed, expectedTests, true); + + public static void ContainsTestsPassed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Passed, expectedTests, false, settings); + + public static void ContainsTestsPassed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Passed, expectedTests); + + public static void ContainsTestsFailed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Failed, expectedTests, false, settings); + + public static void ContainsTestsFailed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Failed, expectedTests); + + private static void ContainsExpectedTestsWithOutcome(IEnumerable outcomedTests, IEnumerable testCases, TestOutcome expectedOutcome, IEnumerable expectedTests, bool matchCount = false, MSTestSettings settings = null) + { + if (matchCount) + { + var expectedCount = expectedTests.Count(); + var outcomedCount = outcomedTests.Count(); + + AssertOutcomeCount(outcomedCount, expectedOutcome, expectedCount); + } + + foreach (var test in expectedTests) + { + var testFound = outcomedTests.Any( + p => test.Equals(p.TestCase?.FullyQualifiedName) + || test.Equals(p.DisplayName) + || test.Equals(p.TestCase.DisplayName)); + + Assert.IsTrue(testFound, GetOutcomeAssertString(test, expectedOutcome)); + } + } + + private static void ContainsExpectedTestsWithOutcome(IEnumerable outcomedTests, TestOutcome expectedOutcome, string[] expectedTests, bool matchCount = false) + { + if (matchCount) + { + var expectedCount = expectedTests.Count(); + AssertOutcomeCount(outcomedTests, expectedOutcome, expectedCount); + } + + foreach (var test in expectedTests) + { + var testFound = outcomedTests.Any(p => p.DisplayName == test); + + Assert.IsTrue(testFound, GetOutcomeAssertString(test, expectedOutcome)); + } + } + + private static string GetOutcomeAssertString(string testName, TestOutcome outcome) + { + switch (outcome) + { + case TestOutcome.None: + return $"\"{testName}\" does not have TestOutcome.None outcome."; + + case TestOutcome.Passed: + return $"\"{testName}\" does not appear in passed tests list."; + + case TestOutcome.Failed: + return $"\"{testName}\" does not appear in failed tests list."; + + case TestOutcome.Skipped: + return $"\"{testName}\" does not appear in skipped tests list."; + + case TestOutcome.NotFound: + return $"\"{testName}\" does not appear in not found tests list."; + } + + return string.Empty; + } + + private static void AssertOutcomeCount(IEnumerable actual, TestOutcome expectedOutcome, int expectedCount) + { + var outcomedTests = actual.Where(i => i.Outcome == expectedOutcome); + var actualCount = outcomedTests.Count(); + + AssertOutcomeCount(actualCount, expectedOutcome, expectedCount); + } + private static void AssertOutcomeCount(int actualCount, TestOutcome expectedOutcome, int expectedCount) + { + Assert.AreEqual(expectedCount, actualCount, $"Test run expected to contain {expectedCount} tests, but ran {actualCount}."); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs new file mode 100644 index 0000000000..eaf65dbb55 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + + [TestClass] + public class FSharpTestProjectTests : CLITestBase + { + private const string TestAssembly = "FSharpTestProject.dll"; + + [TestMethod] + public void ExecuteCsvTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.TestsPassed(testResults, "Test method passing with a . in it"); + Assert.That.PassedTestCount(testResults, 1); + Assert.That.FailedTestCount(testResults, 0); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs new file mode 100644 index 0000000000..cb7896903f --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + using System.Linq; + + [TestClass] + public class TestDataSourceExtensibilityTests : CLITestBase + { + private const string TestAssembly = "FxExtensibilityTestProject.dll"; + + /* + Add tests for: + - Ignored tests are discovered during discovery + - Ignored tests are not expanded (DataRow, DataSource, etc) + */ + + [TestMethod] + public void CustomTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); + } + + [TestMethod] + public void AssertExtensibilityTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, "BasicAssertExtensionTest", "ChainedAssertExtensionTest"); + Assert.That.ContainsTestsFailed(testResults, "BasicFailingAssertExtensionTest", "ChainedFailingAssertExtensionTest"); + } + + [TestMethod] + public void ExecuteCustomTestExtensibilityTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "CustomTestMethod1 - Execution number 1", + "CustomTestMethod1 - Execution number 2", + "CustomTestMethod1 - Execution number 4", + "CustomTestMethod1 - Execution number 5", + "CustomTestClass1 - Execution number 1", + "CustomTestClass1 - Execution number 2", + "CustomTestClass1 - Execution number 4", + "CustomTestClass1 - Execution number 5" + ); + + Assert.That.ContainsTestsFailed(testResults, + "CustomTestMethod1 - Execution number 3", + "CustomTestClass1 - Execution number 3" + ); + } + + [TestMethod] + public void ExecuteCustomTestExtensibilityWithTestDataTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.TestsPassed(testResults, + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)" + ); + + Assert.That.TestsFailed(testResults, + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)" + ); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs new file mode 100644 index 0000000000..b14d2ca58d Binary files /dev/null and b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs differ diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs new file mode 100644 index 0000000000..97b041463a --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs @@ -0,0 +1,358 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DiscoveryAndExecutionTests.Utilities +{ + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + using System.Reflection; + using System.Text; + using System.Text.RegularExpressions; + + internal static class TestCaseFilterFactory + { + private static readonly MethodInfo CachedGetMultiValueMethod; + private static readonly MethodInfo CachedEqualsComparerMethod; + private static readonly MethodInfo CachedContainsComparerMethod; + + static TestCaseFilterFactory() + { + CachedGetMultiValueMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(GetMultiValue), BindingFlags.Static | BindingFlags.NonPublic); + CachedEqualsComparerMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(EqualsComparer), BindingFlags.Static | BindingFlags.NonPublic); + CachedContainsComparerMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(ContainsComparer), BindingFlags.Static | BindingFlags.NonPublic); + } + + public static ITestCaseFilterExpression ParseTestFilter(string filterString) + { + ValidateArg.NotNullOrEmpty(filterString, nameof(filterString)); + if (Regex.IsMatch(filterString, @"\(\s*\)")) + { + throw new FormatException($"Invalid filter, empty parenthesis: {filterString}"); + } + + var tokens = TokenizeFilter(filterString); + + var ops = new Stack(); + var exp = new Stack, bool>>>(); + + // simplified version of microsoft/vstest/src/Microsoft.TestPlatform.Common/Filtering/FilterExpression.cs + + // This is based on standard parsing of in order expression using two stacks (operand stack and operator stack) + // Precedence(And) > Precedence(Or) + foreach (var t in tokens) + { + var token = t.Trim(); + if (string.IsNullOrEmpty(token)) + { + continue; + } + + switch (token) + { + case "&": + case "|": + var op = token == "&" ? Operator.And : Operator.Or; + var top = ops.Count == 0 ? Operator.None : ops.Peek(); + if (ops.Count == 0 || top == Operator.OpenBrace || top < op) + { + ops.Push(op); + continue; + } + MergeExpression(exp, ops.Pop()); + continue; + + case "(": + ops.Push(Operator.OpenBrace); + continue; + + case ")": + if (ops.Count == 0) + { + throw new FormatException($"Invalid filter, missing parenthesis open: {filterString}"); + } + + while (ops.Peek() != Operator.OpenBrace) + { + MergeExpression(exp, ops.Pop()); + if (ops.Count == 0) + { + throw new FormatException($"Invalid filter, missing parenthesis open: {filterString}"); + } + } + ops.Pop(); + continue; + + default: + var e = ConditionExpresion(token); + exp.Push(e); + break; + } + } + + while (ops.Count != 0) + { + MergeExpression(exp, ops.Pop()); + } + + if (exp.Count != 1) + { + throw new FormatException($"Invalid filter, missing operator: {filterString}"); + } + + var lambda = exp.Pop().Compile(); + + return new TestFilterExpression(filterString, lambda); + } + + private class TestFilterExpression : ITestCaseFilterExpression + { + private readonly string filter; + private readonly Func, bool> expression; + + public TestFilterExpression(string filter, Func, bool> expression) + { + this.filter = filter; + this.expression = expression; + } + + public string TestCaseFilterValue => filter; + + public bool MatchTestCase(TestCase testCase, Func propertyValueProvider) => expression(propertyValueProvider); + } + + private static void MergeExpression(Stack, bool>>> exp, Operator op) + { + ValidateArg.NotNull(exp, nameof(exp)); + if (op != Operator.And && op != Operator.Or) + { + throw new ArgumentException($"Unexpected operator: {op}", nameof(op)); + } + if (exp.Count != 2) + { + throw new ArgumentException($"Unexpected expression tree: {exp.Count} elements, expected 2.", nameof(exp)); + } + + var parameter = Expression.Parameter(typeof(Func), "value"); + var right = Expression.Invoke(exp.Pop(), parameter); + var left = Expression.Invoke(exp.Pop(), parameter); + + Expression body = op == Operator.And ? Expression.And(left, right) : Expression.Or(left, right); + + var lambda = Expression.Lambda, bool>>(body, parameter); + + exp.Push(lambda); + } + + private static IEnumerable TokenizeFilter(string filterString) + { + var token = new StringBuilder(filterString.Length); + + var escaping = false; + for (int i = 0; i < filterString.Length; i++) + { + var c = filterString[i]; + + if (escaping) + { + token.Append(c); + escaping = false; + continue; + } + + switch (c) + { + case FilterHelper.EscapeCharacter: + escaping = true; + continue; + + case '&': + case '|': + case '(': + case ')': + if (token.Length != 0) + { + yield return token.ToString(); + token.Clear(); + } + yield return c.ToString(); + continue; + + default: + token.Append(c); + break; + } + } + + if (token.Length != 0) + { + yield return token.ToString(); + } + } + + private static IEnumerable TokenizeCondition(string conditionString) + { + ValidateArg.NotNullOrEmpty(conditionString, nameof(conditionString)); + var token = new StringBuilder(conditionString.Length); + + var escaped = false; + for (int i = 0; i < conditionString.Length; i++) + { + var c = conditionString[i]; + + if (escaped) + { + token.Append(c); + escaped = false; + continue; + } + + switch (c) + { + case '=': + case '~': + case '!': + if (token.Length > 0) + { + yield return token.ToString(); + token.Clear(); + } + + if (c == '!') + { + var op = conditionString[i + 1]; + + if (op == '~' || op == '=') + { + yield return token.ToString() + conditionString[++i]; + continue; + } + } + + yield return c.ToString(); + continue; + + default: + token.Append(c); + break; + } + } + + if (token.Length > 0) + { + yield return token.ToString(); + } + } + + private static string[] GetMultiValue(object value) + { + if (value is string[] i) + { + return i; + } + else if (value != null) + { + return new[] { value.ToString() }; + } + + return null; + } + + private static bool EqualsComparer(string[] values, string value) + { + foreach (var v in values) + { + if (v.Equals(value, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static bool ContainsComparer(string[] values, string value) + { + foreach (var v in values) + { + if (v.IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + + return false; + } + + private static Expression, bool>> ConditionExpresion(string conditionString) + { + ValidateArg.NotNull(conditionString, nameof(conditionString)); + + var condition = TokenizeCondition(conditionString).ToArray(); + + Expression parameterName, expectedValue, parameterValueProvider, expression; + string op; + if (condition.Length == 1) + { + parameterName = Expression.Constant("FullyQualifiedName"); + expectedValue = Expression.Constant(conditionString.Trim()); + op = "~"; + } + else if (condition.Length == 3) + { + parameterName = Expression.Constant(condition[0]); + expectedValue = Expression.Constant(condition[2].Trim()); + op = condition[1]; + } + else + { + throw new FormatException("Invalid ConditionExpresion: " + conditionString); + } + + ParameterExpression parameter = Expression.Parameter(typeof(Func), "p"); + + parameterValueProvider = Expression.Call(CachedGetMultiValueMethod, Expression.Invoke(parameter, parameterName)); + + MethodInfo comparer; + switch (op.Last()) + { + case '=': + comparer = CachedEqualsComparerMethod; + break; + + case '~': + comparer = CachedContainsComparerMethod; + break; + + default: + throw new FormatException($"Invalid operator in {conditionString}: {condition[1]}"); + } + + expression = Expression.Call(comparer, parameterValueProvider, expectedValue); + + if (op[0] == '!') + { + expression = Expression.Not(expression); + } + + + var lambda = Expression.Lambda, bool>>(expression, parameter); + + return lambda; + } + + private enum Operator + { + None, + Or, + And, + OpenBrace, + CloseBrace, + } + } +} diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index 0358dc9835..4ecd580144 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -40,8 +40,9 @@ public void ExecuteCustomTestExtensibilityWithTestDataTests() "CustomTestMethod2 (B)", "CustomTestMethod2 (B)"); + // TODO :: ASK REVIEW :: SHOULD WE CONTINUE RETURNING A PARENT TEST FOR DATA BOUND TESTS? // Parent results should fail and thus failed count should be 7. - this.ValidateFailedTestsCount(7); + // this.ValidateFailedTestsCount(7); this.ValidateFailedTestsContain( TestAssembly, true, diff --git a/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs index 280c4aeecd..1cf38604f8 100644 --- a/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs @@ -26,7 +26,7 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRow // 4 tests of BaseClass.DataRowTestMethod - 3 data row results and 1 parent result // 3 tests of DerivedClass.DataRowTestMethod - 2 data row results and 1 parent result // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests - this.ValidatePassedTestsCount(7); + this.ValidatePassedTestsCount(7 - 2); } [TestMethod] @@ -39,7 +39,7 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp "DataRowTestMethod (DerivedString2)"); // 3 tests of DerivedClass.DataRowTestMethod - 2 datarow result and 1 parent result - this.ValidatePassedTestsCount(3); + this.ValidatePassedTestsCount(3 - 1); } [TestMethod] @@ -53,7 +53,7 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString2,DerivedOptionalString3)"); // 4 tests of DerivedClass.DataRowTestMethodWithSomeOptionalParameters - 3 datarow result and 1 parent result - this.ValidatePassedTestsCount(4); + this.ValidatePassedTestsCount(4 - 1); } [TestMethod] @@ -68,7 +68,7 @@ public void DataRowsExecuteWithAllOptionalParameters() "DataRowTestMethodWithAllOptionalParameters (123,DerivedOptionalString5,DerivedOptionalString6)"); // 5 tests of DerivedClass.DataRowTestMethodWithAllOptionalParameters - 4 datarow result and 1 parent result - this.ValidatePassedTestsCount(5); + this.ValidatePassedTestsCount(5 - 1); } [TestMethod] @@ -83,7 +83,7 @@ public void DataRowsExecuteWithParamsArrayParameter() "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2,DerivedParamsArg3)"); // 5 tests of DerivedClass.DataRowTestMethodWithParamsParameters - 4 datarow result and 1 parent result - this.ValidatePassedTestsCount(5); + this.ValidatePassedTestsCount(5 - 1); } [TestMethod] @@ -97,7 +97,7 @@ public void DataRowsFailWhenInvalidArgumentsProvided() "DataRowTestMethodFailsWithInvalidArguments (2,DerivedRequiredArgument,DerivedOptionalArgument,DerivedExtraArgument)"); // 4 tests of DerivedClass.DataRowTestMethodFailsWithInvalidArguments - 3 datarow result and 1 parent result - this.ValidatePassedTestsCount(4); + this.ValidatePassedTestsCount(4 - 1); } } } diff --git a/test/E2ETests/Smoke.E2E.Tests/TestDataSourceExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/TestDataSourceExtensibilityTests.cs index b23071cd54..43c4c5cfa8 100644 --- a/test/E2ETests/Smoke.E2E.Tests/TestDataSourceExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/TestDataSourceExtensibilityTests.cs @@ -15,120 +15,7 @@ public class TestDataSourceExtensibilityTests : CLITestBase public void ExecuteTestDataSourceExtensibilityTests() { this.InvokeVsTestForExecution(new string[] { TestAssembly }); - this.ValidatePassedTestsContain( - "CustomTestDataSourceTestMethod1 (1,2,3)", - "CustomTestDataSourceTestMethod1 (4,5,6)"); + this.ValidatePassedTestsContain("CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); } - - /* This test needs a reference to "Microsoft.TestPlatform.AdapterUtilities" and "Microsoft.TestPlatform.ObjectModel" NuGet packages. - * It's used to debug FQN changes in test discovery and test execution. - - // using System; - // using System.Collections.Generic; - // using System.Collections.ObjectModel; - // using System.Diagnostics; - // using System.IO; - // using System.Linq; - // using System.Reflection; - // using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; - // using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; - // using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; - // using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; - // using Microsoft.VisualStudio.TestPlatform.ObjectModel; - // using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; - // using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - - [TestMethod] - public void DataSourceTest() - { - var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); - - var unitTestDiscoverer = new UnitTestDiscoverer(); - var logger = new InternalLogger(); - var sink = new InternalSink(); - string runSettingXml = this.GetRunSettingXml(string.Empty, this.GetTestAdapterPath()); - var context = new InternalDiscoveryContext(runSettingXml); - - unitTestDiscoverer.DiscoverTestsInSource(assemblyPath, logger, sink, context); - - var settings = this.GetSettingsWithDebugTrace(true); - var unitTestRunner = new UnitTestRunner(settings); - var testCase = sink.DiscoveredTests.Single(i => i.DisplayName == "CustomTestDataSourceTestMethod1"); - - var unitTestElement = testCase.ToUnitTestElement(assemblyPath); - var testResults = unitTestRunner.RunSingleTest(unitTestElement.TestMethod, new Dictionary()); - - var passedTestResults = testResults.Where(i => i.Outcome == Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome.Passed).Select(i => i.ToTestResult(testCase, DateTimeOffset.Now, DateTimeOffset.Now, settings)); - - var expectedTests = new[] { "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)" }; - foreach (var test in expectedTests) - { - var testFound = passedTestResults.Any( - p => test.Equals(p.TestCase?.FullyQualifiedName) - || test.Equals(p.DisplayName) - || test.Equals(p.TestCase.DisplayName)); - - Assert.IsTrue(testFound, "Test '{0}' does not appear in passed tests list.", test); - } - } - - private MSTestSettings GetSettingsWithDebugTrace(bool captureDebugTraceValue) - { - string runSettingxml = - @" - - " + captureDebugTraceValue + @" - - "; - - return MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsName); - } - - private class InternalLogger : IMessageLogger - { - public void SendMessage(TestMessageLevel testMessageLevel, string message) - { - Debug.WriteLine($"{testMessageLevel}: {message}"); - } - } - - private class InternalSink : Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter.ITestCaseDiscoverySink - { - private readonly List testCases = new List(); - - public ReadOnlyCollection DiscoveredTests => this.testCases.AsReadOnly(); - - public void SendTestCase(TestCase discoveredTest) => this.testCases.Add(discoveredTest); - } - - private class InternalDiscoveryContext : IDiscoveryContext - { - private readonly IRunSettings runSettings; - - public InternalDiscoveryContext(string runSettings) - { - this.runSettings = new InternalRunSettings(runSettings); - } - - public IRunSettings RunSettings => this.runSettings; - - private class InternalRunSettings : IRunSettings - { - private readonly string runSettings; - - public InternalRunSettings(string runSettings) - { - this.runSettings = runSettings; - } - - public string SettingsXml => this.runSettings; - - public ISettingsProvider GetSettings(string settingsName) - { - throw new System.NotImplementedException(); - } - } - } - */ } } diff --git a/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs b/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs index 48d5b906c9..79c09b82ec 100644 --- a/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs @@ -15,9 +15,9 @@ public class TestProjectFSharpTests : CLITestBase public void ExecuteCustomTestExtensibilityTests() { this.InvokeVsTestForExecution(new string[] { TestAssembly }); + this.ValidatePassedTestsContain("Test method passing with a . in it"); this.ValidateFailedTestsCount(0); this.ValidatePassedTestsCount(1); - this.ValidatePassedTestsContain("Test method passing with a . in it"); } } } \ No newline at end of file diff --git a/test/E2ETests/TestAssets/DataSourceTestProject/UnitTest1.cs b/test/E2ETests/TestAssets/DataSourceTestProject/UnitTest1.cs index 16c9c03d98..f444c78a00 100644 --- a/test/E2ETests/TestAssets/DataSourceTestProject/UnitTest1.cs +++ b/test/E2ETests/TestAssets/DataSourceTestProject/UnitTest1.cs @@ -14,7 +14,7 @@ public TestContext TestContext set; } - [TestMethod, DataSource("Microsoft.VisualStudio.TestTools.DataSource.CSV", "a.csv", "a#csv", DataAccessMethod.Sequential)] + [TestMethod, DataSource("Microsoft.VisualStudio.TestTools.DataSource.CSV", "|DataDirectory|\\a.csv", "a#csv", DataAccessMethod.Sequential)] public void CsvTestMethod() { Assert.AreEqual(1, TestContext.DataRow["Item1"]); diff --git a/test/E2ETests/TestAssets/Directory.Build.targets b/test/E2ETests/TestAssets/Directory.Build.targets new file mode 100644 index 0000000000..87043c6abb --- /dev/null +++ b/test/E2ETests/TestAssets/Directory.Build.targets @@ -0,0 +1,12 @@ + + + + + false + false + + + + + + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs index e01171621a..0cdce25911 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs @@ -19,6 +19,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Discovery using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.TestableImplementations; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Moq; @@ -57,7 +58,7 @@ public void Cleanup() [TestMethodV1] public void ConstructorShouldPopulateSettings() { - string runSettingxml = + string runSettingsXml = @" True @@ -75,8 +76,9 @@ public void ConstructorShouldPopulateSettings() } }); - MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsName); + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName); var assemblyEnumerator = new AssemblyEnumerator(adapterSettings); + assemblyEnumerator.RunSettingsXml = runSettingsXml; Assert.IsTrue(MSTestSettings.CurrentSettings.ForcedLegacyMode); Assert.AreEqual("DummyPath\\TestSettings1.testsettings", MSTestSettings.CurrentSettings.TestSettingsFile); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index d800c29e48..ed26490906 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -555,11 +555,9 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid var results = testMethodRunner.RunTestMethod(); // check for datarowIndex - // 1st is parent result. - Assert.AreEqual(-1, results[0].DatarowIndex); - Assert.AreEqual(0, results[1].DatarowIndex); - Assert.AreEqual(1, results[2].DatarowIndex); - Assert.AreEqual(2, results[3].DatarowIndex); + Assert.AreEqual(0, results[0].DatarowIndex); + Assert.AreEqual(1, results[1].DatarowIndex); + Assert.AreEqual(2, results[2].DatarowIndex); } [TestMethodV1] @@ -584,11 +582,9 @@ public void RunTestMethodShoudlRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo var results = testMethodRunner.RunTestMethod(); // check for datarowIndex as only DataSource Tests are Run - // 1st is parent result. - Assert.AreEqual(-1, results[0].DatarowIndex); - Assert.AreEqual(0, results[1].DatarowIndex); - Assert.AreEqual(1, results[2].DatarowIndex); - Assert.AreEqual(2, results[3].DatarowIndex); + Assert.AreEqual(0, results[0].DatarowIndex); + Assert.AreEqual(1, results[1].DatarowIndex); + Assert.AreEqual(2, results[2].DatarowIndex); } [TestMethodV1] @@ -612,9 +608,8 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided var results = testMethodRunner.RunTestMethod(); - // 1st results should be parent result. - Assert.AreEqual(2, results.Length); - Assert.AreEqual("DataRowTestDisplayName", results[1].DisplayName); + Assert.AreEqual(1, results.Length); + Assert.AreEqual("DataRowTestDisplayName", results[0].DisplayName); } [TestMethodV1] @@ -637,9 +632,8 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN var results = testMethodRunner.RunTestMethod(); - // 1st results should be parent result. - Assert.AreEqual(2, results.Length); - Assert.AreEqual("DummyTestMethod (2,DummyString)", results[1].DisplayName); + Assert.AreEqual(1, results.Length); + Assert.AreEqual("DummyTestMethod (2,DummyString)", results[0].DisplayName); } [TestMethodV1] @@ -664,219 +658,8 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests() this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[0].ResultFiles.ToList(), "C:\\temp.txt"); CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTests() - { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); - - var results = testMethodRunner.RunTestMethod(); - - // check for parent result - Assert.AreEqual(4, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTestsContainingSingleTest() - { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1 }); - - var results = testMethodRunner.RunTestMethod(); - - // Parent result should exist. - Assert.AreEqual(2, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTests() - { - UTF.TestResult testResult = new UTF.TestResult - { - ResultFiles = new List() { "C:\\temp.txt" } - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - int dummyIntData2 = 2; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); - - var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); - - // Parent result should exist. - Assert.AreEqual(3, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTestsContainingSingleTest() - { - UTF.TestResult testResult = new UTF.TestResult - { - ResultFiles = new List() { "C:\\temp.txt" } - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - - var attribs = new Attribute[] { dataRowAttribute1 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - - // Parent result should exist. - Assert.AreEqual(2, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() - { - var testMethodAttributeMock = new Mock(); - testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] - { - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } - }); - - var localTestMethodOptions = new TestMethodOptions - { - Timeout = 200, - Executor = testMethodAttributeMock.Object, - TestContext = this.testContextImplementation, - ExpectedException = null - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - var results = testMethodRunner.Execute(); - Assert.AreEqual(2, results.Length); - - // Parent result should not exists as its not data driven test. - Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - } - - [TestMethodV1] - public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataSourceDataDrivenTests() - { - var testExecutedCount = 0; - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => - { - return (testExecutedCount++ == 0) ? - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; - }); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); - - var results = testMethodRunner.RunTestMethod(); - - // check for parent result - Assert.AreEqual(4, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - - // Check for aggregate outcome. - Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[3].Outcome); - } - - [TestMethodV1] - public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataRowDataDrivenTests() - { - var testExecutedCount = 0; - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => - { - return (testExecutedCount++ == 0) ? - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; - }); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - int dummyIntData2 = 2; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); - - var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - - // Parent result should exist. - Assert.AreEqual(3, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); - - // Check for aggregate outcome. - Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); } #region Test data diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config index 73a75bf39f..c11a17f0b5 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config @@ -2,8 +2,8 @@ - - + + diff --git a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config index 1438956c89..a69b64e56d 100644 --- a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config index 637fb47db0..b62b9b9295 100644 --- a/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config index 311ae4c674..aa3e2b95b2 100644 --- a/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config @@ -1,8 +1,8 @@  - - + +