diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestDiscoveryExtensionManager.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestDiscoveryExtensionManager.cs index 83ce5331b0..aae07e8a6f 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestDiscoveryExtensionManager.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestDiscoveryExtensionManager.cs @@ -141,7 +141,7 @@ internal class TestDiscovererMetadata : ITestDiscovererCapabilities /// /// The file Extensions. /// The default Executor Uri. - public TestDiscovererMetadata(IReadOnlyCollection? fileExtensions, string? defaultExecutorUri, AssemblyType assemblyType = default) + public TestDiscovererMetadata(IReadOnlyCollection? fileExtensions, string? defaultExecutorUri, AssemblyType assemblyType = default, bool isDirectoryBased = false) { if (fileExtensions != null && fileExtensions.Count > 0) { @@ -154,6 +154,7 @@ public TestDiscovererMetadata(IReadOnlyCollection? fileExtensions, strin } AssemblyType = assemblyType; + IsDirectoryBased = isDirectoryBased; } /// @@ -182,4 +183,14 @@ public AssemblyType AssemblyType get; private set; } + + /// + /// true if the discoverer plugin is decorated with , + /// false otherwise. + /// + public bool IsDirectoryBased + { + get; + private set; + } } diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestDiscovererPluginInformation.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestDiscovererPluginInformation.cs index 216fbc0ed8..46a6f27adb 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestDiscovererPluginInformation.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestDiscovererPluginInformation.cs @@ -29,6 +29,7 @@ public TestDiscovererPluginInformation(Type testDiscovererType) FileExtensions = GetFileExtensions(testDiscovererType); DefaultExecutorUri = GetDefaultExecutorUri(testDiscovererType); AssemblyType = GetAssemblyType(testDiscovererType); + IsDirectoryBased = GetIsDirectoryBased(testDiscovererType); } } @@ -39,7 +40,7 @@ public override ICollection Metadata { get { - return new object?[] { FileExtensions, DefaultExecutorUri, AssemblyType }; + return new object?[] { FileExtensions, DefaultExecutorUri, AssemblyType, IsDirectoryBased }; } } @@ -71,15 +72,25 @@ public AssemblyType AssemblyType } /// - /// Helper to get file extensions from the FileExtensionAttribute on the discover plugin. + /// true if the discoverer plugin is decorated with , + /// false otherwise. /// - /// Data type of the test discoverer + public bool IsDirectoryBased + { + get; + private set; + } + + /// + /// Helper to get file extensions from the on the discover plugin. + /// + /// Data type of the test discoverer /// List of file extensions - private static List GetFileExtensions(Type testDicovererType) + private static List GetFileExtensions(Type testDiscovererType) { var fileExtensions = new List(); - var attributes = testDicovererType.GetTypeInfo().GetCustomAttributes(typeof(FileExtensionAttribute), false).ToArray(); + var attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(FileExtensionAttribute), inherit: false).ToArray(); if (attributes != null && attributes.Length > 0) { foreach (var attribute in attributes) @@ -96,15 +107,15 @@ private static List GetFileExtensions(Type testDicovererType) } /// - /// Returns the value of default executor Uri on this type. 'Null' if not present. + /// Returns the value of default executor Uri on this type. null if not present. /// /// The test discoverer Type. /// The default executor URI. private static string GetDefaultExecutorUri(Type testDiscovererType) { - string result = string.Empty; + var result = string.Empty; - object[] attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(DefaultExecutorUriAttribute), false).ToArray(); + var attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(DefaultExecutorUriAttribute), inherit: false).ToArray(); if (attributes != null && attributes.Length > 0) { DefaultExecutorUriAttribute executorUriAttribute = (DefaultExecutorUriAttribute)attributes[0]; @@ -119,7 +130,7 @@ private static string GetDefaultExecutorUri(Type testDiscovererType) } /// - /// Helper to get the supported assembly type from the CategoryAttribute on the discover plugin. + /// Helper to get the supported assembly type from the on the discover plugin. /// /// The test discoverer Type. /// Supported assembly type. @@ -134,4 +145,15 @@ private static AssemblyType GetAssemblyType(Type testDiscovererType) Enum.TryParse(category, true, out AssemblyType assemblyType); return assemblyType; } + + /// + /// Returns true if the discoverer plugin is decorated with + /// , false otherwise. + /// + /// Data type of the test discoverer + private static bool GetIsDirectoryBased(Type testDiscovererType) + { + var attribute = testDiscovererType.GetTypeInfo().GetCustomAttribute(typeof(DirectoryBasedTestDiscovererAttribute), inherit: false); + return attribute is DirectoryBasedTestDiscovererAttribute; + } } diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/ITestDiscovererCapabilities.cs b/src/Microsoft.TestPlatform.Common/Interfaces/ITestDiscovererCapabilities.cs index a3b87c4152..c5bca677f4 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/ITestDiscovererCapabilities.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/ITestDiscovererCapabilities.cs @@ -27,4 +27,10 @@ public interface ITestDiscovererCapabilities /// Assembly type that the test discoverer supports. /// AssemblyType AssemblyType { get; } + + /// + /// true if the discoverer plugin is decorated with , + /// false otherwise. + /// + bool IsDirectoryBased { get; } } diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt index 7dc5c58110..c95ceedda6 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ITestDiscovererCapabilities.IsDirectoryBased.get -> bool diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs index 92a85f57ca..aab0cdb74e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs @@ -371,32 +371,62 @@ private static void SetAdapterLoggingSettings(IMessageLogger messageLogger, IRun var result = new Dictionary, IEnumerable>(); var sourcesForWhichNoDiscovererIsAvailable = new List(sources); + sources = sources.Distinct().ToList(); + IEnumerable allDirectoryBasedSources = sources.Where(Directory.Exists).ToList(); + IEnumerable allFileBasedSources = sources.Except(allDirectoryBasedSources).ToList(); + foreach (var discoverer in allDiscoverers) { - var sourcesToCheck = sources; - + var applicableFileBasedSources = allFileBasedSources; if (discoverer.Metadata.AssemblyType is AssemblyType.Native or AssemblyType.Managed) { - assemblyTypeToSoucesMap ??= GetAssemblyTypeToSoucesMap(sources, assemblyProperties); - sourcesToCheck = assemblyTypeToSoucesMap[AssemblyType.None].Concat(assemblyTypeToSoucesMap[discoverer.Metadata.AssemblyType]); + assemblyTypeToSoucesMap ??= GetAssemblyTypeToSoucesMap(applicableFileBasedSources, assemblyProperties); + applicableFileBasedSources = assemblyTypeToSoucesMap[AssemblyType.None].Concat(assemblyTypeToSoucesMap[discoverer.Metadata.AssemblyType]); } // Find the sources which this discoverer can look at. - // Based on whether it is registered for a matching file extension or no file extensions at all. - var matchingSources = (from source in sourcesToCheck - where - (discoverer.Metadata.FileExtension == null - || discoverer.Metadata.FileExtension.Contains( - Path.GetExtension(source), - StringComparer.OrdinalIgnoreCase)) - select source).ToList(); // ToList is required to actually execute the query + var matchingSources = Enumerable.Empty(); + var discovererFileExtensions = discoverer.Metadata.FileExtension; + var discovererIsApplicableToFiles = discovererFileExtensions is not null; + var discovererIsApplicableToDirectories = discoverer.Metadata.IsDirectoryBased; + + if (!discovererIsApplicableToFiles && !discovererIsApplicableToDirectories) + { + // Discoverer is applicable for all sources (regardless of whether they are files or directories). + // Include all files and directories. + matchingSources = applicableFileBasedSources.Concat(allDirectoryBasedSources); + } + else + { + if (discovererIsApplicableToFiles) + { + // Include matching files. + var matchingFileBasedSources = + applicableFileBasedSources.Where(source => + discovererFileExtensions!.Contains( + Path.GetExtension(source), + StringComparer.OrdinalIgnoreCase)); + + matchingSources = matchingSources.Concat(matchingFileBasedSources); + } + + if (discovererIsApplicableToDirectories) + { + // Include all directories. + matchingSources = matchingSources.Concat(allDirectoryBasedSources); + } + } + + matchingSources = matchingSources.ToList(); // ToList is required to actually execute the query // Update the source list for which no matching source is available. if (matchingSources.Any()) { sourcesForWhichNoDiscovererIsAvailable = - sourcesForWhichNoDiscovererIsAvailable.Except(matchingSources, StringComparer.OrdinalIgnoreCase) + sourcesForWhichNoDiscovererIsAvailable + .Except(matchingSources, StringComparer.OrdinalIgnoreCase) .ToList(); + result.Add(discoverer, matchingSources); } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs index 2d87b7b320..7651367a15 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs @@ -238,7 +238,7 @@ private void OnReportTestCases(ICollection testCases) /// /// Verify/Normalize the test source files. /// - /// Paths to source file to look for tests in. + /// Paths to source file (or directory) in which to look for tests. /// logger /// package /// The list of verified sources. @@ -255,7 +255,7 @@ internal static HashSet GetValidSources(IEnumerable? sources, IM // It is possible that runtime provider sent relative source path for remote scenario. string src = !Path.IsPathRooted(source) ? Path.Combine(Directory.GetCurrentDirectory(), source) : source; - if (!File.Exists(src)) + if (!File.Exists(src) && !Directory.Exists(src)) { void SendWarning() { diff --git a/src/Microsoft.TestPlatform.ObjectModel/Adapter/Interfaces/ITestDiscoverer.cs b/src/Microsoft.TestPlatform.ObjectModel/Adapter/Interfaces/ITestDiscoverer.cs index 1083a71535..795cd67f72 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Adapter/Interfaces/ITestDiscoverer.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Adapter/Interfaces/ITestDiscoverer.cs @@ -8,11 +8,21 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; /// -/// Interface implemented to provide tests to the test platform. A class that -// implements this interface will be available for use if its containing -// assembly is either placed in the Extensions folder or is marked as a 'UnitTestExtension' type -// in the vsix package. +/// Interface implemented to provide tests to the test platform. /// +/// +/// +/// A class that implements this interface will be available for use if its containing assembly is either placed in +/// the Extensions folder or is marked as a 'UnitTestExtension' type in the vsix package. +/// +/// +/// Provide one or more s on the implementing class to indicate the set of file +/// extensions that are supported for test discovery. If the discoverer supports discovering tests present inside +/// directories, provide instead. If neither +/// nor is provided, the +/// discoverer will be called for all relevant test files and directories. +/// +/// public interface ITestDiscoverer { /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/DirectoryBasedDiscovererAttribute.cs b/src/Microsoft.TestPlatform.ObjectModel/DirectoryBasedDiscovererAttribute.cs new file mode 100644 index 0000000000..e5995647dd --- /dev/null +++ b/src/Microsoft.TestPlatform.ObjectModel/DirectoryBasedDiscovererAttribute.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; + +/// +/// This attribute is applied to s. It indicates the test discoverer discovers tests +/// present inside a directory (as opposed to the which indicates that the +/// discoverer discovers tests present in files with a specified extension). +/// +/// +/// If neither this attribute nor the is provided on the test discoverer, +/// it will be called for all relevant test files and directories. +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public sealed class DirectoryBasedTestDiscovererAttribute : Attribute +{ +} diff --git a/src/Microsoft.TestPlatform.ObjectModel/FileExtensionAttribute.cs b/src/Microsoft.TestPlatform.ObjectModel/FileExtensionAttribute.cs index d66525d7f3..c2874d62e2 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/FileExtensionAttribute.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/FileExtensionAttribute.cs @@ -3,14 +3,18 @@ using System; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + namespace Microsoft.VisualStudio.TestPlatform.ObjectModel; /// -/// This attribute is applied to ITestDiscoverers. It indicates -/// which file extensions the test discoverer knows how to process. -/// If this attribute is not provided on the test discoverer it will be -/// called for all file types. +/// This attribute is applied to s. It indicates that the discoverer discovers tests +/// present in files with the specified extension. /// +/// +/// If neither this attribute nor the is provided on the test +/// discoverer, it will be called for all relevant test files and directories. +/// [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public sealed class FileExtensionAttribute : Attribute { diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt index ab058de62d..053259ecf8 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +Microsoft.VisualStudio.TestPlatform.ObjectModel.DirectoryBasedTestDiscovererAttribute +Microsoft.VisualStudio.TestPlatform.ObjectModel.DirectoryBasedTestDiscovererAttribute.DirectoryBasedTestDiscovererAttribute() -> void diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestDiscoveryExtensionManagerTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestDiscoveryExtensionManagerTests.cs index 216bf9e494..796ee6b2b8 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestDiscoveryExtensionManagerTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestDiscoveryExtensionManagerTests.cs @@ -84,6 +84,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenFileExtensionsIsNull() var metadata = new TestDiscovererMetadata(null, null); Assert.IsNull(metadata.FileExtension); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -92,6 +93,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenFileExtensionsIsEmpty() var metadata = new TestDiscovererMetadata(new List(), null); Assert.IsNull(metadata.FileExtension); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -100,6 +102,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenDefaultUriIsNull() var metadata = new TestDiscovererMetadata(new List(), null); Assert.IsNull(metadata.DefaultExecutorUri); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -108,6 +111,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenDefaultUriIsEmpty() var metadata = new TestDiscovererMetadata(new List(), " "); Assert.IsNull(metadata.DefaultExecutorUri); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -117,6 +121,7 @@ public void TestDiscovererMetadataCtorSetsFileExtensions() var metadata = new TestDiscovererMetadata(extensions, null); CollectionAssert.AreEqual(extensions, metadata.FileExtension!.ToList()); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -125,6 +130,7 @@ public void TestDiscovererMetadataCtorSetsDefaultUri() var metadata = new TestDiscovererMetadata(null, "executor://helloworld"); Assert.AreEqual("executor://helloworld/", metadata.DefaultExecutorUri!.AbsoluteUri); + Assert.IsFalse(metadata.IsDirectoryBased); } [TestMethod] @@ -133,5 +139,14 @@ public void TestDiscovererMetadataCtorSetsAssemblyType() var metadata = new TestDiscovererMetadata(null, "executor://helloworld", AssemblyType.Native); Assert.AreEqual(AssemblyType.Native, metadata.AssemblyType); + Assert.IsFalse(metadata.IsDirectoryBased); + } + + [TestMethod] + public void TestDiscovererMetadataCtorSetsIsDirectoryBased() + { + var metadata = new TestDiscovererMetadata(null, "executor://helloworld", isDirectoryBased: true); + + Assert.IsTrue(metadata.IsDirectoryBased); } } diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/LazyExtensionTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/LazyExtensionTests.cs index 28cb1041f5..97a1a975b2 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/LazyExtensionTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/LazyExtensionTests.cs @@ -94,6 +94,7 @@ public void MetadataShouldCreateMetadataFromMetadataType() CollectionAssert.AreEqual(new List { "csv" }, metadata.FileExtension!.ToArray()); Assert.AreEqual("executor://unittestexecutor/", metadata.DefaultExecutorUri!.AbsoluteUri); Assert.AreEqual(AssemblyType.Native, metadata.AssemblyType); + Assert.IsFalse(metadata.IsDirectoryBased); } #endregion @@ -120,15 +121,21 @@ public AssemblyType AssemblyType private set; } - public DummyDiscovererCapability(List fileExtensions, string executorUri, AssemblyType assemblyType) + public bool IsDirectoryBased + { + get; + private set; + } + + public DummyDiscovererCapability(List fileExtensions, string executorUri, AssemblyType assemblyType, bool isDirectoryBased) { FileExtension = fileExtensions; DefaultExecutorUri = new Uri(executorUri); AssemblyType = assemblyType; + IsDirectoryBased = isDirectoryBased; } } - [FileExtension("csv")] [DefaultExecutorUri("executor://unittestexecutor")] [Category("native")] diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/TestDiscovererPluginInformationTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/TestDiscovererPluginInformationTests.cs index 2233c13ba6..6b61aa6568 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/TestDiscovererPluginInformationTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/TestDiscovererPluginInformationTests.cs @@ -38,6 +38,7 @@ public void FileExtensionsShouldReturnEmptyListIfADiscovererSupportsNoFileExtens _testPluginInformation = new TestDiscovererPluginInformation(typeof(DummyTestDiscovererWithNoFileExtensions)); Assert.IsNotNull(_testPluginInformation.FileExtensions); Assert.AreEqual(0, _testPluginInformation.FileExtensions.Count); + Assert.IsFalse(_testPluginInformation.IsDirectoryBased); } [TestMethod] @@ -45,6 +46,7 @@ public void FileExtensionsShouldReturnAFileExtensionForADiscoverer() { _testPluginInformation = new TestDiscovererPluginInformation(typeof(DummyTestDiscovererWithOneFileExtensions)); CollectionAssert.AreEqual(new List { "csv" }, _testPluginInformation.FileExtensions); + Assert.IsFalse(_testPluginInformation.IsDirectoryBased); } [TestMethod] @@ -52,6 +54,7 @@ public void FileExtensionsShouldReturnSupportedFileExtensionsForADiscoverer() { _testPluginInformation = new TestDiscovererPluginInformation(typeof(DummyTestDiscovererWithTwoFileExtensions)); CollectionAssert.AreEqual(new List { "csv", "docx" }, _testPluginInformation.FileExtensions); + Assert.IsFalse(_testPluginInformation.IsDirectoryBased); } [TestMethod] @@ -129,6 +132,36 @@ public void MetadataShouldReturnFileExtensionsAndDefaultExecutorUriAndAssemblyTy CollectionAssert.AreEqual(expectedFileExtensions, ((List)testPluginMetada[0]!).ToArray()); Assert.AreEqual("csvexecutor", testPluginMetada[1] as string); Assert.AreEqual(AssemblyType.Managed, Enum.Parse(typeof(AssemblyType), testPluginMetada[2]!.ToString()!)); + Assert.IsFalse(bool.Parse(testPluginMetada[3]!.ToString()!)); + } + + [TestMethod] + public void IsDirectoryBasedShouldReturnTrueIfDiscovererIsDirectoryBased() + { + _testPluginInformation = new TestDiscovererPluginInformation(typeof(DummyDirectoryBasedTestDiscoverer)); + var testPluginMetada = _testPluginInformation.Metadata.ToArray(); + + Assert.IsNotNull(_testPluginInformation.FileExtensions); + Assert.AreEqual(0, _testPluginInformation.FileExtensions.Count); + Assert.IsNotNull(testPluginMetada[0]); + Assert.AreEqual(0, ((List)testPluginMetada[0]!).Count); + + Assert.IsTrue(_testPluginInformation.IsDirectoryBased); + Assert.IsTrue(bool.Parse(testPluginMetada[3]!.ToString()!)); + } + + [TestMethod] + public void FileExtensionsAndIsDirectroyBasedShouldReturnCorrectValuesWhenBothAreSupported() + { + _testPluginInformation = new TestDiscovererPluginInformation(typeof(DummyDirectoryBasedTestDiscovererWithFileExtensions)); + var testPluginMetada = _testPluginInformation.Metadata.ToArray(); + var expectedFileExtensions = new List { "csv", "docx" }; + + CollectionAssert.AreEqual(expectedFileExtensions, _testPluginInformation.FileExtensions); + CollectionAssert.AreEqual(expectedFileExtensions, ((List)testPluginMetada[0]!)); + + Assert.IsTrue(_testPluginInformation.IsDirectoryBased); + Assert.IsTrue(bool.Parse(testPluginMetada[3]!.ToString()!)); } } @@ -193,4 +226,15 @@ public class DummyTestDiscovererWithTwoFileExtensions { } +[DirectoryBasedTestDiscoverer] +public class DummyDirectoryBasedTestDiscoverer +{ +} + +[DirectoryBasedTestDiscoverer] +[FileExtension("csv")] +[FileExtension("docx")] +public class DummyDirectoryBasedTestDiscovererWithFileExtensions +{ +} #endregion diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscovererEnumeratorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscovererEnumeratorTests.cs index 9978da8ae2..20c276ee28 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscovererEnumeratorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscovererEnumeratorTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.IO; using System.Linq; using System.Reflection; using System.Threading; @@ -61,6 +62,9 @@ public void Cleanup() NativeDllTestDiscoverer.Reset(); JsonTestDiscoverer.Reset(); NotImplementedTestDiscoverer.Reset(); + EverythingTestDiscoverer.Reset(); + DirectoryTestDiscoverer.Reset(); + DirectoryAndFileTestDiscoverer.Reset(); } [TestMethod] @@ -98,8 +102,14 @@ public void LoadTestsShouldNotCallIntoDiscoverersIfNoneMatchesSources() _discovererEnumerator.LoadTests(extensionSourceMap, _runSettingsMock.Object, null, _messageLoggerMock.Object); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + CollectionAssert.AreEqual(sources, EverythingTestDiscoverer.Sources!.ToList()); + Assert.IsFalse(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsFalse(NativeDllTestDiscoverer.IsNativeDiscoverTestCalled); + Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); } [TestMethod] @@ -128,8 +138,13 @@ public void LoadTestsShouldCallOnlyNativeDiscovererIfNativeAssembliesPassed() Assert.IsTrue(NativeDllTestDiscoverer.IsNativeDiscoverTestCalled); CollectionAssert.AreEqual(sources, NativeDllTestDiscoverer.Sources!.ToList()); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + CollectionAssert.AreEqual(sources, EverythingTestDiscoverer.Sources!.ToList()); + Assert.IsFalse(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); } [TestMethod] @@ -158,8 +173,13 @@ public void LoadTestsShouldCallOnlyManagedDiscovererIfManagedAssembliesPassed() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); CollectionAssert.AreEqual(sources, ManagedDllTestDiscoverer.Sources!.ToList()); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + CollectionAssert.AreEqual(sources, EverythingTestDiscoverer.Sources!.ToList()); + Assert.IsFalse(NativeDllTestDiscoverer.IsNativeDiscoverTestCalled); Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); } [TestMethod] @@ -196,7 +216,13 @@ public void LoadTestsShouldCallBothNativeAndManagedDiscoverersWithCorrectSources Assert.IsTrue(NativeDllTestDiscoverer.IsNativeDiscoverTestCalled); CollectionAssert.AreEqual(nativeSources, NativeDllTestDiscoverer.Sources!.ToList()); + var allSources = nativeSources.Concat(managedSources).OrderBy(source => source).ToList(); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + CollectionAssert.AreEqual(allSources, EverythingTestDiscoverer.Sources!.OrderBy(source => source).ToList()); + Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); } [TestMethod] @@ -222,13 +248,22 @@ public void LoadTestsShouldCallIntoADiscovererThatMatchesTheSources() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); // Also validate that the right set of arguments were passed on to the discoverer. - CollectionAssert.AreEqual(sources, ManagedDllTestDiscoverer.Sources!.ToList()); + CollectionAssert.AreEqual(sources.Distinct().ToList(), ManagedDllTestDiscoverer.Sources!.ToList()); Assert.AreEqual(_runSettingsMock.Object, DllTestDiscoverer.DiscoveryContext!.RunSettings); Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DllTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, DllTestDiscoverer.MessageLogger); Assert.IsNotNull(DllTestDiscoverer.DiscoverySink); + + CollectionAssert.AreEqual(sources.Distinct().ToList(), EverythingTestDiscoverer.Sources!.ToList()); + Assert.AreEqual(_runSettingsMock.Object, EverythingTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)EverythingTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, EverythingTestDiscoverer.MessageLogger); + Assert.IsNotNull(EverythingTestDiscoverer.DiscoverySink); } [TestMethod] @@ -243,13 +278,23 @@ public void LoadTestsShouldCallIntoMultipleDiscoverersThatMatchesTheSources() typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location, typeof(DiscoveryResultCacheTests).GetTypeInfo().Assembly.Location }; + var jsonsources = new List { "test1.json", "test2.json" }; + + var currentDirectory = Directory.GetCurrentDirectory(); + var directorySources = new List + { + currentDirectory, + Path.GetDirectoryName(currentDirectory)! + }; + var sources = new List(dllsources); sources.AddRange(jsonsources); + sources.AddRange(directorySources); var extensionSourceMap = new Dictionary> { @@ -264,9 +309,12 @@ public void LoadTestsShouldCallIntoMultipleDiscoverersThatMatchesTheSources() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsTrue(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); // Also validate that the right set of arguments were passed on to the discoverer. - CollectionAssert.AreEqual(dllsources, ManagedDllTestDiscoverer.Sources!.ToList()); + CollectionAssert.AreEqual(dllsources.Distinct().ToList(), ManagedDllTestDiscoverer.Sources!.ToList()); Assert.AreEqual(runSettings, DllTestDiscoverer.DiscoveryContext!.RunSettings); Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DllTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, DllTestDiscoverer.MessageLogger); @@ -277,6 +325,26 @@ public void LoadTestsShouldCallIntoMultipleDiscoverersThatMatchesTheSources() Assert.AreEqual(testCaseFilter, ((DiscoveryContext)JsonTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, JsonTestDiscoverer.MessageLogger); Assert.IsNotNull(JsonTestDiscoverer.DiscoverySink); + + var allSources = sources.Distinct().OrderBy(source => source).ToList(); + CollectionAssert.AreEqual(allSources, EverythingTestDiscoverer.Sources!.OrderBy(source => source).ToList()); + Assert.AreEqual(runSettings, EverythingTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)EverythingTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, EverythingTestDiscoverer.MessageLogger); + Assert.IsNotNull(EverythingTestDiscoverer.DiscoverySink); + + CollectionAssert.AreEqual(directorySources, DirectoryTestDiscoverer.Sources!.ToList()); + Assert.AreEqual(runSettings, DirectoryTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DirectoryTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, DirectoryTestDiscoverer.MessageLogger); + Assert.IsNotNull(DirectoryTestDiscoverer.DiscoverySink); + + var jsonAndDirectorySources = jsonsources.Concat(directorySources).OrderBy(source => source).ToList(); + CollectionAssert.AreEqual(jsonAndDirectorySources, DirectoryAndFileTestDiscoverer.Sources!.OrderBy(source => source).ToList()); + Assert.AreEqual(runSettings, DirectoryAndFileTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DirectoryAndFileTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, DirectoryAndFileTestDiscoverer.MessageLogger); + Assert.IsNotNull(DirectoryAndFileTestDiscoverer.DiscoverySink); } [TestMethod] @@ -305,6 +373,9 @@ public void LoadTestsShouldCallIntoOtherDiscoverersWhenCreatingOneFails() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsFalse(SingletonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); // Also validate that the right set of arguments were passed on to the discoverer. CollectionAssert.AreEqual(new List { sources[1] }, ManagedDllTestDiscoverer.Sources!.ToList()); @@ -312,6 +383,12 @@ public void LoadTestsShouldCallIntoOtherDiscoverersWhenCreatingOneFails() Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DllTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, DllTestDiscoverer.MessageLogger); Assert.IsNotNull(DllTestDiscoverer.DiscoverySink); + + CollectionAssert.AreEqual(sources.ToList(), EverythingTestDiscoverer.Sources!.ToList()); + Assert.AreEqual(runSettings, EverythingTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)EverythingTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, EverythingTestDiscoverer.MessageLogger); + Assert.IsNotNull(EverythingTestDiscoverer.DiscoverySink); } [TestMethod] @@ -340,6 +417,9 @@ public void LoadTestsShouldCallIntoOtherDiscoverersEvenIfDiscoveryInOneFails() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsTrue(NotImplementedTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); // Also validate that the right set of arguments were passed on to the discoverer. CollectionAssert.AreEqual(new List { sources[1] }, ManagedDllTestDiscoverer.Sources!.ToList()); @@ -348,6 +428,12 @@ public void LoadTestsShouldCallIntoOtherDiscoverersEvenIfDiscoveryInOneFails() Assert.AreEqual(_messageLoggerMock.Object, DllTestDiscoverer.MessageLogger); Assert.IsNotNull(DllTestDiscoverer.DiscoverySink); + CollectionAssert.AreEqual(sources.ToList(), EverythingTestDiscoverer.Sources!.ToList()); + Assert.AreEqual(runSettings, EverythingTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)EverythingTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, EverythingTestDiscoverer.MessageLogger); + Assert.IsNotNull(EverythingTestDiscoverer.DiscoverySink); + // Check if we log the failure. var message = $"An exception occurred while test discoverer '{typeof(NotImplementedTestDiscoverer).Name}' was loading tests. Exception: The method or operation is not implemented."; @@ -429,6 +515,10 @@ public void LoadTestsShouldNotCallIntoDiscoverersWhenCancelled() // Validate Assert.IsFalse(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsFalse(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); + _messageLoggerMock.Verify(logger => logger.SendMessage(TestMessageLevel.Warning, "Discovery of tests cancelled."), Times.Once); } @@ -511,9 +601,12 @@ public void LoadTestsShouldIterateOverAllExtensionsInTheMapAndDiscoverTests() Assert.IsTrue(ManagedDllTestDiscoverer.IsManagedDiscoverTestCalled); Assert.IsTrue(JsonTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(EverythingTestDiscoverer.IsDiscoverTestCalled); + Assert.IsTrue(DirectoryAndFileTestDiscoverer.IsDiscoverTestCalled); + Assert.IsFalse(DirectoryTestDiscoverer.IsDiscoverTestCalled); // Also validate that the right set of arguments were passed on to the discoverer. - CollectionAssert.AreEqual(dllsources, ManagedDllTestDiscoverer.Sources!.ToList()); + CollectionAssert.AreEqual(dllsources.Distinct().ToList(), ManagedDllTestDiscoverer.Sources!.ToList()); Assert.AreEqual(runSettings, DllTestDiscoverer.DiscoveryContext!.RunSettings); Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DllTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, DllTestDiscoverer.MessageLogger); @@ -524,6 +617,19 @@ public void LoadTestsShouldIterateOverAllExtensionsInTheMapAndDiscoverTests() Assert.AreEqual(testCaseFilter, ((DiscoveryContext)JsonTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); Assert.AreEqual(_messageLoggerMock.Object, JsonTestDiscoverer.MessageLogger); Assert.IsNotNull(JsonTestDiscoverer.DiscoverySink); + + var allSources = jsonsources.Concat(dllsources).Distinct().OrderBy(source => source).ToList(); + CollectionAssert.AreEqual(allSources, EverythingTestDiscoverer.Sources!.OrderBy(source => source).ToList()); + Assert.AreEqual(runSettings, EverythingTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)EverythingTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, EverythingTestDiscoverer.MessageLogger); + Assert.IsNotNull(EverythingTestDiscoverer.DiscoverySink); + + CollectionAssert.AreEqual(jsonsources, DirectoryAndFileTestDiscoverer.Sources!.ToList()); + Assert.AreEqual(runSettings, DirectoryAndFileTestDiscoverer.DiscoveryContext!.RunSettings); + Assert.AreEqual(testCaseFilter, ((DiscoveryContext)DirectoryAndFileTestDiscoverer.DiscoveryContext).FilterExpressionWrapper!.FilterString); + Assert.AreEqual(_messageLoggerMock.Object, DirectoryAndFileTestDiscoverer.MessageLogger); + Assert.IsNotNull(DirectoryAndFileTestDiscoverer.DiscoverySink); } [TestMethod] @@ -768,5 +874,98 @@ public static void Reset() } } + [DefaultExecutorUri("discoverer://everythingdiscoverer")] + private class EverythingTestDiscoverer : ITestDiscoverer + { + public static bool IsDiscoverTestCalled { get; private set; } + + public static IEnumerable? Sources { get; private set; } + + public static IDiscoveryContext? DiscoveryContext { get; private set; } + + public static IMessageLogger? MessageLogger { get; private set; } + + public static ITestCaseDiscoverySink? DiscoverySink { get; private set; } + + public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, + ITestCaseDiscoverySink discoverySink) + { + IsDiscoverTestCalled = true; + Sources = Sources is null ? sources : Sources.Concat(sources); + DiscoveryContext = discoveryContext; + MessageLogger = logger; + DiscoverySink = discoverySink; + } + + public static void Reset() + { + IsDiscoverTestCalled = false; + Sources = null; + } + } + + [DirectoryBasedTestDiscoverer] + [DefaultExecutorUri("discoverer://dirdiscoverer")] + private class DirectoryTestDiscoverer : ITestDiscoverer + { + public static bool IsDiscoverTestCalled { get; private set; } + + public static IEnumerable? Sources { get; private set; } + + public static IDiscoveryContext? DiscoveryContext { get; private set; } + + public static IMessageLogger? MessageLogger { get; private set; } + + public static ITestCaseDiscoverySink? DiscoverySink { get; private set; } + + public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, + ITestCaseDiscoverySink discoverySink) + { + IsDiscoverTestCalled = true; + Sources = Sources is null ? sources : Sources.Concat(sources); + DiscoveryContext = discoveryContext; + MessageLogger = logger; + DiscoverySink = discoverySink; + } + + public static void Reset() + { + IsDiscoverTestCalled = false; + Sources = null; + } + } + + [DirectoryBasedTestDiscoverer] + [FileExtension(".json")] + [DefaultExecutorUri("discoverer://dirandfilediscoverer")] + private class DirectoryAndFileTestDiscoverer : ITestDiscoverer + { + public static bool IsDiscoverTestCalled { get; private set; } + + public static IEnumerable? Sources { get; private set; } + + public static IDiscoveryContext? DiscoveryContext { get; private set; } + + public static IMessageLogger? MessageLogger { get; private set; } + + public static ITestCaseDiscoverySink? DiscoverySink { get; private set; } + + public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, + ITestCaseDiscoverySink discoverySink) + { + IsDiscoverTestCalled = true; + Sources = Sources is null ? sources : Sources.Concat(sources); + DiscoveryContext = discoveryContext; + MessageLogger = logger; + DiscoverySink = discoverySink; + } + + public static void Reset() + { + IsDiscoverTestCalled = false; + Sources = null; + } + } + #endregion }