Support test discovery in sources that are directories

src/Microsoft.TestPlatform.Common/ExtensionFramework/TestDiscoveryExtensionManager.cs

@@ -141,7 +141,7 @@ internal class TestDiscovererMetadata : ITestDiscovererCapabilities
     /// </summary>
     /// <param name="fileExtensions"> The file Extensions. </param>
     /// <param name="defaultExecutorUri"> The default Executor Uri. </param>
-    public TestDiscovererMetadata(IReadOnlyCollection<string>? fileExtensions, string? defaultExecutorUri, AssemblyType assemblyType = default)
+    public TestDiscovererMetadata(IReadOnlyCollection<string>? fileExtensions, string? defaultExecutorUri, AssemblyType assemblyType = default, bool isDirectoryBased = false)
     {
         if (fileExtensions != null && fileExtensions.Count > 0)
         {
@@ -154,6 +154,7 @@ public TestDiscovererMetadata(IReadOnlyCollection<string>? fileExtensions, strin
         }
 
         AssemblyType = assemblyType;
+        IsDirectoryBased = isDirectoryBased;
     }
 
     /// <summary>
@@ -182,4 +183,14 @@ public AssemblyType AssemblyType
         get;
         private set;
     }
+
+    /// <summary>
+    /// <c>true</c> if the discoverer plugin is decorated with <see cref="DirectoryBasedTestDiscovererAttribute"/>,
+    /// <c>false</c> otherwise.
+    /// </summary>
+    public bool IsDirectoryBased
+    {
+        get;
+        private set;
+    }
 }

src/Microsoft.TestPlatform.Common/ExtensionFramework/Utilities/TestDiscovererPluginInformation.cs

@@ -29,6 +29,7 @@ public TestDiscovererPluginInformation(Type testDiscovererType)
             FileExtensions = GetFileExtensions(testDiscovererType);
             DefaultExecutorUri = GetDefaultExecutorUri(testDiscovererType);
             AssemblyType = GetAssemblyType(testDiscovererType);
+            IsDirectoryBased = GetIsDirectoryBased(testDiscovererType);
         }
     }
 
@@ -39,7 +40,7 @@ public override ICollection<object?> Metadata
     {
         get
         {
-            return new object?[] { FileExtensions, DefaultExecutorUri, AssemblyType };
+            return new object?[] { FileExtensions, DefaultExecutorUri, AssemblyType, IsDirectoryBased };
         }
     }
 
@@ -71,15 +72,25 @@ public AssemblyType AssemblyType
     }
 
     /// <summary>
-    /// Helper to get file extensions from the FileExtensionAttribute on the discover plugin.
+    /// <c>true</c> if the discoverer plugin is decorated with <see cref="DirectoryBasedTestDiscovererAttribute"/>,
+    /// <c>false</c> otherwise.
     /// </summary>
-    /// <param name="testDicovererType">Data type of the test discoverer</param>
+    public bool IsDirectoryBased
+    {
+        get;
+        private set;
+    }
+
+    /// <summary>
+    /// Helper to get file extensions from the <see cref="FileExtensionAttribute"/> on the discover plugin.
+    /// </summary>
+    /// <param name="testDiscovererType">Data type of the test discoverer</param>
     /// <returns>List of file extensions</returns>
-    private static List<string> GetFileExtensions(Type testDicovererType)
+    private static List<string> GetFileExtensions(Type testDiscovererType)
     {
         var fileExtensions = new List<string>();
 
-        var attributes = testDicovererType.GetTypeInfo().GetCustomAttributes(typeof(FileExtensionAttribute), false).ToArray();
+        var attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(FileExtensionAttribute), inherit: false).ToArray();
         if (attributes != null && attributes.Length > 0)
         {
             foreach (var attribute in attributes)
@@ -96,15 +107,15 @@ private static List<string> GetFileExtensions(Type testDicovererType)
     }
 
     /// <summary>
-    /// Returns the value of default executor Uri on this type. 'Null' if not present.
+    /// Returns the value of default executor Uri on this type. <c>null</c> if not present.
     /// </summary>
     /// <param name="testDiscovererType"> The test discoverer Type. </param>
     /// <returns> The default executor URI. </returns>
     private static string GetDefaultExecutorUri(Type testDiscovererType)
     {
-        string result = string.Empty;
+        var result = string.Empty;
 
-        object[] attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(DefaultExecutorUriAttribute), false).ToArray();
+        var attributes = testDiscovererType.GetTypeInfo().GetCustomAttributes(typeof(DefaultExecutorUriAttribute), inherit: false).ToArray();
         if (attributes != null && attributes.Length > 0)
         {
             DefaultExecutorUriAttribute executorUriAttribute = (DefaultExecutorUriAttribute)attributes[0];
@@ -119,7 +130,7 @@ private static string GetDefaultExecutorUri(Type testDiscovererType)
     }
 
     /// <summary>
-    /// Helper to get the supported assembly type from the CategoryAttribute on the discover plugin.
+    /// Helper to get the supported assembly type from the <see cref="CategoryAttribute"/> on the discover plugin.
     /// </summary>
     /// <param name="testDiscovererType"> The test discoverer Type. </param>
     /// <returns> Supported assembly type. </returns>
@@ -134,4 +145,15 @@ private static AssemblyType GetAssemblyType(Type testDiscovererType)
         Enum.TryParse(category, true, out AssemblyType assemblyType);
         return assemblyType;
     }
+
+    /// <summary>
+    /// Returns <c>true</c> if the discoverer plugin is decorated with
+    /// <see cref="DirectoryBasedTestDiscovererAttribute"/>, <c>false</c> otherwise.
+    /// </summary>
+    /// <param name="testDiscovererType">Data type of the test discoverer</param>
+    private static bool GetIsDirectoryBased(Type testDiscovererType)
+    {
+        var attribute = testDiscovererType.GetTypeInfo().GetCustomAttribute(typeof(DirectoryBasedTestDiscovererAttribute), inherit: false);
+        return attribute is DirectoryBasedTestDiscovererAttribute;
+    }
 }

src/Microsoft.TestPlatform.Common/Interfaces/ITestDiscovererCapabilities.cs

@@ -27,4 +27,10 @@ public interface ITestDiscovererCapabilities
     /// Assembly type that the test discoverer supports.
     /// </summary>
     AssemblyType AssemblyType { get; }
+
+    /// <summary>
+    /// <c>true</c> if the discoverer plugin is decorated with <see cref="DirectoryBasedTestDiscovererAttribute"/>,
+    /// <c>false</c> otherwise.
+    /// </summary>
+    bool IsDirectoryBased { get; }
 }

src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt

@@ -1 +1,2 @@
 #nullable enable
+Microsoft.VisualStudio.TestPlatform.Common.Interfaces.ITestDiscovererCapabilities.IsDirectoryBased.get -> bool

src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscovererEnumerator.cs

@@ -371,32 +371,62 @@ private static void SetAdapterLoggingSettings(IMessageLogger messageLogger, IRun
         var result = new Dictionary<LazyExtension<ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable<string>>();
         var sourcesForWhichNoDiscovererIsAvailable = new List<string>(sources);
 
+        sources = sources.Distinct().ToList();
+        IEnumerable<string> allDirectoryBasedSources = sources.Where(Directory.Exists).ToList();
+        IEnumerable<string> allFileBasedSources = sources.Except(allDirectoryBasedSources).ToList();
+
         foreach (var discoverer in allDiscoverers)
         {
-            var sourcesToCheck = sources;
-
+            var applicableFileBasedSources = allFileBasedSources;
             if (discoverer.Metadata.AssemblyType is AssemblyType.Native or AssemblyType.Managed)
             {
-                assemblyTypeToSoucesMap ??= GetAssemblyTypeToSoucesMap(sources, assemblyProperties);
-                sourcesToCheck = assemblyTypeToSoucesMap[AssemblyType.None].Concat(assemblyTypeToSoucesMap[discoverer.Metadata.AssemblyType]);
+                assemblyTypeToSoucesMap ??= GetAssemblyTypeToSoucesMap(applicableFileBasedSources, assemblyProperties);
+                applicableFileBasedSources = assemblyTypeToSoucesMap[AssemblyType.None].Concat(assemblyTypeToSoucesMap[discoverer.Metadata.AssemblyType]);
             }
 
             // Find the sources which this discoverer can look at.
-            // Based on whether it is registered for a matching file extension or no file extensions at all.
-            var matchingSources = (from source in sourcesToCheck
-                                   where
-                                       (discoverer.Metadata.FileExtension == null
-                                        || discoverer.Metadata.FileExtension.Contains(
-                                            Path.GetExtension(source),
-                                            StringComparer.OrdinalIgnoreCase))
-                                   select source).ToList(); // ToList is required to actually execute the query
+            var matchingSources = Enumerable.Empty<string>();
+            var discovererFileExtensions = discoverer.Metadata.FileExtension;
+            var discovererIsApplicableToFiles = discovererFileExtensions is not null;
+            var discovererIsApplicableToDirectories = discoverer.Metadata.IsDirectoryBased;
+
+            if (!discovererIsApplicableToFiles && !discovererIsApplicableToDirectories)
+            {
+                // Discoverer is applicable for all sources (regardless of whether they are files or directories).
+                // Include all files and directories.
+                matchingSources = applicableFileBasedSources.Concat(allDirectoryBasedSources);
+            }
+            else
+            {
+                if (discovererIsApplicableToFiles)
+                {
+                    // Include matching files.
+                    var matchingFileBasedSources =
+                        applicableFileBasedSources.Where(source =>
+                            discovererFileExtensions!.Contains(
+                                Path.GetExtension(source),
+                                StringComparer.OrdinalIgnoreCase));
+
+                    matchingSources = matchingSources.Concat(matchingFileBasedSources);
+                }
+
+                if (discovererIsApplicableToDirectories)
+                {
+                    // Include all directories.
+                    matchingSources = matchingSources.Concat(allDirectoryBasedSources);
+                }
+            }
+
+            matchingSources = matchingSources.ToList(); // ToList is required to actually execute the query
 
             // Update the source list for which no matching source is available.
             if (matchingSources.Any())
             {
                 sourcesForWhichNoDiscovererIsAvailable =
-                    sourcesForWhichNoDiscovererIsAvailable.Except(matchingSources, StringComparer.OrdinalIgnoreCase)
+                    sourcesForWhichNoDiscovererIsAvailable
+                        .Except(matchingSources, StringComparer.OrdinalIgnoreCase)
                         .ToList();
+
                 result.Add(discoverer, matchingSources);
             }
         }

src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs

@@ -238,7 +238,7 @@ private void OnReportTestCases(ICollection<TestCase> testCases)
     /// <summary>
     /// Verify/Normalize the test source files.
     /// </summary>
-    /// <param name="sources"> Paths to source file to look for tests in.  </param>
+    /// <param name="sources"> Paths to source file (or directory) in which to look for tests. </param>
     /// <param name="logger">logger</param>
     /// <param name="package">package</param>
     /// <returns> The list of verified sources. </returns>
@@ -255,7 +255,7 @@ internal static HashSet<string> GetValidSources(IEnumerable<string>? sources, IM
             // It is possible that runtime provider sent relative source path for remote scenario.
             string src = !Path.IsPathRooted(source) ? Path.Combine(Directory.GetCurrentDirectory(), source) : source;
 
-            if (!File.Exists(src))
+            if (!File.Exists(src) && !Directory.Exists(src))
             {
                 void SendWarning()
                 {

src/Microsoft.TestPlatform.ObjectModel/Adapter/Interfaces/ITestDiscoverer.cs

@@ -8,11 +8,21 @@
 namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 
 /// <summary>
-/// Interface implemented to provide tests to the test platform.  A class that
-//  implements this interface will be available for use if its containing
-//  assembly is either placed in the Extensions folder or is marked as a 'UnitTestExtension' type
-//  in the vsix package.
+/// Interface implemented to provide tests to the test platform.
 /// </summary>
+/// <remarks>
+/// <para>
+/// A class that implements this interface will be available for use if its containing assembly is either placed in
+/// the Extensions folder or is marked as a 'UnitTestExtension' type in the vsix package.
+/// </para>
+/// <para>
+/// Provide one or more <see cref="FileExtensionAttribute"/>s on the implementing class to indicate the set of file
+/// extensions that are supported for test discovery. If the discoverer supports discovering tests present inside
+/// directories, provide <see cref="DirectoryBasedTestDiscovererAttribute"/> instead. If neither
+/// <see cref="DirectoryBasedTestDiscovererAttribute"/> nor <see cref="FileExtensionAttribute"/> is provided, the
+/// discoverer will be called for all relevant test files and directories.
+/// </para>
+/// </remarks>
 public interface ITestDiscoverer
 {
     /// <summary>

src/Microsoft.TestPlatform.ObjectModel/DirectoryBasedDiscovererAttribute.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+
+namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+/// <summary>
+/// This attribute is applied to <see cref="ITestDiscoverer"/>s. It indicates the test discoverer discovers tests
+/// present inside a directory (as opposed to the <see cref="FileExtensionAttribute"/> which indicates that the
+/// discoverer discovers tests present in files with a specified extension).
+/// </summary>
+/// <remarks>
+/// If neither this attribute nor the <see cref="FileExtensionAttribute"/> is provided on the test discoverer,
+/// it will be called for all relevant test files and directories.
+/// </remarks>
+[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
+public sealed class DirectoryBasedTestDiscovererAttribute : Attribute
+{
+}

src/Microsoft.TestPlatform.ObjectModel/FileExtensionAttribute.cs

@@ -3,14 +3,18 @@
 
 using System;
 
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+
 namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
 /// <summary>
-/// This attribute is applied to ITestDiscoverers.  It indicates
-/// which file extensions the test discoverer knows how to process.
-/// If this attribute is not provided on the test discoverer it will be
-/// called for all file types.
+/// This attribute is applied to <see cref="ITestDiscoverer"/>s. It indicates that the discoverer discovers tests
+/// present in files with the specified extension.
 /// </summary>
+/// <remarks>
+/// If neither this attribute nor the <see cref="DirectoryBasedTestDiscovererAttribute"/> is provided on the test
+/// discoverer, it will be called for all relevant test files and directories.
+/// </remarks>
 [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)]
 public sealed class FileExtensionAttribute : Attribute
 {

src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt

@@ -1 +1,3 @@
 #nullable enable
+Microsoft.VisualStudio.TestPlatform.ObjectModel.DirectoryBasedTestDiscovererAttribute
+Microsoft.VisualStudio.TestPlatform.ObjectModel.DirectoryBasedTestDiscovererAttribute.DirectoryBasedTestDiscovererAttribute() -> void

test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/TestDiscoveryExtensionManagerTests.cs

@@ -84,6 +84,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenFileExtensionsIsNull()
         var metadata = new TestDiscovererMetadata(null, null);
 
         Assert.IsNull(metadata.FileExtension);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -92,6 +93,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenFileExtensionsIsEmpty()
         var metadata = new TestDiscovererMetadata(new List<string>(), null);
 
         Assert.IsNull(metadata.FileExtension);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -100,6 +102,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenDefaultUriIsNull()
         var metadata = new TestDiscovererMetadata(new List<string>(), null);
 
         Assert.IsNull(metadata.DefaultExecutorUri);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -108,6 +111,7 @@ public void TestDiscovererMetadataCtorDoesNotThrowWhenDefaultUriIsEmpty()
         var metadata = new TestDiscovererMetadata(new List<string>(), " ");
 
         Assert.IsNull(metadata.DefaultExecutorUri);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -117,6 +121,7 @@ public void TestDiscovererMetadataCtorSetsFileExtensions()
         var metadata = new TestDiscovererMetadata(extensions, null);
 
         CollectionAssert.AreEqual(extensions, metadata.FileExtension!.ToList());
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -125,6 +130,7 @@ public void TestDiscovererMetadataCtorSetsDefaultUri()
         var metadata = new TestDiscovererMetadata(null, "executor://helloworld");
 
         Assert.AreEqual("executor://helloworld/", metadata.DefaultExecutorUri!.AbsoluteUri);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     [TestMethod]
@@ -133,5 +139,14 @@ public void TestDiscovererMetadataCtorSetsAssemblyType()
         var metadata = new TestDiscovererMetadata(null, "executor://helloworld", AssemblyType.Native);
 
         Assert.AreEqual(AssemblyType.Native, metadata.AssemblyType);
+        Assert.IsFalse(metadata.IsDirectoryBased);
+    }
+
+    [TestMethod]
+    public void TestDiscovererMetadataCtorSetsIsDirectoryBased()
+    {
+        var metadata = new TestDiscovererMetadata(null, "executor://helloworld", isDirectoryBased: true);
+
+        Assert.IsTrue(metadata.IsDirectoryBased);
     }
 }

test/Microsoft.TestPlatform.Common.UnitTests/ExtensionFramework/Utilities/LazyExtensionTests.cs

@@ -94,6 +94,7 @@ public void MetadataShouldCreateMetadataFromMetadataType()
         CollectionAssert.AreEqual(new List<string> { "csv" }, metadata.FileExtension!.ToArray());
         Assert.AreEqual("executor://unittestexecutor/", metadata.DefaultExecutorUri!.AbsoluteUri);
         Assert.AreEqual(AssemblyType.Native, metadata.AssemblyType);
+        Assert.IsFalse(metadata.IsDirectoryBased);
     }
 
     #endregion
@@ -120,15 +121,21 @@ public AssemblyType AssemblyType
             private set;
         }
 
-        public DummyDiscovererCapability(List<string> fileExtensions, string executorUri, AssemblyType assemblyType)
+        public bool IsDirectoryBased
+        {
+            get;
+            private set;
+        }
+
+        public DummyDiscovererCapability(List<string> fileExtensions, string executorUri, AssemblyType assemblyType, bool isDirectoryBased)
         {
             FileExtension = fileExtensions;
             DefaultExecutorUri = new Uri(executorUri);
             AssemblyType = assemblyType;
+            IsDirectoryBased = isDirectoryBased;
         }
     }
 
-
     [FileExtension("csv")]
     [DefaultExecutorUri("executor://unittestexecutor")]
     [Category("native")]