From 244bd36293ca21c50de853d54107fc3ecfef503a Mon Sep 17 00:00:00 2001
From: Niko Schuessler <54370830+nschuessler@users.noreply.github.com>
Date: Mon, 10 Aug 2020 12:50:08 -0700
Subject: [PATCH 1/4] Add diagnostic verification extensibility.

---
 .../AnalyzerTest`1.cs                            | 16 ++++++++++++++++
 .../PublicAPI.Unshipped.txt                      |  1 +
 2 files changed, 17 insertions(+)

diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
index 90c321006..ab68c1763 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
@@ -239,6 +239,20 @@ protected async Task VerifyDiagnosticsAsync((string filename, SourceText content
             await VerifySuppressionDiagnosticsAsync(analyzers, sources, additionalFiles, additionalProjects, additionalMetadataReferences, expected, verifier, cancellationToken).ConfigureAwait(false);
         }
 
+        /// <summary>
+        /// Checks each of the actual <see cref="Diagnostic"/>s found and compares them with the corresponding
+        /// <see cref="DiagnosticResult"/> based on custom test requirements not yet supported by the test framework.
+        /// </summary>
+        /// <param name="analyzers">The analyzers that have been run on the sources.</param>
+        /// <param name="actual">The <see cref="Diagnostic"/>s found by the compiler after running the analyzer
+        /// on the source code.</param>
+        /// <param name="expected">The <see cref="DiagnosticResult"/>s describing the expected
+        /// diagnostics for the sources.</param>
+        /// <param name="verifier">The verifier to use for test assertions.</param>
+        protected virtual void VerifyDiagnosticCustom(ImmutableArray<DiagnosticAnalyzer> analyzers, Diagnostic actual, DiagnosticResult expected, IVerifier verifier)
+        {
+        }
+
         private async Task VerifyGeneratedCodeDiagnosticsAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, (string filename, SourceText content)[] sources, (string filename, SourceText content)[] additionalFiles, ProjectState[] additionalProjects, MetadataReference[] additionalMetadataReferences, DiagnosticResult[] expected, IVerifier verifier, CancellationToken cancellationToken)
         {
             if (TestBehaviors.HasFlag(TestBehaviors.SkipGeneratedCodeCheck)
@@ -371,6 +385,8 @@ private void VerifyDiagnosticResults(IEnumerable<Diagnostic> actualResults, Immu
                         StringComparer.Ordinal,
                         message);
                 }
+
+                VerifyDiagnosticCustom(analyzers, actual, expected, verifier);
             }
         }
 
diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
index a02895c9a..ebc621a47 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
@@ -294,6 +294,7 @@ virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.IsCompilerDiagnos
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetAnalyzerOptions(Microsoft.CodeAnalysis.Project project) -> Microsoft.CodeAnalysis.Diagnostics.AnalyzerOptions
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetDefaultDiagnostic(Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer[] analyzers) -> Microsoft.CodeAnalysis.DiagnosticDescriptor
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.RunAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task
+virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.VerifyDiagnosticCustom(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Diagnostic actual, Microsoft.CodeAnalysis.Testing.DiagnosticResult expected, Microsoft.CodeAnalysis.Testing.IVerifier verifier) -> void
 virtual Microsoft.CodeAnalysis.Testing.CodeActionTest<TVerifier>.FilterCodeActions(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction> actions) -> System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction>
 virtual Microsoft.CodeAnalysis.Testing.DefaultVerifier.CreateMessage(string message) -> string
 virtual Microsoft.CodeAnalysis.Testing.DefaultVerifier.Empty<T>(string collectionName, System.Collections.Generic.IEnumerable<T> collection) -> void

From 67f31e031378be09d778c802cd6d73f322540ad9 Mon Sep 17 00:00:00 2001
From: Niko Schuessler <54370830+nschuessler@users.noreply.github.com>
Date: Mon, 10 Aug 2020 15:09:24 -0700
Subject: [PATCH 2/4] Update API based on review feedback.

---
 .../AnalyzerTest`1.cs                         | 26 ++++++++-----------
 .../PublicAPI.Unshipped.txt                   |  3 ++-
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
index ab68c1763..b0ad1d8de 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
@@ -133,8 +133,18 @@ public string TestCode
         /// </summary>
         public List<string> DisabledDiagnostics { get; } = new List<string>();
 
+        /// <summary>
+        /// Gets or sets the reference assemblies to use.
+        /// </summary>
         public ReferenceAssemblies ReferenceAssemblies { get; set; } = ReferenceAssemblies.Default;
 
+        /// <summary>
+        /// Gets or sets an additional verifier for a diagnostic.
+        /// The action compares actual <see cref="Diagnostic"/> and the expected
+        /// <see cref="DiagnosticResult"/> based on custom test requirements not yet supported by the test framework.
+        /// </summary>
+        public Action<ImmutableArray<DiagnosticAnalyzer>, Diagnostic, DiagnosticResult, IVerifier>? DiagnosticVerifier { get; set; }
+
         /// <summary>
         /// Gets a collection of transformation functions to apply to <see cref="Workspace.Options"/> during diagnostic
         /// or code fix test setup.
@@ -239,20 +249,6 @@ protected async Task VerifyDiagnosticsAsync((string filename, SourceText content
             await VerifySuppressionDiagnosticsAsync(analyzers, sources, additionalFiles, additionalProjects, additionalMetadataReferences, expected, verifier, cancellationToken).ConfigureAwait(false);
         }
 
-        /// <summary>
-        /// Checks each of the actual <see cref="Diagnostic"/>s found and compares them with the corresponding
-        /// <see cref="DiagnosticResult"/> based on custom test requirements not yet supported by the test framework.
-        /// </summary>
-        /// <param name="analyzers">The analyzers that have been run on the sources.</param>
-        /// <param name="actual">The <see cref="Diagnostic"/>s found by the compiler after running the analyzer
-        /// on the source code.</param>
-        /// <param name="expected">The <see cref="DiagnosticResult"/>s describing the expected
-        /// diagnostics for the sources.</param>
-        /// <param name="verifier">The verifier to use for test assertions.</param>
-        protected virtual void VerifyDiagnosticCustom(ImmutableArray<DiagnosticAnalyzer> analyzers, Diagnostic actual, DiagnosticResult expected, IVerifier verifier)
-        {
-        }
-
         private async Task VerifyGeneratedCodeDiagnosticsAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, (string filename, SourceText content)[] sources, (string filename, SourceText content)[] additionalFiles, ProjectState[] additionalProjects, MetadataReference[] additionalMetadataReferences, DiagnosticResult[] expected, IVerifier verifier, CancellationToken cancellationToken)
         {
             if (TestBehaviors.HasFlag(TestBehaviors.SkipGeneratedCodeCheck)
@@ -386,7 +382,7 @@ private void VerifyDiagnosticResults(IEnumerable<Diagnostic> actualResults, Immu
                         message);
                 }
 
-                VerifyDiagnosticCustom(analyzers, actual, expected, verifier);
+                DiagnosticVerifier?.Invoke(analyzers, actual, expected, verifier);
             }
         }
 
diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
index ebc621a47..adb1b4822 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
@@ -3,6 +3,8 @@ Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.AnalyzerTest() -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.get -> Microsoft.CodeAnalysis.Testing.CompilerDiagnostics
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CreateProjectAsync((string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] sources, (string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] additionalFiles, Microsoft.CodeAnalysis.Testing.ProjectState[] additionalProjects, Microsoft.CodeAnalysis.MetadataReference[] additionalMetadataReferences, string language, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<Microsoft.CodeAnalysis.Project>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.get -> System.Action<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer>, Microsoft.CodeAnalysis.Diagnostic, Microsoft.CodeAnalysis.Testing.DiagnosticResult, Microsoft.CodeAnalysis.Testing.IVerifier>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DisabledDiagnostics.get -> System.Collections.Generic.List<string>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.ExpectedDiagnostics.get -> System.Collections.Generic.List<Microsoft.CodeAnalysis.Testing.DiagnosticResult>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.FormatVerifierMessage(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Diagnostic actual, Microsoft.CodeAnalysis.Testing.DiagnosticResult expected, string message) -> string
@@ -294,7 +296,6 @@ virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.IsCompilerDiagnos
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetAnalyzerOptions(Microsoft.CodeAnalysis.Project project) -> Microsoft.CodeAnalysis.Diagnostics.AnalyzerOptions
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetDefaultDiagnostic(Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer[] analyzers) -> Microsoft.CodeAnalysis.DiagnosticDescriptor
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.RunAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task
-virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.VerifyDiagnosticCustom(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Diagnostic actual, Microsoft.CodeAnalysis.Testing.DiagnosticResult expected, Microsoft.CodeAnalysis.Testing.IVerifier verifier) -> void
 virtual Microsoft.CodeAnalysis.Testing.CodeActionTest<TVerifier>.FilterCodeActions(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction> actions) -> System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction>
 virtual Microsoft.CodeAnalysis.Testing.DefaultVerifier.CreateMessage(string message) -> string
 virtual Microsoft.CodeAnalysis.Testing.DefaultVerifier.Empty<T>(string collectionName, System.Collections.Generic.IEnumerable<T> collection) -> void

From 0a98b506c2237e8d6ce4045a554cd57992f12915 Mon Sep 17 00:00:00 2001
From: Niko Schuessler <54370830+nschuessler@users.noreply.github.com>
Date: Mon, 10 Aug 2020 18:07:45 -0700
Subject: [PATCH 3/4] Remove analyzers argument.

---
 .../Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
index b0ad1d8de..d1eac023a 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs
@@ -143,7 +143,7 @@ public string TestCode
         /// The action compares actual <see cref="Diagnostic"/> and the expected
         /// <see cref="DiagnosticResult"/> based on custom test requirements not yet supported by the test framework.
         /// </summary>
-        public Action<ImmutableArray<DiagnosticAnalyzer>, Diagnostic, DiagnosticResult, IVerifier>? DiagnosticVerifier { get; set; }
+        public Action<Diagnostic, DiagnosticResult, IVerifier>? DiagnosticVerifier { get; set; }
 
         /// <summary>
         /// Gets a collection of transformation functions to apply to <see cref="Workspace.Options"/> during diagnostic
@@ -382,7 +382,7 @@ private void VerifyDiagnosticResults(IEnumerable<Diagnostic> actualResults, Immu
                         message);
                 }
 
-                DiagnosticVerifier?.Invoke(analyzers, actual, expected, verifier);
+                DiagnosticVerifier?.Invoke(actual, expected, verifier);
             }
         }
 

From 1fe7f624441e91bae6f133795cef6cca93123af1 Mon Sep 17 00:00:00 2001
From: Niko Schuessler <54370830+nschuessler@users.noreply.github.com>
Date: Mon, 10 Aug 2020 18:29:09 -0700
Subject: [PATCH 4/4] Update public api signatures.

---
 .../PublicAPI.Unshipped.txt                                     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
index adb1b4822..38d425db6 100644
--- a/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
+++ b/src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt
@@ -3,7 +3,7 @@ Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.AnalyzerTest() -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.get -> Microsoft.CodeAnalysis.Testing.CompilerDiagnostics
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CreateProjectAsync((string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] sources, (string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] additionalFiles, Microsoft.CodeAnalysis.Testing.ProjectState[] additionalProjects, Microsoft.CodeAnalysis.MetadataReference[] additionalMetadataReferences, string language, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<Microsoft.CodeAnalysis.Project>
-Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.get -> System.Action<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer>, Microsoft.CodeAnalysis.Diagnostic, Microsoft.CodeAnalysis.Testing.DiagnosticResult, Microsoft.CodeAnalysis.Testing.IVerifier>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.get -> System.Action<Microsoft.CodeAnalysis.Diagnostic, Microsoft.CodeAnalysis.Testing.DiagnosticResult, Microsoft.CodeAnalysis.Testing.IVerifier>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DisabledDiagnostics.get -> System.Collections.Generic.List<string>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.ExpectedDiagnostics.get -> System.Collections.Generic.List<Microsoft.CodeAnalysis.Testing.DiagnosticResult>