Fix validation of diagnostics in source generator tests

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs

@@ -966,7 +966,13 @@ private static bool IsInSourceFile(DiagnosticResult result, (string filename, So
         private async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(EvaluatedProjectState primaryProject, ImmutableArray<EvaluatedProjectState> additionalProjects, ImmutableArray<DiagnosticAnalyzer> analyzers, IVerifier verifier, CancellationToken cancellationToken)
         {
             var solution = await GetSolutionAsync(primaryProject, additionalProjects, verifier, cancellationToken);
-            return await GetSortedDiagnosticsAsync(solution, analyzers, CompilerDiagnostics, cancellationToken);
+            var additionalDiagnostics = primaryProject.AdditionalDiagnostics;
+            foreach (var project in additionalProjects)
+            {
+                additionalDiagnostics = additionalDiagnostics.AddRange(project.AdditionalDiagnostics);
+            }
+
+            return await GetSortedDiagnosticsAsync(solution, analyzers, additionalDiagnostics, CompilerDiagnostics, cancellationToken);
         }
 
         /// <summary>
@@ -975,11 +981,12 @@ private async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Evaluat
         /// </summary>
         /// <param name="solution">The <see cref="Solution"/> that the analyzer(s) will be run on.</param>
         /// <param name="analyzers">The analyzer to run on the documents.</param>
+        /// <param name="additionalDiagnostics">Additional diagnostics reported for the solution, which need to be verified.</param>
         /// <param name="compilerDiagnostics">The behavior of compiler diagnostics in validation scenarios.</param>
         /// <param name="cancellationToken">The <see cref="CancellationToken"/> that the task will observe.</param>
         /// <returns>A collection of <see cref="Diagnostic"/>s that surfaced in the source code, sorted by
         /// <see cref="Diagnostic.Location"/>.</returns>
-        protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solution solution, ImmutableArray<DiagnosticAnalyzer> analyzers, CompilerDiagnostics compilerDiagnostics, CancellationToken cancellationToken)
+        protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solution solution, ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<Diagnostic> additionalDiagnostics, CompilerDiagnostics compilerDiagnostics, CancellationToken cancellationToken)
         {
             var diagnostics = ImmutableArray.CreateBuilder<Diagnostic>();
             foreach (var project in solution.Projects)
@@ -991,6 +998,7 @@ protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solut
                 diagnostics.AddRange(allDiagnostics.Where(diagnostic => !IsCompilerDiagnostic(diagnostic) || IsCompilerDiagnosticIncluded(diagnostic, compilerDiagnostics)));
             }
 
+            diagnostics.AddRange(additionalDiagnostics);
             var results = SortDistinctDiagnostics(diagnostics);
             return results.ToImmutableArray();
         }

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/Model/EvaluatedProjectState.cs

@@ -23,7 +23,8 @@ public EvaluatedProjectState(ProjectState state, ReferenceAssemblies defaultRefe
                 state.Sources.ToImmutableArray(),
                 state.AdditionalFiles.ToImmutableArray(),
                 state.AdditionalProjectReferences.ToImmutableArray(),
-                state.AdditionalReferences.ToImmutableArray())
+                state.AdditionalReferences.ToImmutableArray(),
+                ImmutableArray<Diagnostic>.Empty)
         {
         }
 
@@ -37,7 +38,8 @@ private EvaluatedProjectState(
             ImmutableArray<(string filename, SourceText content)> sources,
             ImmutableArray<(string filename, SourceText content)> additionalFiles,
             ImmutableArray<string> additionalProjectReferences,
-            ImmutableArray<MetadataReference> additionalReferences)
+            ImmutableArray<MetadataReference> additionalReferences,
+            ImmutableArray<Diagnostic> additionalDiagnostics)
         {
             Name = name;
             AssemblyName = assemblyName;
@@ -49,6 +51,7 @@ private EvaluatedProjectState(
             AdditionalFiles = additionalFiles;
             AdditionalProjectReferences = additionalProjectReferences;
             AdditionalReferences = additionalReferences;
+            AdditionalDiagnostics = additionalDiagnostics;
         }
 
         public string Name { get; }
@@ -71,6 +74,8 @@ private EvaluatedProjectState(
 
         public ImmutableArray<MetadataReference> AdditionalReferences { get; }
 
+        public ImmutableArray<Diagnostic> AdditionalDiagnostics { get; }
+
         public EvaluatedProjectState WithSources(ImmutableArray<(string filename, SourceText content)> sources)
         {
             if (sources == Sources)
@@ -81,6 +86,16 @@ public EvaluatedProjectState WithSources(ImmutableArray<(string filename, Source
             return With(sources: sources);
         }
 
+        public EvaluatedProjectState WithAdditionalDiagnostics(ImmutableArray<Diagnostic> additionalDiagnostics)
+        {
+            if (additionalDiagnostics == AdditionalDiagnostics)
+            {
+                return this;
+            }
+
+            return With(additionalDiagnostics: additionalDiagnostics);
+        }
+
         private EvaluatedProjectState With(
             Optional<string> name = default,
             Optional<string> assemblyName = default,
@@ -91,7 +106,8 @@ private EvaluatedProjectState With(
             Optional<ImmutableArray<(string filename, SourceText content)>> sources = default,
             Optional<ImmutableArray<(string filename, SourceText content)>> additionalFiles = default,
             Optional<ImmutableArray<string>> additionalProjectReferences = default,
-            Optional<ImmutableArray<MetadataReference>> additionalReferences = default)
+            Optional<ImmutableArray<MetadataReference>> additionalReferences = default,
+            Optional<ImmutableArray<Diagnostic>> additionalDiagnostics = default)
         {
             return new EvaluatedProjectState(
                 GetValueOrDefault(name, Name),
@@ -103,7 +119,8 @@ private EvaluatedProjectState With(
                 GetValueOrDefault(sources, Sources),
                 GetValueOrDefault(additionalFiles, AdditionalFiles),
                 GetValueOrDefault(additionalProjectReferences, AdditionalProjectReferences),
-                GetValueOrDefault(additionalReferences, AdditionalReferences));
+                GetValueOrDefault(additionalReferences, AdditionalReferences),
+                GetValueOrDefault(additionalDiagnostics, AdditionalDiagnostics));
         }
 
         private static T GetValueOrDefault<T>(Optional<T> optionalValue, T defaultValue)

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt

@@ -8,7 +8,7 @@ Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.set ->
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DisabledDiagnostics.get -> System.Collections.Generic.List<string>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.ExpectedDiagnostics.get -> System.Collections.Generic.List<Microsoft.CodeAnalysis.Testing.DiagnosticResult>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.FormatVerifierMessage(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Diagnostic actual, Microsoft.CodeAnalysis.Testing.DiagnosticResult expected, string message) -> string
-Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetSortedDiagnosticsAsync(Microsoft.CodeAnalysis.Solution solution, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetSortedDiagnosticsAsync(Microsoft.CodeAnalysis.Solution solution, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic> additionalDiagnostics, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MarkupOptions.get -> Microsoft.CodeAnalysis.Testing.MarkupOptions
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MarkupOptions.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MatchDiagnosticsTimeout.get -> System.TimeSpan
@@ -119,6 +119,7 @@ Microsoft.CodeAnalysis.Testing.MetadataReferenceCollection.Add(string path) -> v
 Microsoft.CodeAnalysis.Testing.MetadataReferenceCollection.MetadataReferenceCollection() -> void
 Microsoft.CodeAnalysis.Testing.MetadataReferences
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState
+Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.AdditionalDiagnostics.get -> System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.AdditionalFiles.get -> System.Collections.Immutable.ImmutableArray<(string filename, Microsoft.CodeAnalysis.Text.SourceText content)>
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.AdditionalProjectReferences.get -> System.Collections.Immutable.ImmutableArray<string>
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.AdditionalReferences.get -> System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.MetadataReference>
@@ -130,6 +131,7 @@ Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.Name.get -> string
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.OutputKind.get -> Microsoft.CodeAnalysis.OutputKind
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.ReferenceAssemblies.get -> Microsoft.CodeAnalysis.Testing.ReferenceAssemblies
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.Sources.get -> System.Collections.Immutable.ImmutableArray<(string filename, Microsoft.CodeAnalysis.Text.SourceText content)>
+Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.WithAdditionalDiagnostics(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic> additionalDiagnostics) -> Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState
 Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState.WithSources(System.Collections.Immutable.ImmutableArray<(string filename, Microsoft.CodeAnalysis.Text.SourceText content)> sources) -> Microsoft.CodeAnalysis.Testing.Model.EvaluatedProjectState
 Microsoft.CodeAnalysis.Testing.PackageIdentity
 Microsoft.CodeAnalysis.Testing.PackageIdentity.Id.get -> string

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt

@@ -6,7 +6,7 @@ Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>
 Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.FixedCode.set -> void
 Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.FixedState.get -> Microsoft.CodeAnalysis.Testing.SolutionState
 Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.SourceGeneratorTest() -> void
-Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.VerifySourceGeneratorAsync(Microsoft.CodeAnalysis.Testing.SolutionState testState, Microsoft.CodeAnalysis.Testing.SolutionState fixedState, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task
+Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.VerifySourceGeneratorAsync(Microsoft.CodeAnalysis.Testing.SolutionState testState, Microsoft.CodeAnalysis.Testing.SolutionState fixedState, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>>
 Microsoft.CodeAnalysis.Testing.SourceGeneratorVerifier<TSourceGenerator, TTest, TVerifier>
 Microsoft.CodeAnalysis.Testing.SourceGeneratorVerifier<TSourceGenerator, TTest, TVerifier>.SourceGeneratorVerifier() -> void
 abstract Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.CreateGeneratorDriver(Microsoft.CodeAnalysis.Project project, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.ISourceGenerator> sourceGenerators) -> Microsoft.CodeAnalysis.GeneratorDriver

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs

@@ -42,6 +42,9 @@ public string FixedCode
         protected SourceGeneratorTest()
         {
             FixedState = new SolutionState(DefaultTestProjectName, Language, DefaultFilePathPrefix, DefaultFileExt);
+
+            // Disable test behaviors that don't make sense for source generator scenarios
+            TestBehaviors |= TestBehaviors.SkipSuppressionCheck | TestBehaviors.SkipGeneratedCodeCheck;
         }
 
         protected override IEnumerable<DiagnosticAnalyzer> GetDiagnosticAnalyzers()
@@ -71,8 +74,8 @@ public override async Task RunAsync(CancellationToken cancellationToken = defaul
             var fixedState = rawFixedState.WithProcessedMarkup(MarkupOptions, defaultDiagnostic, supportedDiagnostics, fixableDiagnostics, DefaultFilePath);
 
             await VerifyDiagnosticsAsync(new EvaluatedProjectState(testState, ReferenceAssemblies), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), testState.ExpectedDiagnostics.ToArray(), Verify.PushContext("Diagnostics of test state"), cancellationToken).ConfigureAwait(false);
-            await VerifyDiagnosticsAsync(new EvaluatedProjectState(fixedState, ReferenceAssemblies), fixedState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), fixedState.ExpectedDiagnostics.ToArray(), Verify.PushContext("Diagnostics of fixed state"), cancellationToken).ConfigureAwait(false);
-            await VerifySourceGeneratorAsync(testState, fixedState, Verify, cancellationToken).ConfigureAwait(false);
+            var diagnostics = await VerifySourceGeneratorAsync(testState, fixedState, Verify, cancellationToken).ConfigureAwait(false);
+            await VerifyDiagnosticsAsync(new EvaluatedProjectState(fixedState, ReferenceAssemblies).WithAdditionalDiagnostics(diagnostics), fixedState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), fixedState.ExpectedDiagnostics.ToArray(), Verify.PushContext("Diagnostics of test state with generated sources"), cancellationToken).ConfigureAwait(false);
         }
 
         /// <summary>
@@ -83,12 +86,12 @@ public override async Task RunAsync(CancellationToken cancellationToken = defaul
         /// <param name="verifier">The verifier to use for test assertions.</param>
         /// <param name="cancellationToken">The <see cref="CancellationToken"/> that the task will observe.</param>
         /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
-        protected async Task VerifySourceGeneratorAsync(SolutionState testState, SolutionState fixedState, IVerifier verifier, CancellationToken cancellationToken)
+        protected async Task<ImmutableArray<Diagnostic>> VerifySourceGeneratorAsync(SolutionState testState, SolutionState fixedState, IVerifier verifier, CancellationToken cancellationToken)
         {
-            await VerifySourceGeneratorAsync(Language, GetSourceGenerators().ToImmutableArray(), testState, fixedState, ApplySourceGeneratorAsync, verifier.PushContext("Source generator application"), cancellationToken);
+            return await VerifySourceGeneratorAsync(Language, GetSourceGenerators().ToImmutableArray(), testState, fixedState, ApplySourceGeneratorAsync, verifier.PushContext("Source generator application"), cancellationToken);
         }
 
-        private async Task VerifySourceGeneratorAsync(
+        private async Task<ImmutableArray<Diagnostic>> VerifySourceGeneratorAsync(
             string language,
             ImmutableArray<ISourceGenerator> sourceGenerators,
             SolutionState oldState,
@@ -129,6 +132,8 @@ private async Task VerifySourceGeneratorAsync(
                 verifier.Equal(newState.AdditionalFiles[i].content.ChecksumAlgorithm, actual.ChecksumAlgorithm, $"checksum algorithm of '{newState.AdditionalFiles[i].filename}' was expected to be '{newState.AdditionalFiles[i].content.ChecksumAlgorithm}' but was '{actual.ChecksumAlgorithm}'");
                 verifier.Equal(newState.AdditionalFiles[i].filename, updatedAdditionalDocuments[i].Name, $"file name was expected to be '{newState.AdditionalFiles[i].filename}' but was '{updatedAdditionalDocuments[i].Name}'");
             }
+
+            return diagnostics;
         }
 
         private async Task<(Project project, ImmutableArray<Diagnostic> diagnostics)> ApplySourceGeneratorAsync(ImmutableArray<ISourceGenerator> sourceGenerators, Project project, IVerifier verifier, CancellationToken cancellationToken)

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing/VisualBasicSourceGeneratorTest`2.vb

@@ -22,7 +22,11 @@ Public Class VisualBasicSourceGeneratorTest(Of TSourceGenerator As {ISourceGener
     End Property
 
     Protected Overrides Function CreateGeneratorDriver(project As Project, sourceGenerators As ImmutableArray(Of ISourceGenerator)) As GeneratorDriver
-        Return VisualBasicGeneratorDriver.Create(sourceGenerators, project.AnalyzerOptions.AdditionalFiles, CType(project.ParseOptions, VisualBasicParseOptions), project.AnalyzerOptions.AnalyzerConfigOptionsProvider)
+        Return VisualBasicGeneratorDriver.Create(
+            sourceGenerators,
+            project.AnalyzerOptions.AdditionalFiles,
+            CType(project.ParseOptions, VisualBasicParseOptions),
+            project.AnalyzerOptions.AnalyzerConfigOptionsProvider)
     End Function
 
     Protected Overrides Function CreateCompilationOptions() As CompilationOptions

tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs

@@ -38,6 +38,34 @@ public async Task AddSimpleFile()
             }.RunAsync();
         }
 
+        [Fact]
+        public async Task AddSimpleFileWithDiagnostic()
+        {
+            await new CSharpSourceGeneratorTest<AddEmptyFileWithDiagnostic>
+            {
+                TestState =
+                {
+                    Sources =
+                    {
+                        @"// Comment",
+                    },
+                },
+                FixedState =
+                {
+                    Sources =
+                    {
+                        @"{|#0:|}// Comment",
+                        ("Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\\Microsoft.CodeAnalysis.Testing.TestGenerators.AddEmptyFileWithDiagnostic\\EmptyGeneratedFile.cs", SourceText.From(string.Empty, Encoding.UTF8)),
+                    },
+                    ExpectedDiagnostics =
+                    {
+                        // /0/Test0.cs(1,1): warning SG0001: Message
+                        new DiagnosticResult(AddEmptyFileWithDiagnostic.Descriptor).WithLocation(0),
+                    },
+                },
+            }.RunAsync();
+        }
+
         private class CSharpSourceGeneratorTest<TSourceGenerator> : SourceGeneratorTest<DefaultVerifier>
             where TSourceGenerator : ISourceGenerator, new()
         {

tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddEmptyFile.cs

@@ -4,13 +4,13 @@
 
 namespace Microsoft.CodeAnalysis.Testing.TestGenerators
 {
-    public sealed class AddEmptyFile : ISourceGenerator
+    public class AddEmptyFile : ISourceGenerator
     {
         public void Initialize(GeneratorInitializationContext context)
         {
         }
 
-        public void Execute(GeneratorExecutionContext context)
+        public virtual void Execute(GeneratorExecutionContext context)
         {
             context.AddSource("EmptyGeneratedFile", string.Empty);
         }