Fix failure to validate compiler errors in generated sources

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs

@@ -978,7 +978,7 @@ private async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Evaluat
                 additionalDiagnostics = additionalDiagnostics.AddRange(project.AdditionalDiagnostics);
             }
 
-            return await GetSortedDiagnosticsAsync(solution, analyzers, additionalDiagnostics, CompilerDiagnostics, cancellationToken);
+            return await GetSortedDiagnosticsAsync(solution, analyzers, additionalDiagnostics, CompilerDiagnostics, verifier, cancellationToken);
         }
 
         /// <summary>
@@ -989,10 +989,11 @@ private async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Evaluat
         /// <param name="analyzers">The analyzer to run on the documents.</param>
         /// <param name="additionalDiagnostics">Additional diagnostics reported for the solution, which need to be verified.</param>
         /// <param name="compilerDiagnostics">The behavior of compiler diagnostics in validation scenarios.</param>
+        /// <param name="verifier">The verifier to use for test assertions.</param>
         /// <param name="cancellationToken">The <see cref="CancellationToken"/> that the task will observe.</param>
         /// <returns>A collection of <see cref="Diagnostic"/>s that surfaced in the source code, sorted by
         /// <see cref="Diagnostic.Location"/>.</returns>
-        protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solution solution, ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<Diagnostic> additionalDiagnostics, CompilerDiagnostics compilerDiagnostics, CancellationToken cancellationToken)
+        protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solution solution, ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<Diagnostic> additionalDiagnostics, CompilerDiagnostics compilerDiagnostics, IVerifier verifier, CancellationToken cancellationToken)
         {
             if (analyzers.IsEmpty)
             {
@@ -1002,7 +1003,7 @@ protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solut
             var diagnostics = ImmutableArray.CreateBuilder<Diagnostic>();
             foreach (var project in solution.Projects)
             {
-                var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
+                var compilation = await GetProjectCompilationAsync(project, verifier, cancellationToken).ConfigureAwait(false);
                 var compilationWithAnalyzers = compilation.WithAnalyzers(analyzers, GetAnalyzerOptions(project), cancellationToken);
                 var allDiagnostics = await compilationWithAnalyzers.GetAllDiagnosticsAsync().ConfigureAwait(false);
 
@@ -1014,6 +1015,11 @@ protected async Task<ImmutableArray<Diagnostic>> GetSortedDiagnosticsAsync(Solut
             return results.ToImmutableArray();
         }
 
+        protected virtual Task<Compilation> GetProjectCompilationAsync(Project project, IVerifier verifier, CancellationToken cancellationToken)
+        {
+            return project.GetCompilationAsync(cancellationToken);
+        }
+
         private static bool IsCompilerDiagnostic(Diagnostic diagnostic)
         {
             return diagnostic.Descriptor.CustomTags.Contains(WellKnownDiagnosticTags.Compiler);

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt

@@ -8,7 +8,7 @@ Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DiagnosticVerifier.set ->
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DisabledDiagnostics.get -> System.Collections.Generic.List<string>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.ExpectedDiagnostics.get -> System.Collections.Generic.List<Microsoft.CodeAnalysis.Testing.DiagnosticResult>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.FormatVerifierMessage(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, Microsoft.CodeAnalysis.Diagnostic actual, Microsoft.CodeAnalysis.Testing.DiagnosticResult expected, string message) -> string
-Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetSortedDiagnosticsAsync(Microsoft.CodeAnalysis.Solution solution, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic> additionalDiagnostics, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetSortedDiagnosticsAsync(Microsoft.CodeAnalysis.Solution solution, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer> analyzers, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic> additionalDiagnostics, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.Diagnostic>>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MarkupOptions.get -> Microsoft.CodeAnalysis.Testing.MarkupOptions
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MarkupOptions.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.MatchDiagnosticsTimeout.get -> System.TimeSpan
@@ -326,6 +326,7 @@ virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DefaultFilePathPr
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DefaultTestProjectName.get -> string
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetAnalyzerOptions(Microsoft.CodeAnalysis.Project project) -> Microsoft.CodeAnalysis.Diagnostics.AnalyzerOptions
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetDefaultDiagnostic(Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer[] analyzers) -> Microsoft.CodeAnalysis.DiagnosticDescriptor
+virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.GetProjectCompilationAsync(Microsoft.CodeAnalysis.Project project, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<Microsoft.CodeAnalysis.Compilation>
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.IsCompilerDiagnosticIncluded(Microsoft.CodeAnalysis.Diagnostic diagnostic, Microsoft.CodeAnalysis.Testing.CompilerDiagnostics compilerDiagnostics) -> bool
 virtual Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.RunAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task
 virtual Microsoft.CodeAnalysis.Testing.CodeActionTest<TVerifier>.FilterCodeActions(System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction> actions) -> System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.CodeActions.CodeAction>

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CodeFix.Testing/CodeFixTest`1.cs

@@ -461,7 +461,7 @@ private async Task VerifyFixAsync(
             bool done;
             do
             {
-                var analyzerDiagnostics = await GetSortedDiagnosticsAsync(project.Solution, analyzers, additionalDiagnostics: ImmutableArray<Diagnostic>.Empty, CompilerDiagnostics, cancellationToken).ConfigureAwait(false);
+                var analyzerDiagnostics = await GetSortedDiagnosticsAsync(project.Solution, analyzers, additionalDiagnostics: ImmutableArray<Diagnostic>.Empty, CompilerDiagnostics, verifier, cancellationToken).ConfigureAwait(false);
                 if (analyzerDiagnostics.Length == 0)
                 {
                     break;
@@ -590,7 +590,7 @@ private async Task VerifyFixAsync(
             bool done;
             do
             {
-                var analyzerDiagnostics = await GetSortedDiagnosticsAsync(project.Solution, analyzers, additionalDiagnostics: ImmutableArray<Diagnostic>.Empty, CompilerDiagnostics, cancellationToken).ConfigureAwait(false);
+                var analyzerDiagnostics = await GetSortedDiagnosticsAsync(project.Solution, analyzers, additionalDiagnostics: ImmutableArray<Diagnostic>.Empty, CompilerDiagnostics, verifier, cancellationToken).ConfigureAwait(false);
                 if (analyzerDiagnostics.Length == 0)
                 {
                     break;

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/PublicAPI.Unshipped.txt

@@ -10,4 +10,5 @@ Microsoft.CodeAnalysis.Testing.SourceGeneratorVerifier<TSourceGenerator, TTest,
 abstract Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.CreateGeneratorDriver(Microsoft.CodeAnalysis.Project project, System.Collections.Immutable.ImmutableArray<Microsoft.CodeAnalysis.ISourceGenerator> sourceGenerators) -> Microsoft.CodeAnalysis.GeneratorDriver
 abstract Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.GetSourceGenerators() -> System.Collections.Generic.IEnumerable<Microsoft.CodeAnalysis.ISourceGenerator>
 override Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.GetDiagnosticAnalyzers() -> System.Collections.Generic.IEnumerable<Microsoft.CodeAnalysis.Diagnostics.DiagnosticAnalyzer>
+override Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.GetProjectCompilationAsync(Microsoft.CodeAnalysis.Project project, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<Microsoft.CodeAnalysis.Compilation>
 override Microsoft.CodeAnalysis.Testing.SourceGeneratorTest<TVerifier>.RunAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing/SourceGeneratorTest`1.cs

@@ -45,6 +45,12 @@ public override async Task RunAsync(CancellationToken cancellationToken = defaul
             await VerifyDiagnosticsAsync(new EvaluatedProjectState(testState, ReferenceAssemblies).WithAdditionalDiagnostics(diagnostics), testState.AdditionalProjects.Values.Select(additionalProject => new EvaluatedProjectState(additionalProject, ReferenceAssemblies)).ToImmutableArray(), testState.ExpectedDiagnostics.ToArray(), Verify.PushContext("Diagnostics of test state"), cancellationToken).ConfigureAwait(false);
         }
 
+        protected override async Task<Compilation> GetProjectCompilationAsync(Project project, IVerifier verifier, CancellationToken cancellationToken)
+        {
+            var (finalProject, diagnostics) = await ApplySourceGeneratorAsync(GetSourceGenerators().ToImmutableArray(), project, verifier, cancellationToken).ConfigureAwait(false);
+            return (await finalProject.GetCompilationAsync(cancellationToken).ConfigureAwait(false))!;
+        }
+
         /// <summary>
         /// Called to test a C# source generator when applied on the input source as a string.
         /// </summary>

tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/SourceGeneratorValidationTests.cs

@@ -2,13 +2,15 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Testing.TestGenerators;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.CodeAnalysis.VisualBasic;
 using Xunit;
 
 namespace Microsoft.CodeAnalysis.Testing
@@ -112,6 +114,74 @@ public async Task AddSimpleFileWithWrongExpectedEncoding()
             new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message);
         }
 
+        [Fact]
+        public async Task AddSimpleFileVerifiesCompilerDiagnostics_CSharp()
+        {
+            var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                await new CSharpSourceGeneratorTest<AddFileWithCompileError>
+                {
+                    TestState =
+                    {
+                        Sources =
+                        {
+                            @"class A {",
+                        },
+                        GeneratedSources =
+                        {
+                            (typeof(AddFileWithCompileError), "ErrorGeneratedFile.cs", @"class C {"),
+                        },
+                    },
+                }.RunAsync();
+            });
+
+            var expectedMessage = @"Context: Diagnostics of test state
+Mismatch between number of diagnostics returned, expected ""0"" actual ""2""
+
+Diagnostics:
+// /0/Test0.cs(1,10): error CS1513: } expected
+DiagnosticResult.CompilerError(""CS1513"").WithSpan(1, 10, 1, 10),
+// Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs(1,10): error CS1513: } expected
+DiagnosticResult.CompilerError(""CS1513"").WithSpan(""Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.cs"", 1, 10, 1, 10),
+
+";
+            new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message);
+        }
+
+        [Fact]
+        public async Task AddSimpleFileVerifiesCompilerDiagnostics_VisualBasic()
+        {
+            var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                await new VisualBasicSourceGeneratorTest<AddFileWithCompileError>
+                {
+                    TestState =
+                    {
+                        Sources =
+                        {
+                            "Class A",
+                        },
+                        GeneratedSources =
+                        {
+                            (typeof(AddFileWithCompileError), "ErrorGeneratedFile.vb", "Class C"),
+                        },
+                    },
+                }.RunAsync();
+            });
+
+            var expectedMessage = @"Context: Diagnostics of test state
+Mismatch between number of diagnostics returned, expected ""0"" actual ""2""
+
+Diagnostics:
+// /0/Test0.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'.
+DiagnosticResult.CompilerError(""BC30481"").WithSpan(1, 1, 1, 8),
+// Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb(1) : error BC30481: 'Class' statement must end with a matching 'End Class'.
+DiagnosticResult.CompilerError(""BC30481"").WithSpan(""Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests\Microsoft.CodeAnalysis.Testing.TestGenerators.AddFileWithCompileError\ErrorGeneratedFile.vb"", 1, 1, 1, 8),
+
+";
+            new DefaultVerifier().EqualOrDiff(expectedMessage, exception.Message);
+        }
+
         [Fact]
         public async Task AddSimpleFileWithDiagnostic()
         {
@@ -180,7 +250,39 @@ protected override CompilationOptions CreateCompilationOptions()
 
             protected override ParseOptions CreateParseOptions()
             {
-                return new CSharpParseOptions(LanguageVersion.Default, DocumentationMode.Diagnose);
+                return new CSharpParseOptions(CSharp.LanguageVersion.Default, DocumentationMode.Diagnose);
+            }
+
+            protected override IEnumerable<ISourceGenerator> GetSourceGenerators()
+            {
+                yield return new TSourceGenerator();
+            }
+        }
+
+        private class VisualBasicSourceGeneratorTest<TSourceGenerator> : SourceGeneratorTest<DefaultVerifier>
+            where TSourceGenerator : ISourceGenerator, new()
+        {
+            public override string Language => LanguageNames.VisualBasic;
+
+            protected override string DefaultFileExt => "vb";
+
+            protected override GeneratorDriver CreateGeneratorDriver(Project project, ImmutableArray<ISourceGenerator> sourceGenerators)
+            {
+                return VisualBasicGeneratorDriver.Create(
+                    sourceGenerators,
+                    project.AnalyzerOptions.AdditionalFiles,
+                    (VisualBasicParseOptions)project.ParseOptions!,
+                    project.AnalyzerOptions.AnalyzerConfigOptionsProvider);
+            }
+
+            protected override CompilationOptions CreateCompilationOptions()
+            {
+                return new VisualBasicCompilationOptions(OutputKind.DynamicallyLinkedLibrary);
+            }
+
+            protected override ParseOptions CreateParseOptions()
+            {
+                return new VisualBasicParseOptions(VisualBasic.LanguageVersion.Default, DocumentationMode.Diagnose);
             }
 
             protected override IEnumerable<ISourceGenerator> GetSourceGenerators()

tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.SourceGenerators.Testing.UnitTests/TestGenerators/AddFileWithCompileError.cs

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Testing.TestGenerators
+{
+    public class AddFileWithCompileError : ISourceGenerator
+    {
+        public void Initialize(GeneratorInitializationContext context)
+        {
+        }
+
+        public virtual void Execute(GeneratorExecutionContext context)
+        {
+            switch (context.Compilation.Language)
+            {
+                case LanguageNames.CSharp:
+                    context.AddSource("ErrorGeneratedFile", "class C {");
+                    break;
+
+                case LanguageNames.VisualBasic:
+                    context.AddSource("ErrorGeneratedFile", "Class C");
+                    break;
+
+                default:
+                    throw new NotSupportedException();
+            }
+        }
+    }
+}