Merge pull request #298 from sharwell/quirks

Rename AnalysisExclusions to TestBehaviors

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/AnalyzerTest`1.cs

@@ -98,10 +98,10 @@ public string TestCode
         public Dictionary<string, string> XmlReferences { get; } = new Dictionary<string, string>();
 
         /// <summary>
-        /// Gets or sets a value the analysis exclusions to verify. The default value is a combination of
-        /// <see cref="AnalysisExclusions.GeneratedCode"/> and <see cref="AnalysisExclusions.Suppression"/>.
+        /// Gets or sets the test behaviors applying to this analyzer. The default value is
+        /// <see cref="TestBehaviors.None"/>.
         /// </summary>
-        public AnalysisExclusions Exclusions { get; set; } = AnalysisExclusions.GeneratedCode | AnalysisExclusions.Suppression;
+        public TestBehaviors TestBehaviors { get; set; }
 
         /// <summary>
         /// Gets a collection of diagnostics to explicitly disable in the <see cref="CompilationOptions"/> for projects.
@@ -155,7 +155,7 @@ protected async Task VerifyDiagnosticsAsync((string filename, SourceText content
 
         private async Task VerifyGeneratedCodeDiagnosticsAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, (string filename, SourceText content)[] sources, (string filename, SourceText content)[] additionalFiles, MetadataReference[] additionalMetadataReferences, DiagnosticResult[] expected, IVerifier verifier, CancellationToken cancellationToken)
         {
-            if (!Exclusions.HasFlag(AnalysisExclusions.GeneratedCode))
+            if (TestBehaviors.HasFlag(TestBehaviors.SkipGeneratedCodeCheck))
             {
                 return;
             }
@@ -180,7 +180,7 @@ private async Task VerifyGeneratedCodeDiagnosticsAsync(ImmutableArray<Diagnostic
 
         private async Task VerifySuppressionDiagnosticsAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, (string filename, SourceText content)[] sources, (string filename, SourceText content)[] additionalFiles, MetadataReference[] additionalMetadataReferences, DiagnosticResult[] expected, IVerifier verifier, CancellationToken cancellationToken)
         {
-            if (!Exclusions.HasFlag(AnalysisExclusions.Suppression))
+            if (TestBehaviors.HasFlag(TestBehaviors.SkipSuppressionCheck))
             {
                 return;
             }

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/PublicAPI.Unshipped.txt

@@ -1,18 +1,14 @@
-Microsoft.CodeAnalysis.Testing.AnalysisExclusions
-Microsoft.CodeAnalysis.Testing.AnalysisExclusions.GeneratedCode = 1 -> Microsoft.CodeAnalysis.Testing.AnalysisExclusions
-Microsoft.CodeAnalysis.Testing.AnalysisExclusions.None = 0 -> Microsoft.CodeAnalysis.Testing.AnalysisExclusions
-Microsoft.CodeAnalysis.Testing.AnalysisExclusions.Suppression = 2 -> Microsoft.CodeAnalysis.Testing.AnalysisExclusions
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.AnalyzerTest() -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.get -> Microsoft.CodeAnalysis.Testing.CompilerDiagnostics
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CompilerDiagnostics.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CreateProject((string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] sources, (string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] additionalFiles, Microsoft.CodeAnalysis.MetadataReference[] additionalMetadataReferences, string language) -> Microsoft.CodeAnalysis.Project
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DisabledDiagnostics.get -> System.Collections.Generic.List<string>
-Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.Exclusions.get -> Microsoft.CodeAnalysis.Testing.AnalysisExclusions
-Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.Exclusions.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.ExpectedDiagnostics.get -> System.Collections.Generic.List<Microsoft.CodeAnalysis.Testing.DiagnosticResult>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.OptionsTransforms.get -> System.Collections.Generic.List<System.Func<Microsoft.CodeAnalysis.Options.OptionSet, Microsoft.CodeAnalysis.Options.OptionSet>>
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.SolutionTransforms.get -> System.Collections.Generic.List<System.Func<Microsoft.CodeAnalysis.Solution, Microsoft.CodeAnalysis.ProjectId, Microsoft.CodeAnalysis.Solution>>
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.TestBehaviors.get -> Microsoft.CodeAnalysis.Testing.TestBehaviors
+Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.TestBehaviors.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.TestCode.set -> void
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.TestState.get -> Microsoft.CodeAnalysis.Testing.SolutionState
 Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.VerifyDiagnosticsAsync((string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] sources, (string filename, Microsoft.CodeAnalysis.Text.SourceText content)[] additionalFiles, Microsoft.CodeAnalysis.MetadataReference[] additionalMetadataReferences, Microsoft.CodeAnalysis.Testing.DiagnosticResult[] expected, Microsoft.CodeAnalysis.Testing.IVerifier verifier, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task
@@ -105,6 +101,10 @@ Microsoft.CodeAnalysis.Testing.StateInheritanceMode
 Microsoft.CodeAnalysis.Testing.StateInheritanceMode.AutoInherit = 0 -> Microsoft.CodeAnalysis.Testing.StateInheritanceMode
 Microsoft.CodeAnalysis.Testing.StateInheritanceMode.AutoInheritAll = 2 -> Microsoft.CodeAnalysis.Testing.StateInheritanceMode
 Microsoft.CodeAnalysis.Testing.StateInheritanceMode.Explicit = 1 -> Microsoft.CodeAnalysis.Testing.StateInheritanceMode
+Microsoft.CodeAnalysis.Testing.TestBehaviors
+Microsoft.CodeAnalysis.Testing.TestBehaviors.None = 0 -> Microsoft.CodeAnalysis.Testing.TestBehaviors
+Microsoft.CodeAnalysis.Testing.TestBehaviors.SkipGeneratedCodeCheck = 1 -> Microsoft.CodeAnalysis.Testing.TestBehaviors
+Microsoft.CodeAnalysis.Testing.TestBehaviors.SkipSuppressionCheck = 2 -> Microsoft.CodeAnalysis.Testing.TestBehaviors
 Microsoft.CodeAnalysis.Testing.TestFileMarkupParser
 abstract Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.CreateCompilationOptions() -> Microsoft.CodeAnalysis.CompilationOptions
 abstract Microsoft.CodeAnalysis.Testing.AnalyzerTest<TVerifier>.DefaultFileExt.get -> string

src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing/TestBehaviors.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Testing
+{
+    /// <summary>
+    /// Specifies non-standard analyzer behaviors which impact testing.
+    /// </summary>
+    [Flags]
+    public enum TestBehaviors
+    {
+        /// <summary>
+        /// No special behaviors apply.
+        /// </summary>
+        None = 0,
+
+        /// <summary>
+        /// Skip the generated code exclusion check.
+        /// </summary>
+        /// <remarks>
+        /// <para>By default, the analyzer test framework verifies that analyzer which report diagnostics do not report
+        /// diagnostics in generated code. While some analyzers, e.g. security analyzers, are expected to report
+        /// diagnostics in all code, most analyzers are expected to only report diagnostics in user-created code.</para>
+        /// </remarks>
+        SkipGeneratedCodeCheck = 0x01,
+
+        /// <summary>
+        /// Skip a verification check that diagnostics will not be reported if a <c>#pragma warning disable</c> appears
+        /// at the beginning of the file.
+        /// </summary>
+        SkipSuppressionCheck = 0x02,
+    }
+}

tests/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.Analyzer.Testing.UnitTests/AutoExclusionTests.cs

@@ -106,7 +106,7 @@ public async Task TestCSharpAnalyzerWithoutGeneratedCodeExclusionButAllowedPasse
             await new CSharpReplaceThisWithBaseTest(GeneratedCodeAnalysisFlags.Analyze | GeneratedCodeAnalysisFlags.ReportDiagnostics)
             {
                 TestCode = ReplaceThisWithBaseTestCode,
-                Exclusions = AnalysisExclusions.Suppression,
+                TestBehaviors = TestBehaviors.SkipGeneratedCodeCheck,
             }.RunAsync();
         }
 
@@ -116,7 +116,7 @@ public async Task TestCSharpAnalyzerWithoutSuppressionExclusionButAllowedPasses(
             await new CSharpAnalyzerTest<FirstLineDiagnosticAnalyzer>
             {
                 TestCode = CSharpFirstLineDiagnosticTestCode,
-                Exclusions = AnalysisExclusions.GeneratedCode,
+                TestBehaviors = TestBehaviors.SkipSuppressionCheck,
             }.RunAsync();
         }
 
@@ -184,7 +184,7 @@ public async Task TestVisualBasicAnalyzerWithoutGeneratedCodeExclusionButAllowed
             await new VisualBasicReplaceThisWithBaseTest(GeneratedCodeAnalysisFlags.Analyze | GeneratedCodeAnalysisFlags.ReportDiagnostics)
             {
                 TestCode = ReplaceMyClassWithMyBaseTestCode,
-                Exclusions = AnalysisExclusions.Suppression,
+                TestBehaviors = TestBehaviors.SkipGeneratedCodeCheck,
             }.RunAsync();
         }
 
@@ -194,7 +194,7 @@ public async Task TestVisualBasicAnalyzerWithoutSuppressionExclusionButAllowedPa
             await new VisualBasicAnalyzerTest<FirstLineDiagnosticAnalyzer>
             {
                 TestCode = VisualBasicFirstLineDiagnosticTestCode,
-                Exclusions = AnalysisExclusions.GeneratedCode,
+                TestBehaviors = TestBehaviors.SkipSuppressionCheck,
             }.RunAsync();
         }