Skip to content

Commit

Permalink
Merge pull request #458 from sharwell/content-first
Browse files Browse the repository at this point in the history 
Verify content before iteration counts
  • Loading branch information
@sharwell
sharwell authored Feb 17, 2020
2 parents f1f8ca6 + 3fa7ff4 commit 358cca4
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 40 deletions.
73 changes: 53 additions & 20 deletions src/Microsoft.CodeAnalysis.Testing/Microsoft.CodeAnalysis.CodeFix.Testing/CodeFixTest`1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -342,14 +343,15 @@ private async Task VerifyFixAsync(
SolutionState oldState,
SolutionState newState,
int numberOfIterations,
Func<ImmutableArray<DiagnosticAnalyzer>, ImmutableArray<CodeFixProvider>, int?, string?, Project, int, IVerifier, CancellationToken, Task<Project>> getFixedProject,
Func<ImmutableArray<DiagnosticAnalyzer>, ImmutableArray<CodeFixProvider>, int?, string?, Project, int, IVerifier, CancellationToken, Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)>> getFixedProject,
IVerifier verifier,
CancellationToken cancellationToken)
{
var project = await CreateProjectAsync(oldState.Sources.ToArray(), oldState.AdditionalFiles.ToArray(), oldState.AdditionalProjects.ToArray(), oldState.AdditionalReferences.ToArray(), language, cancellationToken);
var compilerDiagnostics = await GetCompilerDiagnosticsAsync(project, cancellationToken).ConfigureAwait(false);

project = await getFixedProject(analyzers, codeFixProviders, CodeFixIndex, CodeFixEquivalenceKey, project, numberOfIterations, verifier, cancellationToken).ConfigureAwait(false);
ExceptionDispatchInfo? iterationCountFailure;
(project, iterationCountFailure) = await getFixedProject(analyzers, codeFixProviders, CodeFixIndex, CodeFixEquivalenceKey, project, numberOfIterations, verifier, cancellationToken).ConfigureAwait(false);

// After applying all of the code fixes, compare the resulting string to the inputted one
var updatedDocuments = project.Documents.ToArray();
Expand Down Expand Up @@ -377,6 +379,9 @@ private async Task VerifyFixAsync(
verifier.Equal(newState.AdditionalFiles[i].content.ChecksumAlgorithm, actual.ChecksumAlgorithm, $"checksum algorithm of '{newState.AdditionalFiles[i].filename}' was expected to be '{newState.AdditionalFiles[i].content.ChecksumAlgorithm}' but was '{actual.ChecksumAlgorithm}'");
verifier.Equal(newState.AdditionalFiles[i].filename, updatedAdditionalDocuments[i].Name, $"file name was expected to be '{newState.AdditionalFiles[i].filename}' but was '{updatedAdditionalDocuments[i].Name}'");
}

// Validate the iteration counts after validating the content
iterationCountFailure?.Throw();
}

private static bool HasAnyChange(SolutionState oldState, SolutionState newState)
Expand All @@ -385,7 +390,7 @@ private static bool HasAnyChange(SolutionState oldState, SolutionState newState)
|| !oldState.AdditionalFiles.SequenceEqual(newState.AdditionalFiles, SourceFileEqualityComparer.Instance);
}

private async Task<Project> FixEachAnalyzerDiagnosticAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
private async Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)> FixEachAnalyzerDiagnosticAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
{
var expectedNumberOfIterations = numberOfIterations;
if (numberOfIterations < 0)
Expand All @@ -409,7 +414,14 @@ private async Task<Project> FixEachAnalyzerDiagnosticAsync(ImmutableArray<Diagno
break;
}

verifier.True(--numberOfIterations >= -1, "The upper limit for the number of code fix iterations was exceeded");
try
{
verifier.True(--numberOfIterations >= -1, "The upper limit for the number of code fix iterations was exceeded");
}
catch (Exception ex)
{
return (project, ExceptionDispatchInfo.Capture(ex));
}

previousDiagnostics = analyzerDiagnostics;

Expand Down Expand Up @@ -457,16 +469,23 @@ private async Task<Project> FixEachAnalyzerDiagnosticAsync(ImmutableArray<Diagno
}
while (!done);

if (expectedNumberOfIterations >= 0)
try
{
verifier.Equal(expectedNumberOfIterations, expectedNumberOfIterations - numberOfIterations, $"Expected '{expectedNumberOfIterations}' iterations but found '{expectedNumberOfIterations - numberOfIterations}' iterations.");
if (expectedNumberOfIterations >= 0)
{
verifier.Equal(expectedNumberOfIterations, expectedNumberOfIterations - numberOfIterations, $"Expected '{expectedNumberOfIterations}' iterations but found '{expectedNumberOfIterations - numberOfIterations}' iterations.");
}
else
{
verifier.True(numberOfIterations >= 0, "The upper limit for the number of code fix iterations was exceeded");
}
}
else
catch (Exception ex)
{
verifier.True(numberOfIterations >= 0, "The upper limit for the number of code fix iterations was exceeded");
return (project, ExceptionDispatchInfo.Capture(ex));
}

return project;
return (project, null);
}

private static CodeAction? TryGetCodeActionToApply(List<CodeAction> actions, int? codeFixIndex, string? codeFixEquivalenceKey, IVerifier verifier)
Expand Down Expand Up @@ -568,22 +587,22 @@ private static async Task<Project> ApplyFixAsync(Project project, CodeAction cod
return solution.GetProject(project.Id);
}

private Task<Project> FixAllAnalyzerDiagnosticsInDocumentAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
private Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)> FixAllAnalyzerDiagnosticsInDocumentAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
{
return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Document, analyzers, codeFixProviders, codeFixIndex, codeFixEquivalenceKey, project, numberOfIterations, verifier, cancellationToken);
}

private Task<Project> FixAllAnalyzerDiagnosticsInProjectAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
private Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)> FixAllAnalyzerDiagnosticsInProjectAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
{
return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Project, analyzers, codeFixProviders, codeFixIndex, codeFixEquivalenceKey, project, numberOfIterations, verifier, cancellationToken);
}

private Task<Project> FixAllAnalyzerDiagnosticsInSolutionAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
private Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)> FixAllAnalyzerDiagnosticsInSolutionAsync(ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
{
return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Solution, analyzers, codeFixProviders, codeFixIndex, codeFixEquivalenceKey, project, numberOfIterations, verifier, cancellationToken);
}

private async Task<Project> FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope scope, ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
private async Task<(Project project, ExceptionDispatchInfo? iterationCountFailure)> FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope scope, ImmutableArray<DiagnosticAnalyzer> analyzers, ImmutableArray<CodeFixProvider> codeFixProviders, int? codeFixIndex, string? codeFixEquivalenceKey, Project project, int numberOfIterations, IVerifier verifier, CancellationToken cancellationToken)
{
var expectedNumberOfIterations = numberOfIterations;
if (numberOfIterations < 0)
Expand All @@ -607,7 +626,14 @@ private async Task<Project> FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope sco
break;
}

verifier.False(--numberOfIterations < -1, "The upper limit for the number of fix all iterations was exceeded");
try
{
verifier.False(--numberOfIterations < -1, "The upper limit for the number of fix all iterations was exceeded");
}
catch (Exception ex)
{
return (project, ExceptionDispatchInfo.Capture(ex));
}

Diagnostic? firstDiagnostic = null;
CodeFixProvider? effectiveCodeFixProvider = null;
Expand Down Expand Up @@ -660,7 +686,7 @@ private async Task<Project> FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope sco
var action = await fixAllProvider.GetFixAsync(fixAllContext).ConfigureAwait(false);
if (action == null)
{
return project;
return (project, null);
}

var fixedProject = await ApplyFixAsync(project, action, cancellationToken).ConfigureAwait(false);
Expand All @@ -673,16 +699,23 @@ private async Task<Project> FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope sco
}
while (!done);

if (expectedNumberOfIterations >= 0)
try
{
verifier.Equal(expectedNumberOfIterations, expectedNumberOfIterations - numberOfIterations, $"Expected '{expectedNumberOfIterations}' iterations but found '{expectedNumberOfIterations - numberOfIterations}' iterations.");
if (expectedNumberOfIterations >= 0)
{
verifier.Equal(expectedNumberOfIterations, expectedNumberOfIterations - numberOfIterations, $"Expected '{expectedNumberOfIterations}' iterations but found '{expectedNumberOfIterations - numberOfIterations}' iterations.");
}
else
{
verifier.True(numberOfIterations >= 0, "The upper limit for the number of code fix iterations was exceeded");
}
}
else
catch (Exception ex)
{
verifier.True(numberOfIterations >= 0, "The upper limit for the number of code fix iterations was exceeded");
return (project, ExceptionDispatchInfo.Capture(ex));
}

return project;
return (project, null);
}

/// <summary>
Expand Down
46 changes: 26 additions & 20 deletions ...nalysis.Testing/Microsoft.CodeAnalysis.CodeFix.Testing.UnitTests/CodeFixIterationTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ class TestClass {
";
var fixedCode = @"
class TestClass {
int field = 5;
int field = 5;
}
";
var batchFixedCode = @"
Expand All @@ -145,24 +145,24 @@ class TestClass {
}.RunAsync();
});

Assert.Equal($"Context: Iterative code fix application{Environment.NewLine}The upper limit for the number of code fix iterations was exceeded", exception.Message);
new DefaultVerifier().EqualOrDiff($"Context: Iterative code fix application{Environment.NewLine}The upper limit for the number of code fix iterations was exceeded", exception.Message);
}

[Theory]
[InlineData(-1, "The upper limit for the number of code fix iterations was exceeded")]
[InlineData(0, "The upper limit for the number of code fix iterations was exceeded")]
[InlineData(1, "Expected '1' iterations but found '2' iterations.")]
public async Task TestTwoIterationsRequiredButIncrementalDeclaredIncorrectly(int declaredIncrementalIterations, string message)
[InlineData(-1, "The upper limit for the number of code fix iterations was exceeded", " 5")]
[InlineData(0, "The upper limit for the number of code fix iterations was exceeded", " [|4|]")]
[InlineData(1, "Expected '1' iterations but found '2' iterations.", " 5")]
public async Task TestTwoIterationsRequiredButIncrementalDeclaredIncorrectly(int declaredIncrementalIterations, string message, string replacement)
{
var testCode = @"
class TestClass {
int field = [|3|];
}
";
var fixedCode = @"
class TestClass {
int field = 5;
}
var fixedCode = $@"
class TestClass {{
int field = {replacement};
}}
";
var batchFixedCode = @"
class TestClass {
Expand All @@ -175,14 +175,14 @@ class TestClass {
await new CSharpTest
{
TestCode = testCode,
FixedCode = fixedCode,
FixedState = { Sources = { fixedCode }, MarkupHandling = MarkupMode.Allow },
BatchFixedCode = batchFixedCode,
NumberOfIncrementalIterations = declaredIncrementalIterations,
NumberOfFixAllIterations = 2,
}.RunAsync();
});

Assert.Equal($"Context: Iterative code fix application{Environment.NewLine}{message}", exception.Message);
new DefaultVerifier().EqualOrDiff($"Context: Iterative code fix application{Environment.NewLine}{message}", exception.Message);
}

[Fact]
Expand Down Expand Up @@ -213,20 +213,25 @@ class TestClass {
}

[Theory]
[InlineData(-1, "The upper limit for the number of code fix iterations was exceeded")]
[InlineData(0, "The upper limit for the number of fix all iterations was exceeded")]
[InlineData(1, "Expected '1' iterations but found '2' iterations.")]
public async Task TestTwoIterationsRequiredButFixAllDeclaredIncorrectly(int declaredFixAllIterations, string message)
[InlineData(-1, "The upper limit for the number of code fix iterations was exceeded", " 5")]
[InlineData(0, "The upper limit for the number of fix all iterations was exceeded", " [|4|]")]
[InlineData(1, "Expected '1' iterations but found '2' iterations.", " 5")]
public async Task TestTwoIterationsRequiredButFixAllDeclaredIncorrectly(int declaredFixAllIterations, string message, string replacement)
{
var testCode = @"
class TestClass {
int field = [|3|];
}
";
var fixedCode = @"
class TestClass {
var fixedCode = $@"
class TestClass {{
int field = 5;
}
}}
";
var fixAllCode = $@"
class TestClass {{
int field = {replacement};
}}
";

var exception = await Assert.ThrowsAsync<InvalidOperationException>(async () =>
Expand All @@ -235,12 +240,13 @@ class TestClass {
{
TestCode = testCode,
FixedCode = fixedCode,
BatchFixedState = { Sources = { fixAllCode }, MarkupHandling = MarkupMode.Allow },
NumberOfIncrementalIterations = 2,
NumberOfFixAllIterations = declaredFixAllIterations,
}.RunAsync();
});

Assert.Equal($"Context: Fix all in document{Environment.NewLine}{message}", exception.Message);
new DefaultVerifier().EqualOrDiff($"Context: Fix all in document{Environment.NewLine}{message}", exception.Message);
}

[Fact]
Expand Down

0 comments on commit 358cca4

Please sign in to comment.