Skip to content

Commit

Permalink
Merge pull request #15549 from CyrusNajmabadi/farCleanup
Browse files Browse the repository at this point in the history 
Cleanup in FindReferences engine.
  • Loading branch information
@CyrusNajmabadi
CyrusNajmabadi authored Nov 28, 2016
2 parents 8b62f18 + d9e9f88 commit 0bbd65a
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 183 deletions.
29 changes: 0 additions & 29 deletions ...es/Core/Portable/FindSymbols/FindReferences/FindReferencesSearchEngine.ProgressWrapper.cs
View file

This file was deleted.

71 changes: 31 additions & 40 deletions src/Workspaces/Core/Portable/FindSymbols/FindReferences/FindReferencesSearchEngine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

namespace Microsoft.CodeAnalysis.FindSymbols
{
using ProjectToDocumentMap = Dictionary<Project, MultiDictionary<Document, (SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>;

internal partial class FindReferencesSearchEngine
{
private readonly Solution _solution;
Expand Down Expand Up @@ -59,8 +61,10 @@ public async Task FindReferencesAsync(SymbolAndProjectId symbolAndProjectId)
var symbols = await DetermineAllSymbolsAsync(symbolAndProjectId).ConfigureAwait(false);

var projectMap = await CreateProjectMapAsync(symbols).ConfigureAwait(false);
var documentMap = await CreateDocumentMapAsync(projectMap).ConfigureAwait(false);
await ProcessAsync(documentMap).ConfigureAwait(false);
var projectToDocumentMap = await CreateProjectToDocumentMapAsync(projectMap).ConfigureAwait(false);
ValidateProjectToDocumentMap(projectToDocumentMap);

await ProcessAsync(projectToDocumentMap).ConfigureAwait(false);
}
finally
{
Expand All @@ -69,72 +73,59 @@ public async Task FindReferencesAsync(SymbolAndProjectId symbolAndProjectId)
}
}

private async Task ProcessAsync(
ConcurrentDictionary<Document, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> documentMap)
private async Task ProcessAsync(ProjectToDocumentMap projectToDocumentMap)
{
using (Logger.LogBlock(FunctionId.FindReference_ProcessAsync, _cancellationToken))
{
// quick exit
if (documentMap.Count == 0)
if (projectToDocumentMap.Count == 0)
{
return;
}

var wrapper = new ProgressWrapper(_progress, documentMap.Count);

// Get the connected components of the dependency graph and process each individually.
// That way once a component is done we can throw away all the memory associated with
// it.
// For each connected component, we'll process the individual projects from bottom to
// top. i.e. we'll first process the projects with no dependencies. Then the projects
// that depend on those projects, and so and. This way we always have creates the
// dependent compilations when they're needed by later projects. If we went the other
// way (i.e. processed the projects with lots of project dependencies first), then we'd
// have to create all their depedent compilations in order to get their compilation.
// This would be very expensive and would take a lot of time before we got our first
// result.
var connectedProjects = _dependencyGraph.GetDependencySets(_cancellationToken);
var projectMap = CreateProjectMap(documentMap);

await _progressTracker.AddItemsAsync(connectedProjects.Flatten().Count()).ConfigureAwait(false);
foreach (var projectSet in connectedProjects)
// Add a progress item for each (document, symbol, finder) set that we will execute.
// We'll mark the item as completed in "ProcessDocumentAsync".
var totalFindCount = projectToDocumentMap.Sum(
kvp1 => kvp1.Value.Sum(kvp2 => kvp2.Value.Count));
await _progressTracker.AddItemsAsync(totalFindCount).ConfigureAwait(false);

// Now, go through each connected project set and process it independently.
foreach (var connectedProjectSet in connectedProjects)
{
_cancellationToken.ThrowIfCancellationRequested();

await ProcessProjectsAsync(projectSet, projectMap, wrapper).ConfigureAwait(false);
await ProcessProjectsAsync(
connectedProjectSet, projectToDocumentMap).ConfigureAwait(false);
}
}
}

private static readonly Func<Project, Dictionary<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>> s_documentMapGetter =
_ => new Dictionary<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>();

private static readonly Func<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> s_queueGetter =
_ => new List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>();

private static Dictionary<Project, Dictionary<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>> CreateProjectMap(
ConcurrentDictionary<Document, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> map)
{
Contract.Requires(map.Count > 0);

var projectMap = new Dictionary<Project, Dictionary<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>>();
foreach (var kv in map)
{
var documentMap = projectMap.GetOrAdd(kv.Key.Project, s_documentMapGetter);
var queue = documentMap.GetOrAdd(kv.Key, s_queueGetter);

queue.AddRange(kv.Value);
}

ValidateProjectMap(projectMap);
return projectMap;
}

[Conditional("DEBUG")]
private static void ValidateProjectMap(
Dictionary<Project, Dictionary<Document, List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>> projectMap)
private static void ValidateProjectToDocumentMap(
ProjectToDocumentMap projectToDocumentMap)
{
var set = new HashSet<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>();

foreach (var map in projectMap.Values)
foreach (var documentMap in projectToDocumentMap.Values)
{
foreach (var finderList in map.Values)
foreach (var documentToFinderList in documentMap)
{
set.Clear();

foreach (var finder in finderList)
foreach (var finder in documentToFinderList.Value)
{
Contract.Requires(set.Add(finder));
}
Expand Down
28 changes: 7 additions & 21 deletions ...Core/Portable/FindSymbols/FindReferences/FindReferencesSearchEngine_DocumentProcessing.cs
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.FindSymbols.Finders;
using Microsoft.CodeAnalysis.Internal.Log;
using Roslyn.Utilities;

namespace Microsoft.CodeAnalysis.FindSymbols
{
using DocumentMap = MultiDictionary<Document, (SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>;

internal partial class FindReferencesSearchEngine
{
private async Task ProcessDocumentQueueAsync(
Document document,
List<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)> documentQueue,
ProgressWrapper wrapper)
DocumentMap.ValueSet documentQueue)
{
await _progress.OnFindInDocumentStartedAsync(document).ConfigureAwait(false);

Expand All @@ -26,26 +26,13 @@ private async Task ProcessDocumentQueueAsync(
// start cache for this semantic model
FindReferenceCache.Start(model);

#if PARALLEL
Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () =>
{
documentQueue.AsParallel().WithCancellation(cancellationToken).ForAll(symbolAndFinder =>
{
var symbol = symbolAndFinder.Item1;
var finder = symbolAndFinder.Item2;

ProcessDocument(document, symbol, finder, wrapper);
});
});
#else
foreach (var symbolAndFinder in documentQueue)
{
var symbol = symbolAndFinder.symbolAndProjectId;
var finder = symbolAndFinder.finder;

await ProcessDocumentAsync(document, symbol, finder, wrapper).ConfigureAwait(false);
await ProcessDocumentAsync(document, symbol, finder).ConfigureAwait(false);
}
#endif
}
finally
{
Expand All @@ -63,8 +50,7 @@ private async Task ProcessDocumentQueueAsync(
private async Task ProcessDocumentAsync(
Document document,
SymbolAndProjectId symbolAndProjectId,
IReferenceFinder finder,
ProgressWrapper wrapper)
IReferenceFinder finder)
{
using (Logger.LogBlock(FunctionId.FindReference_ProcessDocumentAsync, s_logDocument, document, symbolAndProjectId.Symbol, _cancellationToken))
{
Expand All @@ -78,9 +64,9 @@ private async Task ProcessDocumentAsync(
}
finally
{
await wrapper.IncrementAsync().ConfigureAwait(false);
await _progressTracker.ItemCompletedAsync().ConfigureAwait(false);
}
}
}
}
}
}
84 changes: 21 additions & 63 deletions ...spaces/Core/Portable/FindSymbols/FindReferences/FindReferencesSearchEngine_MapCreation.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
Expand All @@ -13,48 +11,25 @@

namespace Microsoft.CodeAnalysis.FindSymbols
{
using DocumentMap = MultiDictionary<Document, (SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>;
using ProjectMap = MultiDictionary<Project, (SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>;
using ProjectToDocumentMap = Dictionary<Project, MultiDictionary<Document, (SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>;

internal partial class FindReferencesSearchEngine
{
private async Task<ConcurrentDictionary<Document, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>> CreateDocumentMapAsync(
ConcurrentDictionary<Project, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> projectMap)
private async Task<ProjectToDocumentMap> CreateProjectToDocumentMapAsync(ProjectMap projectMap)
{
using (Logger.LogBlock(FunctionId.FindReference_CreateDocumentMapAsync, _cancellationToken))
{
Func<Document, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> createQueue =
d => new ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>();

var documentMap = new ConcurrentDictionary<Document, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>();

#if PARALLEL
Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () =>
{
projectMap.AsParallel().WithCancellation(cancellationToken).ForAll(kvp =>
{
var project = kvp.Key;
var projectQueue = kvp.Value;
var finalMap = new ProjectToDocumentMap();

projectQueue.AsParallel().WithCancellation(cancellationToken).ForAll(symbolAndFinder =>
{
var symbol = symbolAndFinder.Item1;
var finder = symbolAndFinder.Item2;

var documents = finder.DetermineDocumentsToSearch(symbol, project, cancellationToken) ?? SpecializedCollections.EmptyEnumerable<Document>();
foreach (var document in documents.Distinct().WhereNotNull())
{
if (includeDocument(document))
{
documentMap.GetOrAdd(document, createQueue).Enqueue(symbolAndFinder);
}
}
});
});
});
#else
foreach (var kvp in projectMap)
{
var project = kvp.Key;
var projectQueue = kvp.Value;

var documentMap = new DocumentMap();

foreach (var symbolAndFinder in projectQueue)
{
_cancellationToken.ThrowIfCancellationRequested();
Expand All @@ -68,46 +43,30 @@ internal partial class FindReferencesSearchEngine
{
if (_documents == null || _documents.Contains(document))
{
documentMap.GetOrAdd(document, createQueue).Enqueue(symbolAndFinder);
documentMap.Add(document, symbolAndFinder);
}
}
}

Contract.ThrowIfTrue(documentMap.Any(kvp1 => kvp1.Value.Count != kvp1.Value.ToSet().Count));

if (documentMap.Count > 0)
{
finalMap.Add(project, documentMap);
}
}
#endif

Contract.ThrowIfTrue(documentMap.Any(kvp => kvp.Value.Count != kvp.Value.ToSet().Count));
return documentMap;
return finalMap;
}
}

private async Task<ConcurrentDictionary<Project, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>> CreateProjectMapAsync(
ConcurrentSet<SymbolAndProjectId> symbols)
private async Task<ProjectMap> CreateProjectMapAsync(ConcurrentSet<SymbolAndProjectId> symbols)
{
using (Logger.LogBlock(FunctionId.FindReference_CreateProjectMapAsync, _cancellationToken))
{
Func<Project, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>> createQueue =
p => new ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>();

var projectMap = new ConcurrentDictionary<Project, ConcurrentQueue<(SymbolAndProjectId symbolAndProjectId, IReferenceFinder finder)>>();

#if PARALLEL
Roslyn.Utilities.TaskExtensions.RethrowIncorrectAggregateExceptions(cancellationToken, () =>
{
symbols.AsParallel().WithCancellation(cancellationToken).ForAll(s =>
{
finders.AsParallel().WithCancellation(cancellationToken).ForAll(f =>
{
var projects = f.DetermineProjectsToSearch(s, solution, cancellationToken) ?? SpecializedCollections.EmptyEnumerable<Project>();
foreach (var project in projects.Distinct())
{
projectMap.GetOrAdd(project, createQueue).Enqueue(ValueTuple.Create(s, f));
}
});
});
});
#else
var projectMap = new ProjectMap();

var scope = _documents != null ? _documents.Select(d => d.Project).ToImmutableHashSet() : null;
var scope = _documents?.Select(d => d.Project).ToImmutableHashSet();
foreach (var symbolAndProjectId in symbols)
{
foreach (var finder in _finders)
Expand All @@ -119,12 +78,11 @@ internal partial class FindReferencesSearchEngine
{
if (scope == null || scope.Contains(project))
{
projectMap.GetOrAdd(project, createQueue).Enqueue((symbolAndProjectId, finder));
projectMap.Add(project, (symbolAndProjectId, finder));
}
}
}
}
#endif

Contract.ThrowIfTrue(projectMap.Any(kvp => kvp.Value.Count != kvp.Value.ToSet().Count));
return projectMap;
Expand Down
Loading

0 comments on commit 0bbd65a

Please sign in to comment.