Skip to content

Commit

Permalink
Cache the asset graph after each build, and read it in at startup
Browse files Browse the repository at this point in the history
  • Loading branch information
@jakemac53
jakemac53 committed Feb 17, 2016
1 parent 7f0ce90 commit a3df592
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 99 deletions.
10 changes: 4 additions & 6 deletions lib/src/generate/build.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import '../asset/cache.dart';
import '../asset/file_based.dart';
import '../asset/reader.dart';
import '../asset/writer.dart';
import '../asset_graph/graph.dart';
import '../package_graph/package_graph.dart';
import 'build_impl.dart';
import 'build_result.dart';
Expand Down Expand Up @@ -52,8 +51,7 @@ Future<BuildResult> build(List<List<Phase>> phaseGroups,
writer ??=
new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));

var buildImpl = new BuildImpl(
new AssetGraph(), reader, writer, packageGraph, phaseGroups);
var buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups);

/// Run the build!
var futureResult = buildImpl.runBuild();
Expand Down Expand Up @@ -105,7 +103,7 @@ Stream<BuildResult> watch(List<List<Phase>> phaseGroups,
new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));
directoryWatcherFactory ??= defaultDirectoryWatcherFactory;
var watchImpl = new WatchImpl(directoryWatcherFactory, debounceDelay, cache,
new AssetGraph(), reader, writer, packageGraph, phaseGroups);
reader, writer, packageGraph, phaseGroups);

var resultStream = watchImpl.runWatch();

Expand All @@ -120,8 +118,8 @@ Stream<BuildResult> watch(List<List<Phase>> phaseGroups,

/// Given [terminateEventStream], call [onTerminate] the first time an event is
/// seen. If a second event is recieved, simply exit.
StreamSubscription _setupTerminateLogic(Stream terminateEventStream,
Future onTerminate()) {
StreamSubscription _setupTerminateLogic(
Stream terminateEventStream, Future onTerminate()) {
terminateEventStream ??= ProcessSignal.SIGINT.watch();
int numEventsSeen = 0;
var terminateListener;
Expand Down
83 changes: 52 additions & 31 deletions lib/src/generate/build_impl.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import '../asset/exceptions.dart';
import '../asset/id.dart';
import '../asset/reader.dart';
import '../asset/writer.dart';
import '../asset_graph/exceptions.dart';
import '../asset_graph/graph.dart';
import '../asset_graph/node.dart';
import '../builder/builder.dart';
Expand All @@ -25,7 +26,9 @@ import 'phase.dart';

/// Class which manages running builds.
class BuildImpl {
final AssetGraph _assetGraph;
AssetGraph _assetGraph;
AssetGraph get assetGraph => _assetGraph;

final AssetReader _reader;
final AssetWriter _writer;
final PackageGraph _packageGraph;
Expand All @@ -34,8 +37,7 @@ class BuildImpl {
bool _buildRunning = false;
final _logger = new Logger('Build');

BuildImpl(this._assetGraph, this._reader, this._writer, this._packageGraph,
this._phaseGroups);
BuildImpl(this._reader, this._writer, this._packageGraph, this._phaseGroups);

/// Runs a build
///
Expand All @@ -49,6 +51,12 @@ class BuildImpl {
if (_buildRunning) throw const ConcurrentBuildException();
_buildRunning = true;

/// Initialize the [assetGraph] if its not yet set up.
if (_assetGraph == null) {
_logger.info('Reading cached dependency graph');
_assetGraph = await _readAssetGraph();
}

/// Wait while all inputs are collected.
_logger.info('Initializing inputs');
await _initializeInputsByPackage();
Expand All @@ -61,14 +69,10 @@ class BuildImpl {
_logger.info('Running build phases');
var result = await _runPhases();

/// Write out the new build_outputs file.
var allOuputs = _assetGraph.allNodes
.where((node) => node is GeneratedAssetNode && node.wasOutput);
var buildOutputsAsset = new Asset(
_buildOutputsId,
JSON.encode(
allOuputs.map((output) => output.id.serialize()).toList()));
await _writer.writeAsString(buildOutputsAsset);
/// Write out the dependency graph file.
var assetGraphAsset =
new Asset(_assetGraphId, JSON.encode(_assetGraph.serialize()));
await _writer.writeAsString(assetGraphAsset);

return result;
} catch (e, s) {
Expand All @@ -79,27 +83,45 @@ class BuildImpl {
}
}

/// Asset containing previous build outputs.
AssetId get _buildOutputsId =>
new AssetId(_packageGraph.root.name, '.build/build_outputs.json');
/// Asset containing previous asset dependency graph.
AssetId get _assetGraphId =>
new AssetId(_packageGraph.root.name, '.build/asset_graph.json');

/// Reads in the [assetGraph] from disk.
Future<AssetGraph> _readAssetGraph() async {
if (!await _reader.hasInput(_assetGraphId)) return new AssetGraph();
try {
_logger.info('Reading cached asset graph.');
var graph = new AssetGraph.deserialize(
JSON.decode(await _reader.readAsString(_assetGraphId)));
/// TODO(jakemac): Only invalidate nodes which need invalidating, which
/// will give us incremental builds on startup.
graph.allNodes.where((node) => node is GeneratedAssetNode)
.forEach((node) => node.needsUpdate = true);
return graph;
} on AssetGraphVersionException catch (_) {
/// Start fresh if the cached asset_graph version doesn't match up with
/// the current version. We don't currently support old graph versions.
_logger.info('Throwing away cached asset graph due to version mismatch.');
return new AssetGraph();
}
}

/// Deletes all previous output files.
Future _deletePreviousOutputs() async {
if (await _reader.hasInput(_buildOutputsId)) {
/// Cache file exists, delete all outputs which don't appear in the
/// [_assetGraph], or are marked as needing an update.
///
/// Removes all files from [_inputsByPackage] regardless of state.
var previousOutputs =
JSON.decode(await _reader.readAsString(_buildOutputsId));
await _writer.delete(_buildOutputsId);
_inputsByPackage[_buildOutputsId.package]?.remove(_buildOutputsId);
await Future.wait(previousOutputs.map((output) async {
var outputId = new AssetId.deserialize(output);
_inputsByPackage[outputId.package]?.remove(outputId);
var node = _assetGraph.get(outputId);
if (node == null || (node as GeneratedAssetNode).needsUpdate) {
await _writer.delete(outputId);
/// TODO(jakemac): need a cleaner way of telling if the current graph was
/// generated from cache or if its just a brand new graph.
if (await _reader.hasInput(_assetGraphId)) {
await _writer.delete(_assetGraphId);
_inputsByPackage[_assetGraphId.package]?.remove(_assetGraphId);
/// Remove all output nodes from [_inputsByPackage], and delete all assets
/// that need updates.
await Future.wait(_assetGraph.allNodes
.where((node) => node is GeneratedAssetNode)
.map((node) async {
_inputsByPackage[node.id.package]?.remove(node.id);
if (node.needsUpdate) {
await _writer.delete(node.id);
}
}));
return;
Expand Down Expand Up @@ -280,8 +302,7 @@ class BuildImpl {
for (var output in expectedOutputs) {
inputNode.outputs.add(output);
_assetGraph.addIfAbsent(
output,
() => new GeneratedAssetNode(input, true, false, output));
output, () => new GeneratedAssetNode(input, true, false, output));
}

/// Skip the build step if none of the outputs need updating.
Expand Down
102 changes: 60 additions & 42 deletions lib/src/generate/watch_impl.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ class WatchImpl {
/// The [AssetCache] being used for builds.
final AssetCache _assetCache;

/// The [AssetGraph] being managed by [_buildImpl]
final AssetGraph _assetGraph;
/// The [AssetGraph] being shared with [_buildImpl]
AssetGraph get _assetGraph => _buildImpl.assetGraph;

/// The [BuildImpl] being used to run builds.
final BuildImpl _buildImpl;
Expand All @@ -40,15 +40,18 @@ class WatchImpl {
/// Injectable factory for creating directory watchers.
final DirectoryWatcherFactory _directoryWatcherFactory;

/// A logger to use!
final _logger = new Logger('Watch');

/// The [PackageGraph] for the current program.
final PackageGraph _packageGraph;

/// Shared [AssetWriter] with [_buildImpl]
final AssetWriter _writer;

/// A future that completes when the current build is done.
Future _currentBuild;

/// A logger to use!
final _logger = new Logger('Watch');

/// Whether or not another build is scheduled.
bool _nextBuildScheduled;

Expand All @@ -65,15 +68,13 @@ class WatchImpl {
this._directoryWatcherFactory,
this._debounceDelay,
this._assetCache,
AssetGraph assetGraph,
AssetReader reader,
AssetWriter writer,
PackageGraph packageGraph,
List<List<Phase>> phaseGroups)
: _assetGraph = assetGraph,
_packageGraph = packageGraph,
_buildImpl = new BuildImpl(
assetGraph, reader, writer, packageGraph, phaseGroups);
: _packageGraph = packageGraph,
_writer = writer,
_buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups);

/// Completes after the current build is done, and stops further builds from
/// happening.
Expand Down Expand Up @@ -126,51 +127,64 @@ class WatchImpl {
}
assert(_nextBuildScheduled == false);

/// Remove any updates that were generated outputs or otherwise not
/// interesting.
var updatesToRemove = updatedInputs.keys.where(_shouldSkipInput).toList();
updatesToRemove.forEach(updatedInputs.remove);
if (updatedInputs.isEmpty && !force) {
/// Copy [updatedInputs] so that it doesn't get modified after this point.
/// Any further updates will be scheduled for the next build.
///
/// Only copy the "interesting" outputs.
var updatedInputsCopy = <AssetId, ChangeType>{};
updatedInputs.forEach((input, changeType) {
if (_shouldSkipInput(input)) return;
updatedInputsCopy[input] = changeType;
});
updatedInputs.clear();
if (updatedInputsCopy.isEmpty && !force) {
return;
}

_logger.info('Preparing for next build');
_logger.info('Clearing cache for invalidated assets');
void clearNodeAndDeps(AssetId id, ChangeType rootChangeType) {
Future clearNodeAndDeps(
AssetId id, AssetId primaryInput, ChangeType rootChangeType) async {
var node = _assetGraph.get(id);
if (node == null) return;
if (node is GeneratedAssetNode) {
node.needsUpdate = true;
}
_assetCache.remove(id);
for (var output in node.outputs) {
clearNodeAndDeps(output, rootChangeType);
}

/// Update all ouputs of this asset as well.
await Future.wait(node.outputs.map((output) =>
clearNodeAndDeps(output, primaryInput, rootChangeType)));

/// For deletes, prune the graph.
if (rootChangeType == ChangeType.REMOVE) {
if (id == primaryInput && rootChangeType == ChangeType.REMOVE) {
_assetGraph.remove(id);
}
if (node is GeneratedAssetNode) {
node.needsUpdate = true;
if (rootChangeType == ChangeType.REMOVE &&
node.primaryInput == primaryInput) {
_assetGraph.remove(id);
await _writer.delete(id);
}
}
}
for (var input in updatedInputs.keys) {
clearNodeAndDeps(input, updatedInputs[input]);
}
updatedInputs.clear();

_logger.info('Starting build');
_currentBuild = _buildImpl.runBuild();
_currentBuild.then((result) {
if (result.status == BuildStatus.Success) {
_logger.info('Build completed successfully');
} else {
_logger.warning('Build failed');
}
_resultStreamController.add(result);
_currentBuild = null;
if (_nextBuildScheduled) {
_nextBuildScheduled = false;
doBuild();
}
Future
.wait(updatedInputsCopy.keys.map((input) =>
clearNodeAndDeps(input, input, updatedInputsCopy[input])))
.then((_) {
_logger.info('Starting build');
_currentBuild = _buildImpl.runBuild();
_currentBuild.then((result) {
if (result.status == BuildStatus.Success) {
_logger.info('Build completed successfully');
} else {
_logger.warning('Build failed');
}
_resultStreamController.add(result);
_currentBuild = null;
if (_nextBuildScheduled) {
_nextBuildScheduled = false;
doBuild();
}
});
});
}

Expand All @@ -189,6 +203,10 @@ class WatchImpl {
_allListeners.add(watcher.events.listen((WatchEvent e) {
_logger.fine('Got WatchEvent for path ${e.path}');
var id = new AssetId(package.name, path.normalize(e.path));
var node = _assetGraph.get(id);
// Short circuit for deletes of nodes that aren't in the graph.
if (e.type == ChangeType.REMOVE && node == null) return;

updatedInputs[id] = e.type;
scheduleBuild();
}));
Expand Down
7 changes: 5 additions & 2 deletions test/common/assets.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,8 @@ void addAssets(Iterable<Asset> assets, InMemoryAssetWriter writer) {
}
}

AssetNode makeAssetNode([String assetIdString]) =>
new AssetNode(makeAssetId(assetIdString));
AssetNode makeAssetNode([String assetIdString, List<AssetId> outputs]) {
var node = new AssetNode(makeAssetId(assetIdString));
if (outputs != null) node.outputs.addAll(outputs);
return node;
}
Loading

0 comments on commit a3df592

Please sign in to comment.