Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache the asset graph after each build, and read it in at startup #50

Merged
merged 1 commit into from
Feb 17, 2016
Merged

Cache the asset graph after each build, and read it in at startup #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions lib/src/generate/build.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import '../asset/cache.dart';
import '../asset/file_based.dart';
import '../asset/reader.dart';
import '../asset/writer.dart';
import '../asset_graph/graph.dart';
import '../package_graph/package_graph.dart';
import 'build_impl.dart';
import 'build_result.dart';
Expand Down Expand Up @@ -52,8 +51,7 @@ Future<BuildResult> build(List<List<Phase>> phaseGroups,
writer ??=
new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));

var buildImpl = new BuildImpl(
new AssetGraph(), reader, writer, packageGraph, phaseGroups);
var buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups);

/// Run the build!
var futureResult = buildImpl.runBuild();
Expand Down Expand Up @@ -105,7 +103,7 @@ Stream<BuildResult> watch(List<List<Phase>> phaseGroups,
new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));
directoryWatcherFactory ??= defaultDirectoryWatcherFactory;
var watchImpl = new WatchImpl(directoryWatcherFactory, debounceDelay, cache,
new AssetGraph(), reader, writer, packageGraph, phaseGroups);
reader, writer, packageGraph, phaseGroups);

var resultStream = watchImpl.runWatch();

Expand All @@ -120,8 +118,8 @@ Stream<BuildResult> watch(List<List<Phase>> phaseGroups,

/// Given [terminateEventStream], call [onTerminate] the first time an event is
/// seen. If a second event is recieved, simply exit.
StreamSubscription _setupTerminateLogic(Stream terminateEventStream,
Future onTerminate()) {
StreamSubscription _setupTerminateLogic(
Stream terminateEventStream, Future onTerminate()) {
terminateEventStream ??= ProcessSignal.SIGINT.watch();
int numEventsSeen = 0;
var terminateListener;
Expand Down
83 changes: 52 additions & 31 deletions lib/src/generate/build_impl.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import '../asset/exceptions.dart';
import '../asset/id.dart';
import '../asset/reader.dart';
import '../asset/writer.dart';
import '../asset_graph/exceptions.dart';
import '../asset_graph/graph.dart';
import '../asset_graph/node.dart';
import '../builder/builder.dart';
Expand All @@ -25,7 +26,9 @@ import 'phase.dart';

/// Class which manages running builds.
class BuildImpl {
final AssetGraph _assetGraph;
AssetGraph _assetGraph;
AssetGraph get assetGraph => _assetGraph;

final AssetReader _reader;
final AssetWriter _writer;
final PackageGraph _packageGraph;
Expand All @@ -34,8 +37,7 @@ class BuildImpl {
bool _buildRunning = false;
final _logger = new Logger('Build');

BuildImpl(this._assetGraph, this._reader, this._writer, this._packageGraph,
this._phaseGroups);
BuildImpl(this._reader, this._writer, this._packageGraph, this._phaseGroups);

/// Runs a build
///
Expand All @@ -49,6 +51,12 @@ class BuildImpl {
if (_buildRunning) throw const ConcurrentBuildException();
_buildRunning = true;

/// Initialize the [assetGraph] if its not yet set up.
if (_assetGraph == null) {
_logger.info('Reading cached dependency graph');
_assetGraph = await _readAssetGraph();
}

/// Wait while all inputs are collected.
_logger.info('Initializing inputs');
await _initializeInputsByPackage();
Expand All @@ -61,14 +69,10 @@ class BuildImpl {
_logger.info('Running build phases');
var result = await _runPhases();

/// Write out the new build_outputs file.
var allOuputs = _assetGraph.allNodes
.where((node) => node is GeneratedAssetNode && node.wasOutput);
var buildOutputsAsset = new Asset(
_buildOutputsId,
JSON.encode(
allOuputs.map((output) => output.id.serialize()).toList()));
await _writer.writeAsString(buildOutputsAsset);
/// Write out the dependency graph file.
var assetGraphAsset =
new Asset(_assetGraphId, JSON.encode(_assetGraph.serialize()));
await _writer.writeAsString(assetGraphAsset);

return result;
} catch (e, s) {
Expand All @@ -79,27 +83,45 @@ class BuildImpl {
}
}

/// Asset containing previous build outputs.
AssetId get _buildOutputsId =>
new AssetId(_packageGraph.root.name, '.build/build_outputs.json');
/// Asset containing previous asset dependency graph.
AssetId get _assetGraphId =>
new AssetId(_packageGraph.root.name, '.build/asset_graph.json');

/// Reads in the [assetGraph] from disk.
Future<AssetGraph> _readAssetGraph() async {
if (!await _reader.hasInput(_assetGraphId)) return new AssetGraph();
try {
_logger.info('Reading cached asset graph.');
var graph = new AssetGraph.deserialize(
JSON.decode(await _reader.readAsString(_assetGraphId)));
/// TODO(jakemac): Only invalidate nodes which need invalidating, which
/// will give us incremental builds on startup.
graph.allNodes.where((node) => node is GeneratedAssetNode)
.forEach((node) => node.needsUpdate = true);
return graph;
} on AssetGraphVersionException catch (_) {
/// Start fresh if the cached asset_graph version doesn't match up with
/// the current version. We don't currently support old graph versions.
_logger.info('Throwing away cached asset graph due to version mismatch.');
return new AssetGraph();
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In context, this is a reasonable return, but taken alone it has a bad code smell. Add a comment explaining why ignoring the error & returning an empty AssetGraph is the right way to handle this.

Is it worthwhile to log the error?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done, logged a message as well

}
}

/// Deletes all previous output files.
Future _deletePreviousOutputs() async {
if (await _reader.hasInput(_buildOutputsId)) {
/// Cache file exists, delete all outputs which don't appear in the
/// [_assetGraph], or are marked as needing an update.
///
/// Removes all files from [_inputsByPackage] regardless of state.
var previousOutputs =
JSON.decode(await _reader.readAsString(_buildOutputsId));
await _writer.delete(_buildOutputsId);
_inputsByPackage[_buildOutputsId.package]?.remove(_buildOutputsId);
await Future.wait(previousOutputs.map((output) async {
var outputId = new AssetId.deserialize(output);
_inputsByPackage[outputId.package]?.remove(outputId);
var node = _assetGraph.get(outputId);
if (node == null || (node as GeneratedAssetNode).needsUpdate) {
await _writer.delete(outputId);
/// TODO(jakemac): need a cleaner way of telling if the current graph was
/// generated from cache or if its just a brand new graph.
if (await _reader.hasInput(_assetGraphId)) {
await _writer.delete(_assetGraphId);
_inputsByPackage[_assetGraphId.package]?.remove(_assetGraphId);
/// Remove all output nodes from [_inputsByPackage], and delete all assets
/// that need updates.
await Future.wait(_assetGraph.allNodes
.where((node) => node is GeneratedAssetNode)
.map((node) async {
_inputsByPackage[node.id.package]?.remove(node.id);
if (node.needsUpdate) {
await _writer.delete(node.id);
}
}));
return;
Expand Down Expand Up @@ -280,8 +302,7 @@ class BuildImpl {
for (var output in expectedOutputs) {
inputNode.outputs.add(output);
_assetGraph.addIfAbsent(
output,
() => new GeneratedAssetNode(input, true, false, output));
output, () => new GeneratedAssetNode(input, true, false, output));
}

/// Skip the build step if none of the outputs need updating.
Expand Down
102 changes: 60 additions & 42 deletions lib/src/generate/watch_impl.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ class WatchImpl {
/// The [AssetCache] being used for builds.
final AssetCache _assetCache;

/// The [AssetGraph] being managed by [_buildImpl]
final AssetGraph _assetGraph;
/// The [AssetGraph] being shared with [_buildImpl]
AssetGraph get _assetGraph => _buildImpl.assetGraph;

/// The [BuildImpl] being used to run builds.
final BuildImpl _buildImpl;
Expand All @@ -40,15 +40,18 @@ class WatchImpl {
/// Injectable factory for creating directory watchers.
final DirectoryWatcherFactory _directoryWatcherFactory;

/// A logger to use!
final _logger = new Logger('Watch');

/// The [PackageGraph] for the current program.
final PackageGraph _packageGraph;

/// Shared [AssetWriter] with [_buildImpl]
final AssetWriter _writer;

/// A future that completes when the current build is done.
Future _currentBuild;

/// A logger to use!
final _logger = new Logger('Watch');

/// Whether or not another build is scheduled.
bool _nextBuildScheduled;

Expand All @@ -65,15 +68,13 @@ class WatchImpl {
this._directoryWatcherFactory,
this._debounceDelay,
this._assetCache,
AssetGraph assetGraph,
AssetReader reader,
AssetWriter writer,
PackageGraph packageGraph,
List<List<Phase>> phaseGroups)
: _assetGraph = assetGraph,
_packageGraph = packageGraph,
_buildImpl = new BuildImpl(
assetGraph, reader, writer, packageGraph, phaseGroups);
: _packageGraph = packageGraph,
_writer = writer,
_buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups);

/// Completes after the current build is done, and stops further builds from
/// happening.
Expand Down Expand Up @@ -126,51 +127,64 @@ class WatchImpl {
}
assert(_nextBuildScheduled == false);

/// Remove any updates that were generated outputs or otherwise not
/// interesting.
var updatesToRemove = updatedInputs.keys.where(_shouldSkipInput).toList();
updatesToRemove.forEach(updatedInputs.remove);
if (updatedInputs.isEmpty && !force) {
/// Copy [updatedInputs] so that it doesn't get modified after this point.
/// Any further updates will be scheduled for the next build.
///
/// Only copy the "interesting" outputs.
var updatedInputsCopy = <AssetId, ChangeType>{};
updatedInputs.forEach((input, changeType) {
if (_shouldSkipInput(input)) return;
updatedInputsCopy[input] = changeType;
});
updatedInputs.clear();
if (updatedInputsCopy.isEmpty && !force) {
return;
}

_logger.info('Preparing for next build');
_logger.info('Clearing cache for invalidated assets');
void clearNodeAndDeps(AssetId id, ChangeType rootChangeType) {
Future clearNodeAndDeps(
AssetId id, AssetId primaryInput, ChangeType rootChangeType) async {
var node = _assetGraph.get(id);
if (node == null) return;
if (node is GeneratedAssetNode) {
node.needsUpdate = true;
}
_assetCache.remove(id);
for (var output in node.outputs) {
clearNodeAndDeps(output, rootChangeType);
}

/// Update all ouputs of this asset as well.
await Future.wait(node.outputs.map((output) =>
clearNodeAndDeps(output, primaryInput, rootChangeType)));

/// For deletes, prune the graph.
if (rootChangeType == ChangeType.REMOVE) {
if (id == primaryInput && rootChangeType == ChangeType.REMOVE) {
_assetGraph.remove(id);
}
if (node is GeneratedAssetNode) {
node.needsUpdate = true;
if (rootChangeType == ChangeType.REMOVE &&
node.primaryInput == primaryInput) {
_assetGraph.remove(id);
await _writer.delete(id);
}
}
}
for (var input in updatedInputs.keys) {
clearNodeAndDeps(input, updatedInputs[input]);
}
updatedInputs.clear();

_logger.info('Starting build');
_currentBuild = _buildImpl.runBuild();
_currentBuild.then((result) {
if (result.status == BuildStatus.Success) {
_logger.info('Build completed successfully');
} else {
_logger.warning('Build failed');
}
_resultStreamController.add(result);
_currentBuild = null;
if (_nextBuildScheduled) {
_nextBuildScheduled = false;
doBuild();
}
Future
.wait(updatedInputsCopy.keys.map((input) =>
clearNodeAndDeps(input, input, updatedInputsCopy[input])))
.then((_) {
_logger.info('Starting build');
_currentBuild = _buildImpl.runBuild();
_currentBuild.then((result) {
if (result.status == BuildStatus.Success) {
_logger.info('Build completed successfully');
} else {
_logger.warning('Build failed');
}
_resultStreamController.add(result);
_currentBuild = null;
if (_nextBuildScheduled) {
_nextBuildScheduled = false;
doBuild();
}
});
});
}

Expand All @@ -189,6 +203,10 @@ class WatchImpl {
_allListeners.add(watcher.events.listen((WatchEvent e) {
_logger.fine('Got WatchEvent for path ${e.path}');
var id = new AssetId(package.name, path.normalize(e.path));
var node = _assetGraph.get(id);
// Short circuit for deletes of nodes that aren't in the graph.
if (e.type == ChangeType.REMOVE && node == null) return;

updatedInputs[id] = e.type;
scheduleBuild();
}));
Expand Down
7 changes: 5 additions & 2 deletions test/common/assets.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,8 @@ void addAssets(Iterable<Asset> assets, InMemoryAssetWriter writer) {
}
}

AssetNode makeAssetNode([String assetIdString]) =>
new AssetNode(makeAssetId(assetIdString));
AssetNode makeAssetNode([String assetIdString, List<AssetId> outputs]) {
var node = new AssetNode(makeAssetId(assetIdString));
if (outputs != null) node.outputs.addAll(outputs);
return node;
}
Loading