From a3df592f424443a32a7045b5a48cadaa08c3a100 Mon Sep 17 00:00:00 2001 From: Jacob MacDonald Date: Wed, 17 Feb 2016 09:24:59 -0800 Subject: [PATCH] Cache the asset graph after each build, and read it in at startup --- lib/src/generate/build.dart | 10 ++- lib/src/generate/build_impl.dart | 83 +++++++++++++++---------- lib/src/generate/watch_impl.dart | 102 ++++++++++++++++++------------- test/common/assets.dart | 7 ++- test/generate/build_test.dart | 40 +++++++----- test/generate/watch_test.dart | 18 +++++- 6 files changed, 161 insertions(+), 99 deletions(-) diff --git a/lib/src/generate/build.dart b/lib/src/generate/build.dart index f6068be43..7491f3fe1 100644 --- a/lib/src/generate/build.dart +++ b/lib/src/generate/build.dart @@ -10,7 +10,6 @@ import '../asset/cache.dart'; import '../asset/file_based.dart'; import '../asset/reader.dart'; import '../asset/writer.dart'; -import '../asset_graph/graph.dart'; import '../package_graph/package_graph.dart'; import 'build_impl.dart'; import 'build_result.dart'; @@ -52,8 +51,7 @@ Future build(List> phaseGroups, writer ??= new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph)); - var buildImpl = new BuildImpl( - new AssetGraph(), reader, writer, packageGraph, phaseGroups); + var buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups); /// Run the build! var futureResult = buildImpl.runBuild(); @@ -105,7 +103,7 @@ Stream watch(List> phaseGroups, new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph)); directoryWatcherFactory ??= defaultDirectoryWatcherFactory; var watchImpl = new WatchImpl(directoryWatcherFactory, debounceDelay, cache, - new AssetGraph(), reader, writer, packageGraph, phaseGroups); + reader, writer, packageGraph, phaseGroups); var resultStream = watchImpl.runWatch(); @@ -120,8 +118,8 @@ Stream watch(List> phaseGroups, /// Given [terminateEventStream], call [onTerminate] the first time an event is /// seen. If a second event is recieved, simply exit. -StreamSubscription _setupTerminateLogic(Stream terminateEventStream, - Future onTerminate()) { +StreamSubscription _setupTerminateLogic( + Stream terminateEventStream, Future onTerminate()) { terminateEventStream ??= ProcessSignal.SIGINT.watch(); int numEventsSeen = 0; var terminateListener; diff --git a/lib/src/generate/build_impl.dart b/lib/src/generate/build_impl.dart index 83992cfa1..8b3cbf404 100644 --- a/lib/src/generate/build_impl.dart +++ b/lib/src/generate/build_impl.dart @@ -13,6 +13,7 @@ import '../asset/exceptions.dart'; import '../asset/id.dart'; import '../asset/reader.dart'; import '../asset/writer.dart'; +import '../asset_graph/exceptions.dart'; import '../asset_graph/graph.dart'; import '../asset_graph/node.dart'; import '../builder/builder.dart'; @@ -25,7 +26,9 @@ import 'phase.dart'; /// Class which manages running builds. class BuildImpl { - final AssetGraph _assetGraph; + AssetGraph _assetGraph; + AssetGraph get assetGraph => _assetGraph; + final AssetReader _reader; final AssetWriter _writer; final PackageGraph _packageGraph; @@ -34,8 +37,7 @@ class BuildImpl { bool _buildRunning = false; final _logger = new Logger('Build'); - BuildImpl(this._assetGraph, this._reader, this._writer, this._packageGraph, - this._phaseGroups); + BuildImpl(this._reader, this._writer, this._packageGraph, this._phaseGroups); /// Runs a build /// @@ -49,6 +51,12 @@ class BuildImpl { if (_buildRunning) throw const ConcurrentBuildException(); _buildRunning = true; + /// Initialize the [assetGraph] if its not yet set up. + if (_assetGraph == null) { + _logger.info('Reading cached dependency graph'); + _assetGraph = await _readAssetGraph(); + } + /// Wait while all inputs are collected. _logger.info('Initializing inputs'); await _initializeInputsByPackage(); @@ -61,14 +69,10 @@ class BuildImpl { _logger.info('Running build phases'); var result = await _runPhases(); - /// Write out the new build_outputs file. - var allOuputs = _assetGraph.allNodes - .where((node) => node is GeneratedAssetNode && node.wasOutput); - var buildOutputsAsset = new Asset( - _buildOutputsId, - JSON.encode( - allOuputs.map((output) => output.id.serialize()).toList())); - await _writer.writeAsString(buildOutputsAsset); + /// Write out the dependency graph file. + var assetGraphAsset = + new Asset(_assetGraphId, JSON.encode(_assetGraph.serialize())); + await _writer.writeAsString(assetGraphAsset); return result; } catch (e, s) { @@ -79,27 +83,45 @@ class BuildImpl { } } - /// Asset containing previous build outputs. - AssetId get _buildOutputsId => - new AssetId(_packageGraph.root.name, '.build/build_outputs.json'); + /// Asset containing previous asset dependency graph. + AssetId get _assetGraphId => + new AssetId(_packageGraph.root.name, '.build/asset_graph.json'); + + /// Reads in the [assetGraph] from disk. + Future _readAssetGraph() async { + if (!await _reader.hasInput(_assetGraphId)) return new AssetGraph(); + try { + _logger.info('Reading cached asset graph.'); + var graph = new AssetGraph.deserialize( + JSON.decode(await _reader.readAsString(_assetGraphId))); + /// TODO(jakemac): Only invalidate nodes which need invalidating, which + /// will give us incremental builds on startup. + graph.allNodes.where((node) => node is GeneratedAssetNode) + .forEach((node) => node.needsUpdate = true); + return graph; + } on AssetGraphVersionException catch (_) { + /// Start fresh if the cached asset_graph version doesn't match up with + /// the current version. We don't currently support old graph versions. + _logger.info('Throwing away cached asset graph due to version mismatch.'); + return new AssetGraph(); + } + } /// Deletes all previous output files. Future _deletePreviousOutputs() async { - if (await _reader.hasInput(_buildOutputsId)) { - /// Cache file exists, delete all outputs which don't appear in the - /// [_assetGraph], or are marked as needing an update. - /// - /// Removes all files from [_inputsByPackage] regardless of state. - var previousOutputs = - JSON.decode(await _reader.readAsString(_buildOutputsId)); - await _writer.delete(_buildOutputsId); - _inputsByPackage[_buildOutputsId.package]?.remove(_buildOutputsId); - await Future.wait(previousOutputs.map((output) async { - var outputId = new AssetId.deserialize(output); - _inputsByPackage[outputId.package]?.remove(outputId); - var node = _assetGraph.get(outputId); - if (node == null || (node as GeneratedAssetNode).needsUpdate) { - await _writer.delete(outputId); + /// TODO(jakemac): need a cleaner way of telling if the current graph was + /// generated from cache or if its just a brand new graph. + if (await _reader.hasInput(_assetGraphId)) { + await _writer.delete(_assetGraphId); + _inputsByPackage[_assetGraphId.package]?.remove(_assetGraphId); + /// Remove all output nodes from [_inputsByPackage], and delete all assets + /// that need updates. + await Future.wait(_assetGraph.allNodes + .where((node) => node is GeneratedAssetNode) + .map((node) async { + _inputsByPackage[node.id.package]?.remove(node.id); + if (node.needsUpdate) { + await _writer.delete(node.id); } })); return; @@ -280,8 +302,7 @@ class BuildImpl { for (var output in expectedOutputs) { inputNode.outputs.add(output); _assetGraph.addIfAbsent( - output, - () => new GeneratedAssetNode(input, true, false, output)); + output, () => new GeneratedAssetNode(input, true, false, output)); } /// Skip the build step if none of the outputs need updating. diff --git a/lib/src/generate/watch_impl.dart b/lib/src/generate/watch_impl.dart index 3a9fc51fe..c04d7367e 100644 --- a/lib/src/generate/watch_impl.dart +++ b/lib/src/generate/watch_impl.dart @@ -28,8 +28,8 @@ class WatchImpl { /// The [AssetCache] being used for builds. final AssetCache _assetCache; - /// The [AssetGraph] being managed by [_buildImpl] - final AssetGraph _assetGraph; + /// The [AssetGraph] being shared with [_buildImpl] + AssetGraph get _assetGraph => _buildImpl.assetGraph; /// The [BuildImpl] being used to run builds. final BuildImpl _buildImpl; @@ -40,15 +40,18 @@ class WatchImpl { /// Injectable factory for creating directory watchers. final DirectoryWatcherFactory _directoryWatcherFactory; + /// A logger to use! + final _logger = new Logger('Watch'); + /// The [PackageGraph] for the current program. final PackageGraph _packageGraph; + /// Shared [AssetWriter] with [_buildImpl] + final AssetWriter _writer; + /// A future that completes when the current build is done. Future _currentBuild; - /// A logger to use! - final _logger = new Logger('Watch'); - /// Whether or not another build is scheduled. bool _nextBuildScheduled; @@ -65,15 +68,13 @@ class WatchImpl { this._directoryWatcherFactory, this._debounceDelay, this._assetCache, - AssetGraph assetGraph, AssetReader reader, AssetWriter writer, PackageGraph packageGraph, List> phaseGroups) - : _assetGraph = assetGraph, - _packageGraph = packageGraph, - _buildImpl = new BuildImpl( - assetGraph, reader, writer, packageGraph, phaseGroups); + : _packageGraph = packageGraph, + _writer = writer, + _buildImpl = new BuildImpl(reader, writer, packageGraph, phaseGroups); /// Completes after the current build is done, and stops further builds from /// happening. @@ -126,51 +127,64 @@ class WatchImpl { } assert(_nextBuildScheduled == false); - /// Remove any updates that were generated outputs or otherwise not - /// interesting. - var updatesToRemove = updatedInputs.keys.where(_shouldSkipInput).toList(); - updatesToRemove.forEach(updatedInputs.remove); - if (updatedInputs.isEmpty && !force) { + /// Copy [updatedInputs] so that it doesn't get modified after this point. + /// Any further updates will be scheduled for the next build. + /// + /// Only copy the "interesting" outputs. + var updatedInputsCopy = {}; + updatedInputs.forEach((input, changeType) { + if (_shouldSkipInput(input)) return; + updatedInputsCopy[input] = changeType; + }); + updatedInputs.clear(); + if (updatedInputsCopy.isEmpty && !force) { return; } _logger.info('Preparing for next build'); - _logger.info('Clearing cache for invalidated assets'); - void clearNodeAndDeps(AssetId id, ChangeType rootChangeType) { + Future clearNodeAndDeps( + AssetId id, AssetId primaryInput, ChangeType rootChangeType) async { var node = _assetGraph.get(id); if (node == null) return; - if (node is GeneratedAssetNode) { - node.needsUpdate = true; - } _assetCache.remove(id); - for (var output in node.outputs) { - clearNodeAndDeps(output, rootChangeType); - } + + /// Update all ouputs of this asset as well. + await Future.wait(node.outputs.map((output) => + clearNodeAndDeps(output, primaryInput, rootChangeType))); /// For deletes, prune the graph. - if (rootChangeType == ChangeType.REMOVE) { + if (id == primaryInput && rootChangeType == ChangeType.REMOVE) { _assetGraph.remove(id); } + if (node is GeneratedAssetNode) { + node.needsUpdate = true; + if (rootChangeType == ChangeType.REMOVE && + node.primaryInput == primaryInput) { + _assetGraph.remove(id); + await _writer.delete(id); + } + } } - for (var input in updatedInputs.keys) { - clearNodeAndDeps(input, updatedInputs[input]); - } - updatedInputs.clear(); - _logger.info('Starting build'); - _currentBuild = _buildImpl.runBuild(); - _currentBuild.then((result) { - if (result.status == BuildStatus.Success) { - _logger.info('Build completed successfully'); - } else { - _logger.warning('Build failed'); - } - _resultStreamController.add(result); - _currentBuild = null; - if (_nextBuildScheduled) { - _nextBuildScheduled = false; - doBuild(); - } + Future + .wait(updatedInputsCopy.keys.map((input) => + clearNodeAndDeps(input, input, updatedInputsCopy[input]))) + .then((_) { + _logger.info('Starting build'); + _currentBuild = _buildImpl.runBuild(); + _currentBuild.then((result) { + if (result.status == BuildStatus.Success) { + _logger.info('Build completed successfully'); + } else { + _logger.warning('Build failed'); + } + _resultStreamController.add(result); + _currentBuild = null; + if (_nextBuildScheduled) { + _nextBuildScheduled = false; + doBuild(); + } + }); }); } @@ -189,6 +203,10 @@ class WatchImpl { _allListeners.add(watcher.events.listen((WatchEvent e) { _logger.fine('Got WatchEvent for path ${e.path}'); var id = new AssetId(package.name, path.normalize(e.path)); + var node = _assetGraph.get(id); + // Short circuit for deletes of nodes that aren't in the graph. + if (e.type == ChangeType.REMOVE && node == null) return; + updatedInputs[id] = e.type; scheduleBuild(); })); diff --git a/test/common/assets.dart b/test/common/assets.dart index 583f1a192..d023af108 100644 --- a/test/common/assets.dart +++ b/test/common/assets.dart @@ -35,5 +35,8 @@ void addAssets(Iterable assets, InMemoryAssetWriter writer) { } } -AssetNode makeAssetNode([String assetIdString]) => - new AssetNode(makeAssetId(assetIdString)); +AssetNode makeAssetNode([String assetIdString, List outputs]) { + var node = new AssetNode(makeAssetId(assetIdString)); + if (outputs != null) node.outputs.addAll(outputs); + return node; +} diff --git a/test/generate/build_test.dart b/test/generate/build_test.dart index 8d558ea0f..36f7f30d6 100644 --- a/test/generate/build_test.dart +++ b/test/generate/build_test.dart @@ -8,6 +8,7 @@ import 'package:logging/logging.dart'; import 'package:test/test.dart'; import 'package:build/build.dart'; +import 'package:build/src/asset_graph/graph.dart'; import '../common/common.dart'; @@ -166,19 +167,20 @@ main() { new Phase([new CopyBuilder()], [new InputSet('a')]), ] ]; + var emptyGraph = new AssetGraph(); await testPhases( phases, { 'a|lib/a.txt': 'a', 'a|lib/a.txt.copy': 'a', - 'a|.build/build_outputs.json': '[]', + 'a|.build/asset_graph.json': JSON.encode(emptyGraph.serialize()), }, status: BuildStatus.Failure, exceptionMatcher: invalidOutputException); }); }); - test('tracks previous outputs in a build_outputs.json file', () async { + test('tracks dependency graph in a asset_graph.json file', () async { var phases = [ [ new Phase([new CopyBuilder()], [new InputSet('a')]), @@ -189,15 +191,20 @@ main() { outputs: {'a|web/a.txt.copy': 'a', 'a|lib/b.txt.copy': 'b'}, writer: writer); - var outputId = makeAssetId('a|.build/build_outputs.json'); - expect(writer.assets, contains(outputId)); - var outputs = JSON.decode(writer.assets[outputId]); - expect( - outputs, - unorderedEquals([ - ['a', 'web/a.txt.copy'], - ['a', 'lib/b.txt.copy'], - ])); + var graphId = makeAssetId('a|.build/asset_graph.json'); + expect(writer.assets, contains(graphId)); + var cachedGraph = + new AssetGraph.deserialize(JSON.decode(writer.assets[graphId])); + + var expectedGraph = new AssetGraph(); + var aCopyNode = makeAssetNode('a|web/a.txt.copy'); + expectedGraph.add(aCopyNode); + expectedGraph.add(makeAssetNode('a|web/a.txt', [aCopyNode.id])); + var bCopyNode = makeAssetNode('a|lib/b.txt.copy'); + expectedGraph.add(bCopyNode); + expectedGraph.add(makeAssetNode('a|lib/b.txt', [bCopyNode.id])); + + expect(cachedGraph, equalsAssetGraph(expectedGraph)); }); test('outputs from previous full builds shouldn\'t be inputs to later ones', @@ -216,7 +223,7 @@ main() { await testPhases(phases, inputs, outputs: outputs, writer: writer); }); - test('can recover from a deleted build_outputs.json cache', () async { + test('can recover from a deleted asset_graph.json cache', () async { var phases = [ [ new Phase([new CopyBuilder()], [new InputSet('a')]), @@ -228,8 +235,8 @@ main() { // First run, nothing special. await testPhases(phases, inputs, outputs: outputs, writer: writer); - // Delete the `build_outputs.json` file! - var outputId = makeAssetId('a|.build/build_outputs.json'); + // Delete the `asset_graph.json` file! + var outputId = makeAssetId('a|.build/asset_graph.json'); await writer.delete(outputId); // Second run, should have no extra outputs. @@ -261,7 +268,10 @@ testPhases(List> phases, Map inputs, } var result = await build(phases, - reader: reader, writer: writer, packageGraph: packageGraph, logLevel: Level.OFF); + reader: reader, + writer: writer, + packageGraph: packageGraph, + logLevel: Level.OFF); expect(result.status, status, reason: 'Exception:\n${result.exception}\n' 'Stack Trace:\n${result.stackTrace}'); diff --git a/test/generate/watch_test.dart b/test/generate/watch_test.dart index f865e2400..427075473 100644 --- a/test/generate/watch_test.dart +++ b/test/generate/watch_test.dart @@ -2,12 +2,14 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'package:logging/logging.dart'; import 'package:test/test.dart'; import 'package:watcher/watcher.dart'; import 'package:build/build.dart'; +import 'package:build/src/asset_graph/graph.dart'; import '../common/common.dart'; @@ -99,7 +101,7 @@ main() { expect(writer.assets[makeAssetId('a|web/b.txt.copy')], 'b'); }); - test('rebuilds properly update build_outputs.json', () async { + test('rebuilds properly update asset_graph.json', () async { var phases = [ [ new Phase([new CopyBuilder()], [new InputSet('a')]), @@ -124,8 +126,18 @@ main() { result = await nextResult(results); checkOutputs({'a|web/c.txt.copy': 'c'}, result, writer.assets); - expect(writer.assets[makeAssetId('a|.build/build_outputs.json')], - '[["a","web/b.txt.copy"],["a","web/c.txt.copy"]]'); + var cachedGraph = new AssetGraph.deserialize(JSON + .decode(writer.assets[makeAssetId('a|.build/asset_graph.json')])); + + var expectedGraph = new AssetGraph(); + var bCopyNode = makeAssetNode('a|web/b.txt.copy'); + expectedGraph.add(bCopyNode); + expectedGraph.add(makeAssetNode('a|web/b.txt', [bCopyNode.id])); + var cCopyNode = makeAssetNode('a|web/c.txt.copy'); + expectedGraph.add(cCopyNode); + expectedGraph.add(makeAssetNode('a|web/c.txt', [cCopyNode.id])); + + expect(cachedGraph, equalsAssetGraph(expectedGraph)); }); });