add caching of assets

.gitignore

@@ -4,5 +4,8 @@ build/
 .pub/
 pubspec.lock
 
+# Generated by tool/create_all_test.dart
+tool/test_all.dart
+
 # Include .packages files from tests which are hand coded
 !test/fixtures/**/.packages

lib/src/asset/cache.dart

@@ -0,0 +1,95 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+import 'dart:async';
+import 'dart:convert';
+
+import 'asset.dart';
+import 'id.dart';
+import 'reader.dart';
+import 'writer.dart';
+
+/// A basic [Asset] cache by [AssetId].
+class AssetCache {
+  /// Caches the values for this cache.
+  final Map<AssetId, Asset> _cache = {};
+
+  /// Whether or not this value exists in the cache.
+  bool contains(AssetId id) => _cache.containsKey(id);
+
+  /// Get [id] from the cache.
+  Asset get(AssetId id) => _cache[id];
+
+  /// Put [asset] into the cache.
+  ///
+  /// By default it will not overwrite existing assets, and will throw if
+  /// [asset] already exists.
+  void put(Asset asset, {bool overwrite: false}) {
+    if (!overwrite && contains(asset.id)) {
+      throw new ArgumentError('$asset already exists in the cache.');
+    }
+    _cache[asset.id] = asset;
+  }
+
+  /// Removes [id] from the cache, and returns the [Asset] for it if present.
+  Asset remove(AssetId id) => _cache.remove(id);
+}
+
+/// An [AssetReader] which takes both an [AssetCache] and an [AssetReader]. It
+/// will first look up values in the [AssetCache], and if not present it will
+/// use the [AssetReader] and add the [Asset] to the [AssetCache].
+class CachedAssetReader extends AssetReader {
+  final AssetCache _cache;
+  final AssetReader _reader;
+  /// Cache of ongoing reads by [AssetId].
+  final _pendingReads = <AssetId, Future<String>>{};
+  /// Cache of ongoing hasInput checks by [AssetId].
+  final _pendingHasInputChecks = <AssetId, Future<bool>>{};
+
+  CachedAssetReader(this._cache, this._reader);
+
+  @override
+  Future<bool> hasInput(AssetId id) {
+    if (_cache.contains(id)) return new Future.value(true);
+
+    _pendingHasInputChecks.putIfAbsent(id, () async {
+      var exists = await _reader.hasInput(id);
+      _pendingHasInputChecks.remove(id);
+      return exists;
+    });
+    return _pendingHasInputChecks[id];
+  }
+
+  @override
+  Future<String> readAsString(AssetId id, {Encoding encoding: UTF8}) {
+    if (_cache.contains(id)) {
+      return new Future.value(_cache.get(id).stringContents);
+    }
+
+    _pendingReads.putIfAbsent(id, () async {
+      var content = await _reader.readAsString(id, encoding: encoding);
+      _cache.put(new Asset(id, content));
+      _pendingReads.remove(id);
+      return content;
+    });
+    return _pendingReads[id];
+  }
+}
+
+/// An [AssetWriter] which takes both an [AssetCache] and an [AssetWriter]. It
+/// writes [Asset]s to both always.
+///
+/// Writes are done synchronously to the cache, and then asynchronously to the
+/// [AssetWriter].
+class CachedAssetWriter extends AssetWriter {
+  final AssetCache _cache;
+  final AssetWriter _writer;
+
+  CachedAssetWriter(this._cache, this._writer);
+
+  @override
+  Future writeAsString(Asset asset, {Encoding encoding: UTF8}) {
+    _cache.put(asset);
+    return _writer.writeAsString(asset, encoding: encoding);
+  }
+}

lib/src/generate/build.dart

@@ -8,6 +8,7 @@ import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
 
 import '../asset/asset.dart';
+import '../asset/cache.dart';
 import '../asset/file_based.dart';
 import '../asset/id.dart';
 import '../asset/reader.dart';
@@ -43,8 +44,11 @@ Future<BuildResult> build(List<List<Phase>> phaseGroups,
   onLog ??= print;
   var logListener = Logger.root.onRecord.listen(onLog);
   packageGraph ??= new PackageGraph.forThisPackage();
-  reader ??= new FileBasedAssetReader(packageGraph);
-  writer ??= new FileBasedAssetWriter(packageGraph);
+  var cache = new AssetCache();
+  reader ??=
+      new CachedAssetReader(cache, new FileBasedAssetReader(packageGraph));
+  writer ??=
+      new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));
   var result = runZoned(() {
     _validatePhases(phaseGroups);
     return _runPhases(phaseGroups);

test/asset/cache_test.dart

@@ -0,0 +1,174 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+@TestOn('vm')
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+import 'package:build/build.dart';
+import 'package:build/src/asset/cache.dart';
+
+import '../common/common.dart';
+
+main() {
+  AssetCache cache;
+  final a = makeAsset();
+  final b = makeAsset();
+  final c = makeAsset();
+  final otherA = makeAsset('${a.id}', 'some other content');
+
+  group('AssetCache', () {
+    setUp(() {
+      cache = new AssetCache();
+      cache..put(a)..put(b)..put(c);
+    });
+
+    test('can put assets', () {
+      var d = makeAsset();
+      expect(cache.put(d), null);
+      expect(cache.get(d.id), d);
+    });
+
+    test('can get assets', () {
+      expect(cache.get(a.id), a);
+      expect(cache.get(b.id), b);
+      expect(cache.get(c.id), c);
+      expect(cache.get(makeAssetId()), isNull);
+    });
+
+    test('can check for existence of assets', () {
+      expect(cache.contains(a.id), isTrue);
+      expect(cache.contains(b.id), isTrue);
+      expect(cache.contains(c.id), isTrue);
+      expect(cache.contains(makeAssetId()), isFalse);
+    });
+
+    test('can remove assets', () {
+      expect(cache.remove(b.id), b);
+      expect(cache.contains(b.id), isFalse);
+      expect(cache.get(b.id), isNull);
+    });
+
+    test('put throws if the asset exists and overwrite == false', () {
+      var asset = makeAsset('${a.id}');
+      expect(() => cache.put(asset), throwsArgumentError);
+    });
+
+    test('put doesn\'t throw if the asset exists and overwrite == true', () {
+      cache.put(otherA, overwrite: true);
+      expect(cache.get(otherA.id), otherA);
+      expect(cache.get(a.id), isNot(a));
+    });
+  });
+
+  group('CachedAssetReader', () {
+    CachedAssetReader reader;
+    InMemoryAssetReader childReader;
+    Map<AssetId, String> childReaderAssets;
+
+    setUp(() {
+      cache = new AssetCache();
+      childReaderAssets = {};
+      childReader = new InMemoryAssetReader(childReaderAssets);
+      reader = new CachedAssetReader(cache, childReader);
+    });
+
+    test('reads from the cache first', () async {
+      expect(a.stringContents, isNot(otherA.stringContents));
+      expect(a.id, otherA.id);
+
+      childReaderAssets[otherA.id] = otherA.stringContents;
+      cache.put(a);
+      expect(await reader.readAsString(a.id), a.stringContents);
+    });
+
+    test('falls back on the childReader', () async {
+      childReaderAssets[a.id] = a.stringContents;
+      expect(await reader.readAsString(a.id), a.stringContents);
+    });
+
+    test('reads add values to the cache', () async {
+      childReaderAssets[a.id] = a.stringContents;
+      await reader.readAsString(a.id);
+      expect(cache.get(a.id), equalsAsset(a));
+      childReaderAssets.remove(a.id);
+      expect(await reader.readAsString(a.id), a.stringContents);
+    });
+
+    test('hasInput checks the cache and child reader', () async {
+      expect(await reader.hasInput(a.id), isFalse);
+      cache.put(a);
+      expect(await reader.hasInput(a.id), isTrue);
+      childReaderAssets[a.id] = a.stringContents;
+      expect(await reader.hasInput(a.id), isTrue);
+      cache.remove(a.id);
+      expect(await reader.hasInput(a.id), isTrue);
+      childReaderAssets.remove(a.id);
+      expect(await reader.hasInput(a.id), isFalse);
+    });
+
+    test('Multiple readAsString calls wait on the same future', () async {
+      childReaderAssets[a.id] = a.stringContents;
+      var futures = [];
+      futures.add(reader.readAsString(a.id));
+      futures.add(reader.readAsString(a.id));
+      expect(futures[0], futures[1]);
+      await Future.wait(futures);
+
+      // Subsequent calls should not return the same future.
+      expect(reader.readAsString(a.id), isNot(futures[0]));
+    });
+
+    test('Multiple hasInput calls return the same future', () async {
+      childReaderAssets[a.id] = a.stringContents;
+      var futures = [];
+      futures.add(reader.hasInput(a.id));
+      futures.add(reader.hasInput(a.id));
+      expect(futures[0], futures[1]);
+      await Future.wait(futures);
+
+      // Subsequent calls should not return the same future.
+      expect(reader.hasInput(a.id), isNot(futures[0]));
+    });
+  });
+
+  group('CachedAssetWriter', () {
+    CachedAssetWriter writer;
+    InMemoryAssetWriter childWriter;
+    Map<AssetId, String> childWriterAssets;
+
+    setUp(() {
+      cache = new AssetCache();
+      childWriter = new InMemoryAssetWriter();
+      childWriterAssets = childWriter.assets;
+      writer = new CachedAssetWriter(cache, childWriter);
+    });
+
+    test('writes to the cache synchronously', () {
+      writer.writeAsString(a);
+      expect(cache.get(a.id), a);
+      expect(childWriterAssets[a.id], isNull);
+    });
+
+    test('writes to the cache and the child writer', () async {
+      await writer.writeAsString(a);
+      expect(cache.get(a.id), a);
+      expect(childWriterAssets[a.id], a.stringContents);
+
+      await writer.writeAsString(b);
+      expect(cache.get(b.id), b);
+      expect(childWriterAssets[b.id], b.stringContents);
+    });
+
+    test('multiple sync writes for the same asset throws', () async {
+      writer.writeAsString(a);
+      expect(() => writer.writeAsString(a), throwsArgumentError);
+    });
+
+    test('multiple async writes for the same asset throws', () async {
+      await writer.writeAsString(a);
+      expect(() => writer.writeAsString(a), throwsArgumentError);
+    });
+  });
+}

test/common/matchers.dart

@@ -9,3 +9,21 @@ final assetNotFoundException = new isInstanceOf<AssetNotFoundException>();
 final invalidInputException = new isInstanceOf<InvalidInputException>();
 final invalidOutputException = new isInstanceOf<InvalidOutputException>();
 final packageNotFoundException = new isInstanceOf<PackageNotFoundException>();
+
+equalsAsset(Asset expected) => new _AssetMatcher(expected);
+
+class _AssetMatcher extends Matcher {
+  final Asset _expected;
+
+  const _AssetMatcher(this._expected);
+
+  @override
+  bool matches(item, _) =>
+      item is Asset &&
+      item.id == _expected.id &&
+      item.stringContents == _expected.stringContents;
+
+  @override
+  Description describe(Description description) =>
+      description.addDescriptionOf(_expected);
+}