diff --git a/pkgs/test_descriptor/.travis.yml b/pkgs/test_descriptor/.travis.yml index 415bad401..48803a1c0 100644 --- a/pkgs/test_descriptor/.travis.yml +++ b/pkgs/test_descriptor/.travis.yml @@ -2,7 +2,7 @@ language: dart dart: - dev - - 2.0.0 + - stable dart_task: - test @@ -14,7 +14,7 @@ matrix: - dart: dev dart_task: dartanalyzer: --fatal-infos --fatal-warnings . - - dart: 2.0.0 + - dart: stable dart_task: dartanalyzer: --fatal-warnings . diff --git a/pkgs/test_descriptor/CHANGELOG.md b/pkgs/test_descriptor/CHANGELOG.md index 46908ef16..a3b352908 100644 --- a/pkgs/test_descriptor/CHANGELOG.md +++ b/pkgs/test_descriptor/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.2.0 + +* Add an `ArchiveDescriptor` class and a corresponding `archive()` function that + can create and validate Zip and TAR archives. + ## 1.1.1 * Update to lowercase Dart core library constants. diff --git a/pkgs/test_descriptor/lib/src/archive_descriptor.dart b/pkgs/test_descriptor/lib/src/archive_descriptor.dart new file mode 100644 index 000000000..564958da3 --- /dev/null +++ b/pkgs/test_descriptor/lib/src/archive_descriptor.dart @@ -0,0 +1,180 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:async/async.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; + +import 'descriptor.dart'; +import 'directory_descriptor.dart'; +import 'file_descriptor.dart'; +import 'sandbox.dart'; +import 'utils.dart'; + +/// A [Descriptor] describing files in a Tar or Zip archive. +/// +/// The format is determined by the descriptor's file extension. +@sealed +class ArchiveDescriptor extends Descriptor implements FileDescriptor { + /// Descriptors for entries in this archive. + final List contents; + + /// Returns a `package:archive` [Archive] object that contains the contents of + /// this file. + Future get archive async { + var archive = Archive(); + (await _files(contents)).forEach(archive.addFile); + return archive; + } + + File get io => File(p.join(sandbox, name)); + + /// Returns [ArchiveFile]s for each file in [descriptors]. + /// + /// If [parent] is passed, it's used as the parent directory for filenames. + Future> _files(Iterable descriptors, + [String parent]) async { + return (await waitAndReportErrors(descriptors.map((descriptor) async { + var fullName = + parent == null ? descriptor.name : "$parent/${descriptor.name}"; + + if (descriptor is FileDescriptor) { + var bytes = await collectBytes(descriptor.readAsBytes()); + return [ + ArchiveFile(fullName, bytes.length, bytes) + // Setting the mode and mod time are necessary to work around + // brendan-duncan/archive#76. + ..mode = 428 + ..lastModTime = DateTime.now().millisecondsSinceEpoch ~/ 1000 + ]; + } else if (descriptor is DirectoryDescriptor) { + return await _files(descriptor.contents, fullName); + } else { + throw UnsupportedError( + "An archive can only be created from FileDescriptors and " + "DirectoryDescriptors."); + } + }))) + .expand((files) => files); + } + + ArchiveDescriptor(String name, Iterable contents) + : contents = List.unmodifiable(contents), + super(name); + + Future create([String parent]) async { + var path = p.join(parent ?? sandbox, name); + var file = File(path).openWrite(); + try { + try { + await readAsBytes().listen(file.add).asFuture(); + } finally { + await file.close(); + } + } catch (_) { + await File(path).delete(); + rethrow; + } + } + + Future read() async => throw UnsupportedError( + "ArchiveDescriptor.read() is not supported. Use Archive.readAsBytes() " + "instead."); + + Stream> readAsBytes() => Stream.fromFuture(() async { + return _encodeFunction()(await archive); + }()); + + Future validate([String parent]) async { + // Access this first so we eaerly throw an error for a path with an invalid + // extension. + var decoder = _decodeFunction(); + + var fullPath = p.join(parent ?? sandbox, name); + var pretty = prettyPath(fullPath); + if (!(await File(fullPath).exists())) { + fail('File not found: "$pretty".'); + } + + var bytes = await File(fullPath).readAsBytes(); + Archive archive; + try { + archive = decoder(bytes); + } catch (_) { + // Catch every error to work around brendan-duncan/archive#77. + fail('File "$pretty" is not a valid archive.'); + } + + // Because validators expect to validate against a real filesystem, we have + // to extract the archive to a temp directory and run validation on that. + var tempDir = await Directory.systemTemp + .createTempSync('dart_test_') + .resolveSymbolicLinks(); + + try { + await waitAndReportErrors(archive.files.map((file) async { + var path = p.join(tempDir, file.name); + await Directory(p.dirname(path)).create(recursive: true); + await File(path).writeAsBytes(file.content as List); + })); + + await waitAndReportErrors(contents.map((entry) async { + try { + await entry.validate(tempDir); + } on TestFailure catch (error) { + // Replace the temporary directory with the path to the archive to + // make the error more user-friendly. + fail(error.message.replaceAll(tempDir, pretty)); + } + })); + } finally { + await Directory(tempDir).delete(recursive: true); + } + } + + /// Returns the function to use to encode this file to binary, based on its + /// [name]. + List Function(Archive) _encodeFunction() { + if (name.endsWith(".zip")) { + return ZipEncoder().encode; + } else if (name.endsWith(".tar")) { + return TarEncoder().encode; + } else if (name.endsWith(".tar.gz") || + name.endsWith(".tar.gzip") || + name.endsWith(".tgz")) { + return (archive) => GZipEncoder().encode(TarEncoder().encode(archive)); + } else if (name.endsWith(".tar.bz2") || name.endsWith(".tar.bzip2")) { + return (archive) => BZip2Encoder().encode(TarEncoder().encode(archive)); + } else { + throw UnsupportedError("Unknown file format $name."); + } + } + + /// Returns the function to use to decode this file from binary, based on its + /// [name]. + Archive Function(List) _decodeFunction() { + if (name.endsWith(".zip")) { + return ZipDecoder().decodeBytes; + } else if (name.endsWith(".tar")) { + return TarDecoder().decodeBytes; + } else if (name.endsWith(".tar.gz") || + name.endsWith(".tar.gzip") || + name.endsWith(".tgz")) { + return (archive) => + TarDecoder().decodeBytes(GZipDecoder().decodeBytes(archive)); + } else if (name.endsWith(".tar.bz2") || name.endsWith(".tar.bzip2")) { + return (archive) => + TarDecoder().decodeBytes(BZip2Decoder().decodeBytes(archive)); + } else { + throw UnsupportedError("Unknown file format $name."); + } + } + + String describe() => describeDirectory(name, contents); +} diff --git a/pkgs/test_descriptor/lib/src/directory_descriptor.dart b/pkgs/test_descriptor/lib/src/directory_descriptor.dart index d6ba56a81..c505885e4 100644 --- a/pkgs/test_descriptor/lib/src/directory_descriptor.dart +++ b/pkgs/test_descriptor/lib/src/directory_descriptor.dart @@ -7,7 +7,6 @@ import 'dart:io'; import 'package:async/async.dart'; import 'package:path/path.dart' as p; -import 'package:term_glyph/term_glyph.dart' as glyph; import 'package:test/test.dart'; import 'descriptor.dart'; @@ -122,23 +121,5 @@ class DirectoryDescriptor extends Descriptor { })); } - String describe() { - if (contents.isEmpty) return name; - - var buffer = StringBuffer(); - buffer.writeln(name); - for (var entry in contents.take(contents.length - 1)) { - var entryString = - prefixLines(entry.describe(), '${glyph.verticalLine} ', - first: '${glyph.teeRight}${glyph.horizontalLine}' - '${glyph.horizontalLine} '); - buffer.writeln(entryString); - } - - var lastEntryString = prefixLines(contents.last.describe(), ' ', - first: '${glyph.bottomLeftCorner}${glyph.horizontalLine}' - '${glyph.horizontalLine} '); - buffer.write(lastEntryString); - return buffer.toString(); - } + String describe() => describeDirectory(name, contents); } diff --git a/pkgs/test_descriptor/lib/src/utils.dart b/pkgs/test_descriptor/lib/src/utils.dart index 9a14b7000..5dad79479 100644 --- a/pkgs/test_descriptor/lib/src/utils.dart +++ b/pkgs/test_descriptor/lib/src/utils.dart @@ -9,6 +9,7 @@ import 'package:path/path.dart' as p; import 'package:term_glyph/term_glyph.dart' as glyph; import 'package:test/test.dart'; +import 'descriptor.dart'; import 'sandbox.dart'; /// A UTF-8 codec that allows malformed byte sequences. @@ -25,6 +26,27 @@ String addBullet(String text) => /// Converts [strings] to a bulleted list. String bullet(Iterable strings) => strings.map(addBullet).join("\n"); +/// Returns a human-readable description of a directory with the given [name] +/// and [contents]. +String describeDirectory(String name, List contents) { + if (contents.isEmpty) return name; + + var buffer = StringBuffer(); + buffer.writeln(name); + for (var entry in contents.take(contents.length - 1)) { + var entryString = prefixLines(entry.describe(), '${glyph.verticalLine} ', + first: '${glyph.teeRight}${glyph.horizontalLine}' + '${glyph.horizontalLine} '); + buffer.writeln(entryString); + } + + var lastEntryString = prefixLines(contents.last.describe(), ' ', + first: '${glyph.bottomLeftCorner}${glyph.horizontalLine}' + '${glyph.horizontalLine} '); + buffer.write(lastEntryString); + return buffer.toString(); +} + /// Prepends each line in [text] with [prefix]. /// /// If [first] or [last] is passed, the first and last lines, respectively, are @@ -67,7 +89,7 @@ bool matchesAll(Pattern pattern, String string) => /// Like [Future.wait] with `eagerError: true`, but reports errors after the /// first using [registerException] rather than silently ignoring them. -Future waitAndReportErrors(Iterable futures) { +Future> waitAndReportErrors(Iterable> futures) { var errored = false; return Future.wait(futures.map((future) { // Avoid async/await so that we synchronously add error handlers for the diff --git a/pkgs/test_descriptor/lib/test_descriptor.dart b/pkgs/test_descriptor/lib/test_descriptor.dart index cc3371425..ea576454a 100644 --- a/pkgs/test_descriptor/lib/test_descriptor.dart +++ b/pkgs/test_descriptor/lib/test_descriptor.dart @@ -5,6 +5,7 @@ import 'package:path/path.dart' as p; import 'package:test/test.dart'; +import 'src/archive_descriptor.dart'; import 'src/descriptor.dart'; import 'src/directory_descriptor.dart'; import 'src/file_descriptor.dart'; @@ -12,6 +13,7 @@ import 'src/nothing_descriptor.dart'; import 'src/pattern_descriptor.dart'; import 'src/sandbox.dart'; +export 'src/archive_descriptor.dart'; export 'src/descriptor.dart'; export 'src/directory_descriptor.dart'; export 'src/file_descriptor.dart'; @@ -72,5 +74,18 @@ PatternDescriptor filePattern(Pattern name, [contents]) => PatternDescriptor dirPattern(Pattern name, [Iterable contents]) => pattern(name, (realName) => dir(realName, contents)); +/// Creates a new [ArchiveDescriptor] with [name] and [contents]. +/// +/// [Descriptor.create] creates an archive with the given files and directories +/// within it, and [Descriptor.validate] validates that the archive contains the +/// given contents. It *doesn't* require that no other children exist. To ensure +/// that a particular child doesn't exist, use [nothing]. +/// +/// The type of the archive is determined by [name]'s file extension. It +/// supports `.zip`, `.tar`, `.tar.gz`/`.tar.gzip`/`.tgz`, and +/// `.tar.bz2`/`.tar.bzip2` files. +ArchiveDescriptor archive(String name, [Iterable contents]) => + ArchiveDescriptor(name, contents ?? []); + /// Returns [path] within the [sandbox] directory. String path(String path) => p.join(sandbox, path); diff --git a/pkgs/test_descriptor/pubspec.yaml b/pkgs/test_descriptor/pubspec.yaml index d39384e23..b50a18b91 100644 --- a/pkgs/test_descriptor/pubspec.yaml +++ b/pkgs/test_descriptor/pubspec.yaml @@ -1,5 +1,5 @@ name: test_descriptor -version: 1.1.1 +version: 1.2.0 description: An API for defining and verifying directory structures. author: Dart Team homepage: https://github.com/dart-lang/test_descriptor @@ -8,12 +8,14 @@ environment: sdk: '>=2.0.0 <3.0.0' dependencies: - async: '>=1.10.0 <3.0.0' + archive: '^2.0.0' + async: '>=1.13.0 <3.0.0' collection: '^1.5.0' matcher: '^0.12.0' + meta: '^1.1.7' path: '^1.0.0' stack_trace: '^1.0.0' - test: '>=0.12.19 <2.0.0' + test: '^1.6.0' term_glyph: '^1.0.0' dev_dependencies: diff --git a/pkgs/test_descriptor/test/archive_test.dart b/pkgs/test_descriptor/test/archive_test.dart new file mode 100644 index 000000000..a413fb96d --- /dev/null +++ b/pkgs/test_descriptor/test/archive_test.dart @@ -0,0 +1,281 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +@TestOn('vm') + +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:async/async.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; + +import 'utils.dart'; + +void main() { + group("create()", () { + test("creates an empty archive", () async { + await d.archive("test.tar").create(); + + var archive = + TarDecoder().decodeBytes(File(d.path("test.tar")).readAsBytesSync()); + expect(archive.files, isEmpty); + }); + + test("creates an archive with files", () async { + await d.archive("test.tar", [ + d.file("file1.txt", "contents 1"), + d.file("file2.txt", "contents 2") + ]).create(); + + var files = TarDecoder() + .decodeBytes(File(d.path("test.tar")).readAsBytesSync()) + .files; + expect(files.length, equals(2)); + _expectFile(files[0], "file1.txt", "contents 1"); + _expectFile(files[1], "file2.txt", "contents 2"); + }); + + test("creates an archive with files in a directory", () async { + await d.archive("test.tar", [ + d.dir("dir", [ + d.file("file1.txt", "contents 1"), + d.file("file2.txt", "contents 2") + ]) + ]).create(); + + var files = TarDecoder() + .decodeBytes(File(d.path("test.tar")).readAsBytesSync()) + .files; + expect(files.length, equals(2)); + _expectFile(files[0], "dir/file1.txt", "contents 1"); + _expectFile(files[1], "dir/file2.txt", "contents 2"); + }); + + test("creates an archive with files in a nested directory", () async { + await d.archive("test.tar", [ + d.dir("dir", [ + d.dir("subdir", [ + d.file("file1.txt", "contents 1"), + d.file("file2.txt", "contents 2") + ]) + ]) + ]).create(); + + var files = TarDecoder() + .decodeBytes(File(d.path("test.tar")).readAsBytesSync()) + .files; + expect(files.length, equals(2)); + _expectFile(files[0], "dir/subdir/file1.txt", "contents 1"); + _expectFile(files[1], "dir/subdir/file2.txt", "contents 2"); + }); + + group("creates a file in", () { + test("zip format", () async { + await d.archive("test.zip", [d.file("file.txt", "contents")]).create(); + + var archive = ZipDecoder() + .decodeBytes(File(d.path("test.zip")).readAsBytesSync()); + _expectFile(archive.files.single, "file.txt", "contents"); + }); + + group("gzip tar format", () { + for (var extension in [".tar.gz", ".tar.gzip", ".tgz"]) { + test("with $extension", () async { + await d.archive( + "test$extension", [d.file("file.txt", "contents")]).create(); + + var archive = TarDecoder().decodeBytes(GZipDecoder() + .decodeBytes(File(d.path("test$extension")).readAsBytesSync())); + _expectFile(archive.files.single, "file.txt", "contents"); + }); + } + }); + + group("bzip2 tar format", () { + for (var extension in [".tar.bz2", ".tar.bzip2"]) { + test("with $extension", () async { + await d.archive( + "test$extension", [d.file("file.txt", "contents")]).create(); + + var archive = TarDecoder().decodeBytes(BZip2Decoder() + .decodeBytes(File(d.path("test$extension")).readAsBytesSync())); + _expectFile(archive.files.single, "file.txt", "contents"); + }); + } + }); + }); + + group("gracefully rejects", () { + test("an uncreatable descriptor", () async { + await expectLater( + d.archive("test.tar", [d.filePattern(RegExp(r"^foo-"))]).create(), + throwsUnsupportedError); + await d.nothing("test.tar").validate(); + }); + + test("a non-file non-directory descriptor", () async { + await expectLater( + d.archive("test.tar", [d.nothing("file.txt")]).create(), + throwsUnsupportedError); + await d.nothing("test.tar").validate(); + }); + + test("an unknown file extension", () async { + await expectLater( + d.archive("test.asdf", [d.nothing("file.txt")]).create(), + throwsUnsupportedError); + }); + }); + }); + + group("validate()", () { + group("with an empty archive", () { + test("succeeds if an empty archive exists", () async { + File(d.path("test.tar")) + .writeAsBytesSync(TarEncoder().encode(Archive())); + await d.archive("test.tar").validate(); + }); + + test("succeeds if a non-empty archive exists", () async { + File(d.path("test.tar")).writeAsBytesSync( + TarEncoder().encode(Archive()..addFile(_file("file.txt")))); + await d.archive("test.tar").validate(); + }); + + test("fails if no archive exists", () { + expect(d.archive("test.tar").validate(), + throwsA(toString(startsWith('File not found: "test.tar".')))); + }); + + test("fails if an invalid archive exists", () { + d.file("test.tar", "not a valid tar file").create(); + expect( + d.archive("test.tar").validate(), + throwsA(toString( + startsWith('File "test.tar" is not a valid archive.')))); + }); + }); + + test("succeeds if an archive contains a matching file", () async { + File(d.path("test.tar")).writeAsBytesSync(TarEncoder() + .encode(Archive()..addFile(_file("file.txt", "contents")))); + await d.archive("test.tar", [d.file("file.txt", "contents")]).validate(); + }); + + test("fails if an archive doesn't contain a file", () async { + File(d.path("test.tar")).writeAsBytesSync(TarEncoder().encode(Archive())); + expect( + d.archive("test.tar", [d.file("file.txt", "contents")]).validate(), + throwsA( + toString(startsWith('File not found: "test.tar/file.txt".')))); + }); + + test("fails if an archive contains a non-matching file", () async { + File(d.path("test.tar")).writeAsBytesSync(TarEncoder() + .encode(Archive()..addFile(_file("file.txt", "wrong contents")))); + expect( + d.archive("test.tar", [d.file("file.txt", "contents")]).validate(), + throwsA(toString( + startsWith('File "test.tar/file.txt" should contain:')))); + }); + + test("succeeds if an archive contains a file matching a pattern", () async { + File(d.path("test.tar")).writeAsBytesSync(TarEncoder() + .encode(Archive()..addFile(_file("file.txt", "contents")))); + await d.archive("test.tar", + [d.filePattern(RegExp(r"f..e\.txt"), "contents")]).validate(); + }); + + group("validates a file in", () { + test("zip format", () async { + File(d.path("test.zip")).writeAsBytesSync(ZipEncoder() + .encode(Archive()..addFile(_file("file.txt", "contents")))); + + await d + .archive("test.zip", [d.file("file.txt", "contents")]).validate(); + }); + + group("gzip tar format", () { + for (var extension in [".tar.gz", ".tar.gzip", ".tgz"]) { + test("with $extension", () async { + File(d.path("test$extension")).writeAsBytesSync(GZipEncoder() + .encode(TarEncoder().encode( + Archive()..addFile(_file("file.txt", "contents"))))); + + await d.archive( + "test$extension", [d.file("file.txt", "contents")]).validate(); + }); + } + }); + + group("bzip2 tar format", () { + for (var extension in [".tar.bz2", ".tar.bzip2"]) { + test("with $extension", () async { + File(d.path("test$extension")).writeAsBytesSync(BZip2Encoder() + .encode(TarEncoder().encode( + Archive()..addFile(_file("file.txt", "contents"))))); + + await d.archive( + "test$extension", [d.file("file.txt", "contents")]).validate(); + }); + } + }); + }); + + test("gracefully rejects an unknown file format", () { + expect(d.archive("test.asdf").validate(), throwsUnsupportedError); + }); + }); + + test("read() is unsupported", () { + expect(d.archive("test.tar").read(), throwsUnsupportedError); + }); + + test("readAsBytes() returns the contents of the archive", () async { + var descriptor = d.archive("test.tar", + [d.file("file1.txt", "contents 1"), d.file("file2.txt", "contents 2")]); + + var files = TarDecoder() + .decodeBytes(await collectBytes(descriptor.readAsBytes())) + .files; + expect(files.length, equals(2)); + _expectFile(files[0], "file1.txt", "contents 1"); + _expectFile(files[1], "file2.txt", "contents 2"); + }); + + test("archive returns the in-memory contents", () async { + var archive = await d.archive("test.tar", [ + d.file("file1.txt", "contents 1"), + d.file("file2.txt", "contents 2") + ]).archive; + + var files = archive.files; + expect(files.length, equals(2)); + _expectFile(files[0], "file1.txt", "contents 1"); + _expectFile(files[1], "file2.txt", "contents 2"); + }); + + test("io refers to the file within the sandbox", () { + expect(d.file('test.tar').io.path, equals(p.join(d.sandbox, 'test.tar'))); + }); +} + +/// Asserts that [file] has the given [name] and [contents]. +void _expectFile(ArchiveFile file, String name, String contents) { + expect(file.name, equals(name)); + expect(utf8.decode(file.content as List), equals(contents)); +} + +/// Creates an [ArchiveFile] with the given [name] and [contents]. +ArchiveFile _file(String name, [String contents]) { + var bytes = utf8.encode(contents ?? ""); + return ArchiveFile(name, bytes.length, bytes) + // Setting the mode and mod time are necessary to work around + // brendan-duncan/archive#76. + ..mode = 428 + ..lastModTime = DateTime.now().millisecondsSinceEpoch ~/ 1000; +}