From 826c188c98c11ddb772795906db86a02ac1de5a9 Mon Sep 17 00:00:00 2001 From: evanweible-wf Date: Tue, 18 Aug 2015 10:51:14 -0500 Subject: [PATCH] #9 Task: Code Coverage --- .gitignore | 4 +- README.md | 19 + lib/src/dart_dev_cli.dart | 2 + lib/src/tasks/config.dart | 2 + lib/src/tasks/coverage/api.dart | 435 ++++++++++++++++++ lib/src/tasks/coverage/cli.dart | 81 ++++ lib/src/tasks/coverage/config.dart | 13 + lib/src/tasks/init/api.dart | 1 + lib/src/tasks/task.dart | 9 +- lib/src/util.dart | 21 + lib/util.dart | 1 + pubspec.yaml | 2 + .../browser/lib/coverage_browser.dart | 9 + test/fixtures/coverage/browser/pubspec.yaml | 7 + .../browser/test/browser_custom_test.dart | 14 + .../browser/test/browser_custom_test.html | 5 + .../coverage/browser/test/browser_test.dart | 11 + .../coverage/no_coverage_package/pubspec.yaml | 5 + .../fixtures/coverage/vm/lib/coverage_vm.dart | 9 + test/fixtures/coverage/vm/pubspec.yaml | 7 + test/fixtures/coverage/vm/test/vm_test.dart | 11 + test/integration/coverage_test.dart | 48 ++ tool/dev.dart | 1 + 23 files changed, 715 insertions(+), 2 deletions(-) create mode 100644 lib/src/tasks/coverage/api.dart create mode 100644 lib/src/tasks/coverage/cli.dart create mode 100644 lib/src/tasks/coverage/config.dart create mode 100644 test/fixtures/coverage/browser/lib/coverage_browser.dart create mode 100644 test/fixtures/coverage/browser/pubspec.yaml create mode 100644 test/fixtures/coverage/browser/test/browser_custom_test.dart create mode 100644 test/fixtures/coverage/browser/test/browser_custom_test.html create mode 100644 test/fixtures/coverage/browser/test/browser_test.dart create mode 100644 test/fixtures/coverage/no_coverage_package/pubspec.yaml create mode 100644 test/fixtures/coverage/vm/lib/coverage_vm.dart create mode 100644 test/fixtures/coverage/vm/pubspec.yaml create mode 100644 test/fixtures/coverage/vm/test/vm_test.dart create mode 100644 test/integration/coverage_test.dart diff --git a/.gitignore b/.gitignore index b03490d3..654e680f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .pub packages -pubspec.lock \ No newline at end of file +pubspec.lock + +coverage/ \ No newline at end of file diff --git a/README.md b/README.md index 0ae7780e..e6354d56 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ All Dart (https://dartlang.org) projects eventually share a common set of development requirements: - Tests (unit, integration, and functional) +- Code coverage - Consistent code formatting - Static analysis to detect issues - Examples for manual testing/exploration @@ -57,6 +58,7 @@ static analysis - you just need to know how to use the `dart_dev` tool. ## Supported Tasks - **Tests:** runs test suites (unit, integration, and functional) via the [`test` package test runner](https://github.com/dart-lang/test). +- **Coverage:** collects coverage over test suites (unit, integration, and functional) and generates a report. Uses the [`coverage` package](https://github.com/dart-lang/coverage). - **Code Formatting:** runs the [`dartfmt` tool from the `dart_style` package](https://github.com/dart-lang/dart_style) over source code. - **Static Analysis:** runs the [`dartanalyzer`](https://www.dartlang.org/tools/analyzer/) over source code. - **Serving Examples:** uses [`pub serve`](https://www.dartlang.org/tools/pub/cmd/pub-serve.html) to serve the project examples. @@ -91,6 +93,9 @@ main(args) async { // Define the directories where the LICENSE should be applied. config.copyLicense.directories = ['example/', 'lib/']; + + // Configure whether or not the HTML coverage report should be generated. + config.coverage.html = false; // Configure the port on which examples should be served. config.examples.port = 9000; @@ -120,6 +125,7 @@ see the help usage. Try it out by running any of the following tasks: # with the alias ddev analyze ddev copy-license +ddev coverage ddev examples ddev format ddev test @@ -127,6 +133,7 @@ ddev test # without the alias pub run dart_dev analyze pub run dart_dev copy-license +pub run dart_dev coverage pub run dart_dev examples pub run dart_dev format pub run dart_dev test @@ -146,6 +153,7 @@ main(args) async { // Available config objects: // config.analyze // config.copyLicense + // config.coverage // config.examples // config.format // config.init @@ -172,6 +180,15 @@ Name | Type | Default | Description `directories` | `List` | `['lib/']` | All source files in these directories will have the LICENSE header applied. `licensePath` | `String` | `LICENSE` | Path to the source LICENSE file that will be copied to all source files. +### `coverage` config +All configuration options for the `coverage` task are found on the `config.coverage` object. +However, the `coverage` task also uses the test suite configuration from the `config.test` object. + +Name | Type | Default | Description +---------- | -------------- | ----------- | ----------- +`html` | `bool` | `true` | Whether or not to generate the HTML report. +`output` | `String` | `coverage/` | Output directory for coverage artifacts. +`reportOn` | `List` | `['lib/']` | List of paths to include in the generated coverage report (LCOV and HTML). ### `examples` Config All configuration options for the `examples` task are found on the `config.examples` object. @@ -220,6 +237,7 @@ Supported tasks: analyze copy-license + coverage examples format init @@ -228,6 +246,7 @@ Supported tasks: - Static analysis: `ddev analyze` - Applying license to source files: `ddev copy-license` +- Code coverage: `ddev coverage` - Serving examples: `ddev examples` - Dart formatter: `ddev format` - Initialization: `ddev init` diff --git a/lib/src/dart_dev_cli.dart b/lib/src/dart_dev_cli.dart index db7a2efc..94257bf7 100644 --- a/lib/src/dart_dev_cli.dart +++ b/lib/src/dart_dev_cli.dart @@ -31,6 +31,7 @@ import 'package:dart_dev/src/tasks/config.dart'; import 'package:dart_dev/src/tasks/analyze/cli.dart'; import 'package:dart_dev/src/tasks/copy_license/cli.dart'; +import 'package:dart_dev/src/tasks/coverage/cli.dart'; import 'package:dart_dev/src/tasks/examples/cli.dart'; import 'package:dart_dev/src/tasks/format/cli.dart'; import 'package:dart_dev/src/tasks/init/cli.dart'; @@ -54,6 +55,7 @@ String _topLevelUsage = _parser.usage; dev(List args) async { registerTask(new AnalyzeCli(), config.analyze); registerTask(new CopyLicenseCli(), config.copyLicense); + registerTask(new CoverageCli(), config.coverage); registerTask(new ExamplesCli(), config.examples); registerTask(new FormatCli(), config.format); registerTask(new InitCli(), config.init); diff --git a/lib/src/tasks/config.dart b/lib/src/tasks/config.dart index 976e215f..51abacfa 100644 --- a/lib/src/tasks/config.dart +++ b/lib/src/tasks/config.dart @@ -16,6 +16,7 @@ library dart_dev.src.tasks.config; import 'package:dart_dev/src/tasks/analyze/config.dart'; import 'package:dart_dev/src/tasks/copy_license/config.dart'; +import 'package:dart_dev/src/tasks/coverage/config.dart'; import 'package:dart_dev/src/tasks/examples/config.dart'; import 'package:dart_dev/src/tasks/format/config.dart'; import 'package:dart_dev/src/tasks/init/config.dart'; @@ -26,6 +27,7 @@ Config config = new Config(); class Config { AnalyzeConfig analyze = new AnalyzeConfig(); CopyLicenseConfig copyLicense = new CopyLicenseConfig(); + CoverageConfig coverage = new CoverageConfig(); ExamplesConfig examples = new ExamplesConfig(); FormatConfig format = new FormatConfig(); InitConfig init = new InitConfig(); diff --git a/lib/src/tasks/coverage/api.dart b/lib/src/tasks/coverage/api.dart new file mode 100644 index 00000000..3e6129e2 --- /dev/null +++ b/lib/src/tasks/coverage/api.dart @@ -0,0 +1,435 @@ +library dart_dev.src.tasks.coverage.api; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:dart_dev/util.dart' show Reporter, TaskProcess, getOpenPort; +import 'package:path/path.dart' as path; + +import 'package:dart_dev/src/tasks/coverage/config.dart'; +import 'package:dart_dev/src/tasks/task.dart'; + +const String _testFilePattern = '_test.dart'; + +class CoverageResult extends TaskResult { + final File collection; + final Directory report; + final File reportIndex; + final File lcov; + final Iterable tests; + + CoverageResult.fail( + Iterable this.tests, File this.collection, File this.lcov, + {Directory report}) + : super.fail(), + this.report = report, + reportIndex = + report != null ? new File('${report.path}/index.html') : null; + + CoverageResult.success( + Iterable this.tests, File this.collection, File this.lcov, + {Directory report}) + : super.success(), + this.report = report, + reportIndex = + report != null ? new File('${report.path}/index.html') : null; +} + +class CoverageTask extends Task { + /// Collect and format coverage for the given suite of [tests]. The result of + /// the coverage task will be returned once it has completed. + /// + /// Each file path in [tests] will be run as a test. Each directory path in + /// [tests] will be searched (recursively) for all files ending in + /// "_test.dart" and all matching files will be run as tests. + /// + /// If [html] is true, `genhtml` will be used to generate an HTML report of + /// the collected coverage and the report will be opened. + static Future run(List tests, + {bool html: defaultHtml, + String output: defaultOutput, + List reportOn: defaultReportOn}) async { + CoverageTask coverage = + new CoverageTask._(tests, html: html, output: output); + await coverage._collect(); + await coverage._format(reportOn); + + if (html) { + await coverage._generateReport(); + } + return new CoverageResult.success( + coverage.tests, coverage.collection, coverage.lcov, + report: coverage.report); + } + + /// Collect and format coverage for the given suite of [tests]. The + /// [CoverageTask] instance will be returned as soon as it is started. Output + /// from the sub tasks will be available in stream format so that immediate + /// progress can be monitored. The result of the coverage task will be + /// available from the `done` Future on the task. + /// + /// Each file path in [tests] will be run as a test. Each directory path in + /// [tests] will be searched (recursively) for all files ending in + /// "_test.dart" and all matching files will be run as tests. + /// + /// If [html] is true, `genhtml` will be used to generate an HTML report of + /// the collected coverage and the report will be opened. + static CoverageTask start(List tests, + {bool html: defaultHtml, + String output: defaultOutput, + List reportOn: defaultReportOn}) { + CoverageTask coverage = + new CoverageTask._(tests, html: html, output: output); + + // Execute the coverage collection and formatting, but don't wait on it. + () async { + await coverage._collect(); + await coverage._format(reportOn); + + if (html) { + await coverage._generateReport(); + } + CoverageResult result = new CoverageResult.success( + coverage.tests, coverage.collection, coverage.lcov, + report: coverage.report); + coverage._done.complete(result); + }(); + + return coverage; + } + + /// JSON formatted coverage. Output from the coverage package. + File _collection; + + /// Combination of the underlying process stdouts. + StreamController _coverageOutput = new StreamController(); + + /// Combination of the underlying process stderrs. + StreamController _coverageErrorOutput = new StreamController(); + + /// Completes when collection, formatting, and report generation is finished. + Completer _done = new Completer(); + + /// LCOV formatted coverage. + File _lcov; + + /// List of test files to run and collect coverage from. This list is + /// generated from the given list of test paths by adding all files and + /// searching all directories for valid test files. + List _files = []; + + /// File created to run the test in a browser. Need to store it so it can be + /// cleaned up after the test finishes. + File _lastHtmlFile; + + /// Process used to run the tests. Need to store it so it can be killed after + /// the coverage collection has completed. + TaskProcess _lastTestProcess; + + /// Directory to output all coverage related artifacts. + Directory _outputDirectory; + + CoverageTask._(List tests, + {bool html: defaultHtml, String output: defaultOutput}) + : _outputDirectory = new Directory(output) { + // Build the list of test files. + tests.forEach((path) { + if (path.endsWith(_testFilePattern) && + FileSystemEntity.isFileSync(path)) { + _files.add(new File(path)); + } else if (FileSystemEntity.isDirectorySync(path)) { + Directory dir = new Directory(path); + List children = dir.listSync(recursive: true); + Iterable validTests = + children.where((FileSystemEntity e) { + Uri uri = Uri.parse(e.absolute.path); + return ( + // Is a file, not a directory. + e is File && + // Is not a package dependency file. + !(Uri.parse(e.path).pathSegments.contains('packages')) && + // Is a valid test file. + e.path.endsWith(_testFilePattern)); + }); + _files.addAll(validTests); + } + }); + } + + /// Generated file with the coverage collection information in JSON format. + File get collection => _collection; + + /// Completes when the coverage collection, formatting, and optional report + /// generation has finished. Completes with a [CoverageResult] instance. + Future get done => _done.future; + + /// Combination of the underlying process stderrs, including individual test + /// runs and the collection of coverage from each, the formatting of the + /// complete coverage data set, and the generation of an HTML report if + /// applicable. Each item in the stream is a line. + Stream get errorOutput => _coverageErrorOutput.stream; + + /// Generated file with the coverage collection information in LCOV format. + File get lcov => _lcov; + + /// Combination of the underlying process stdouts, including individual test + /// runs and the collection of coverage from each, the formatting of the + /// complete coverage data set, and the generation of an HTML report if + /// applicable. Each item in the stream is a line. + Stream get output => _coverageOutput.stream; + + /// Directory containing the generated coverage report. + Directory get report => _outputDirectory; + + /// All test files (expanded from the given list of test paths). + /// This is the exact list of tests that were run for coverage collection. + Iterable get tests => _files.map((f) => f.path); + + Future _collect() async { + List collections = []; + for (int i = 0; i < _files.length; i++) { + File collection = new File(path.join( + '${_outputDirectory.path}/collection', '${_files[i].path}.json')); + int observatoryPort; + + // Run the test and obtain the observatory port for coverage collection. + try { + observatoryPort = await _test(_files[i]); + } on TestException { + _coverageErrorOutput.add('Tests failed: ${_files[i].path}'); + continue; + } + + // Collect the coverage from observatory. + String executable = 'pub'; + List args = [ + 'run', + 'coverage:collect_coverage', + '--port=${observatoryPort}', + '-o', + collection.path + ]; + + _coverageOutput.add(''); + _coverageOutput.add('Collecting coverage for ${_files[i].path}'); + _coverageOutput.add('$executable ${args.join(' ')}\n'); + + TaskProcess process = new TaskProcess(executable, args); + process.stdout.listen((l) => _coverageOutput.add(' $l')); + process.stderr.listen((l) => _coverageErrorOutput.add(' $l')); + await process.done; + _killTest(); + if (await process.exitCode > 0) continue; + collections.add(collection); + } + + // Merge all individual coverage collection files into one. + _collection = _merge(collections); + } + + Future _format(List reportOn) async { + _lcov = new File('${_outputDirectory.path}/coverage.lcov'); + _lcov.createSync(); + + String executable = 'pub'; + List args = [ + 'run', + 'coverage:format_coverage', + '-l', + '--package-root=packages', + '-i', + collection.path, + '-o', + lcov.path, + '--verbose' + ]; + args.addAll(reportOn.map((p) => '--report-on=$p')); + + _coverageOutput.add(''); + _coverageOutput.add('Formatting coverage'); + _coverageOutput.add('$executable ${args.join(' ')}\n'); + + TaskProcess process = new TaskProcess(executable, args); + process.stdout.listen((l) => _coverageOutput.add(' $l')); + process.stderr.listen((l) => _coverageErrorOutput.add(' $l')); + await process.done; + } + + Future _generateReport() async { + String executable = 'genhtml'; + List args = ['-o', _outputDirectory.path, lcov.path]; + + _coverageOutput.add(''); + _coverageOutput.add('Generating HTML report...'); + _coverageOutput.add('$executable ${args.join(' ')}\n'); + + TaskProcess process = new TaskProcess(executable, args); + process.stdout.listen((l) => _coverageOutput.add(' $l')); + process.stderr.listen((l) => _coverageErrorOutput.add(' $l')); + await process.done; + } + + void _killTest() { + _lastTestProcess.kill(); + _lastTestProcess = null; + if (_lastHtmlFile != null) { + _lastHtmlFile.deleteSync(); + } + } + + File _merge(List collections) { + if (collections.isEmpty) throw new ArgumentError( + 'Cannot merge an empty list of coverages.'); + + Map mergedJson = JSON.decode(collections.first.readAsStringSync()); + for (int i = 1; i < collections.length; i++) { + Map coverageJson = JSON.decode(collections[i].readAsStringSync()); + mergedJson['coverage'].addAll(coverageJson['coverage']); + collections[i].deleteSync(); + } + + File coverage = new File('${_outputDirectory.path}/coverage.json'); + if (coverage.existsSync()) { + coverage.deleteSync(); + } + coverage.createSync(); + coverage.writeAsStringSync(JSON.encode(mergedJson)); + return coverage; + } + + Future _test(File file) async { + // Look for a correlating HTML file. + String htmlPath = file.absolute.path; + htmlPath = htmlPath.substring(0, htmlPath.length - '.dart'.length); + htmlPath = '$htmlPath.html'; + File customHtmlFile = new File(htmlPath); + + // Build or modify the HTML file to properly load the test. + File htmlFile; + if (customHtmlFile.existsSync()) { + // A custom HTML file exists, but is designed for the test package's + // test runner. A slightly modified version of that file is needed. + htmlFile = _lastHtmlFile = new File('${customHtmlFile.path}.temp.html'); + file.createSync(); + String contents = customHtmlFile.readAsStringSync(); + String testFile = file.uri.pathSegments.last; + var linkP1 = + new RegExp(r''; + contents = contents.replaceFirst(dartJsScript, testScript); + htmlFile.writeAsStringSync(contents); + } else { + // Create an HTML file that simply loads the test file. + htmlFile = _lastHtmlFile = new File('${file.path}.temp.html'); + htmlFile.createSync(); + String testFile = file.uri.pathSegments.last; + htmlFile.writeAsStringSync( + ''); + } + + // Determine if this is a VM test or a browser test. + bool isBrowserTest; + if (customHtmlFile.existsSync()) { + isBrowserTest = true; + } else { + // Run analysis on file in "Server" category and look for "Library not + // found" errors, which indicates a `dart:html` import. + ProcessResult pr = await Process.run( + 'dart2js', + [ + '--analyze-only', + '--categories=Server', + '--package-root=packages', + file.path + ], + runInShell: true); + // TODO: When dart2js has fixed the issue with their exitcode we should + // rely on the exitcode instead of the stdout. + isBrowserTest = pr.stdout != null && + (pr.stdout as String).contains('Error: Library not found'); + } + + String _observatoryFailPattern = 'Could not start Observatory HTTP server'; + RegExp _observatoryPortPattern = new RegExp( + r'Observatory listening (at|on) http:\/\/127\.0\.0\.1:(\d+)'); + + String _testsFailedPattern = 'Some tests failed.'; + String _testsPassedPattern = 'All tests passed!'; + + if (isBrowserTest) { + // Run the test in content-shell. + String executable = 'content_shell'; + List args = [htmlFile.path]; + _coverageOutput.add(''); + _coverageOutput.add('Running test suite ${file.path}'); + _coverageOutput.add('$executable ${args.join(' ')}\n'); + TaskProcess process = + _lastTestProcess = new TaskProcess('content_shell', args); + process.stdout.listen((l) => _coverageOutput.add(' $l')); + + int observatoryPort; + // Note: content-shell dumps render tree to stderr. + await for (String line in process.stderr) { + _coverageOutput.add(' $line'); + if (line.contains(_observatoryFailPattern)) { + throw new TestException(); + } + if (line.contains(_observatoryPortPattern)) { + Match m = _observatoryPortPattern.firstMatch(line); + observatoryPort = int.parse(m.group(2)); + } + if (line.contains(_testsFailedPattern)) { + throw new TestException(); + } + if (line.contains(_testsPassedPattern)) { + break; + } + } + + return observatoryPort; + } else { + // Find an open port to observe the Dart VM on. + int port = await getOpenPort(); + + // Run the test on the Dart VM. + String executable = 'dart'; + List args = ['--observe=$port', file.path]; + _coverageOutput.add(''); + _coverageOutput.add('Running test suite ${file.path}'); + _coverageOutput.add('$executable ${args.join(' ')}\n'); + TaskProcess process = + _lastTestProcess = new TaskProcess(executable, args); + process.stderr.listen((l) => _coverageErrorOutput.add(' $l')); + + await for (String line in process.stdout) { + _coverageOutput.add(' $line'); + if (line.contains(_observatoryFailPattern)) { + throw new TestException(); + } + if (line.contains(_testsFailedPattern)) { + throw new TestException(); + } + if (line.contains(_testsPassedPattern)) { + break; + } + } + + return port; + } + } +} + +class TestException implements Exception {} diff --git a/lib/src/tasks/coverage/cli.dart b/lib/src/tasks/coverage/cli.dart new file mode 100644 index 00000000..9b0c413b --- /dev/null +++ b/lib/src/tasks/coverage/cli.dart @@ -0,0 +1,81 @@ +library dart_dev.src.tasks.coverage.cli; + +import 'dart:async'; +import 'dart:io'; + +import 'package:args/args.dart'; + +import 'package:dart_dev/util.dart' show hasImmediateDependency, reporter; + +import 'package:dart_dev/src/tasks/cli.dart'; +import 'package:dart_dev/src/tasks/config.dart'; +import 'package:dart_dev/src/tasks/coverage/api.dart'; +import 'package:dart_dev/src/tasks/coverage/config.dart'; +import 'package:dart_dev/src/tasks/test/config.dart'; + +class CoverageCli extends TaskCli { + final ArgParser argParser = new ArgParser() + ..addFlag('unit', + defaultsTo: defaultUnit, help: 'Includes the unit test suite.') + ..addFlag('integration', + defaultsTo: defaultIntegration, + help: 'Includes the integration test suite.') + ..addFlag('html', + negatable: true, + defaultsTo: defaultHtml, + help: 'Generate and open an HTML report.') + ..addFlag('open', + negatable: true, + defaultsTo: true, + help: 'Open the HTML report automatically.'); + + final String command = 'coverage'; + + Future run(ArgResults parsedArgs) async { + if (!hasImmediateDependency('coverage')) return new CliResult.fail( + 'Package "coverage" must be an immediate dependency in order to run its executables.'); + + bool unit = parsedArgs['unit']; + bool integration = parsedArgs['integration']; + + if (!unit && !integration) { + return new CliResult.fail( + 'No tests were selected. Include at least one of --unit or --integration.'); + } + + bool html = TaskCli.valueOf('html', parsedArgs, config.coverage.html); + bool open = TaskCli.valueOf('open', parsedArgs, true); + + List tests = []; + if (unit) { + tests.addAll(config.test.unitTests); + } + if (integration) { + tests.addAll(config.test.integrationTests); + } + if (tests.isEmpty) { + if (unit && config.test.unitTests.isEmpty) { + return new CliResult.fail( + 'This project does not specify any unit tests.'); + } + if (integration && config.test.integrationTests.isEmpty) { + return new CliResult.fail( + 'This project does not specify any integration tests.'); + } + } + + CoverageTask task = CoverageTask.start(tests, + html: html, + output: config.coverage.output, + reportOn: config.coverage.reportOn); + reporter.logGroup('Collecting coverage', + outputStream: task.output, errorStream: task.errorOutput); + CoverageResult result = await task.done; + if (result.successful && open) { + Process.run('open', [result.reportIndex.path]); + } + return result.successful + ? new CliResult.success('Coverage collected.') + : new CliResult.fail('Coverage failed.'); + } +} diff --git a/lib/src/tasks/coverage/config.dart b/lib/src/tasks/coverage/config.dart new file mode 100644 index 00000000..37230a7d --- /dev/null +++ b/lib/src/tasks/coverage/config.dart @@ -0,0 +1,13 @@ +library dart_dev.src.tasks.coverage.config; + +import 'package:dart_dev/src/tasks/config.dart'; + +const bool defaultHtml = true; +const String defaultOutput = 'coverage/'; +const List defaultReportOn = const ['lib/']; + +class CoverageConfig extends TaskConfig { + bool html = defaultHtml; + String output = defaultOutput; + List reportOn = defaultReportOn; +} diff --git a/lib/src/tasks/init/api.dart b/lib/src/tasks/init/api.dart index 3bd9b4e5..0d33dd24 100644 --- a/lib/src/tasks/init/api.dart +++ b/lib/src/tasks/init/api.dart @@ -31,6 +31,7 @@ main(List args) async { // Available task configurations: // config.analyze // config.copyLicense + // config.coverage // config.examples // config.format // config.test diff --git a/lib/src/tasks/task.dart b/lib/src/tasks/task.dart index 7b39a40c..7c14a258 100644 --- a/lib/src/tasks/task.dart +++ b/lib/src/tasks/task.dart @@ -17,6 +17,13 @@ library dart_dev.src.tasks.task; import 'dart:async'; abstract class Task { - Future get done; bool successful; + Future get done; +} + +abstract class TaskResult { + bool _successful; + TaskResult.fail() : _successful = false; + TaskResult.success() : _successful = true; + bool get successful => _successful; } diff --git a/lib/src/util.dart b/lib/src/util.dart index 3d2e6bfc..a91336a0 100644 --- a/lib/src/util.dart +++ b/lib/src/util.dart @@ -14,10 +14,31 @@ library dart_dev.src.util; +import 'dart:async'; import 'dart:io'; import 'package:yaml/yaml.dart'; +/// Returns an open port by creating a temporary Socket. +/// Borrowed from coverage package https://github.com/dart-lang/coverage/blob/master/lib/src/util.dart#L49-L66 +Future getOpenPort() async { + ServerSocket socket; + + try { + socket = await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V4, 0); + } catch (_) { + // try again v/ V6 only. Slight possibility that V4 is disabled + socket = await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V6, 0, + v6Only: true); + } + + try { + return socket.port; + } finally { + await socket.close(); + } +} + bool hasImmediateDependency(String packageName) { File pubspec = new File('pubspec.yaml'); Map pubspecYaml = loadYaml(pubspec.readAsStringSync()); diff --git a/lib/util.dart b/lib/util.dart index 6953269c..aad98634 100644 --- a/lib/util.dart +++ b/lib/util.dart @@ -18,6 +18,7 @@ export 'package:dart_dev/src/reporter.dart' show Reporter, reporter; export 'package:dart_dev/src/task_process.dart' show TaskProcess; export 'package:dart_dev/src/util.dart' show + getOpenPort, hasImmediateDependency, parseArgsFromCommand, parseExecutableFromCommand; diff --git a/pubspec.yaml b/pubspec.yaml index 9b1424ab..7258c6a8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,7 +3,9 @@ version: 0.0.0 dependencies: ansicolor: "^0.0.9" args: "^0.13.0" + coverage: "^0.7.2" dart_style: "^0.2.0" + path: "^1.3.6" test: "^0.12.0" yaml: "^2.1.0" executables: diff --git a/test/fixtures/coverage/browser/lib/coverage_browser.dart b/test/fixtures/coverage/browser/lib/coverage_browser.dart new file mode 100644 index 00000000..0db3bb17 --- /dev/null +++ b/test/fixtures/coverage/browser/lib/coverage_browser.dart @@ -0,0 +1,9 @@ +library coverage_browser; + +import 'dart:html'; + +void notCovered() { + print('nope'); +} + +bool works() => document is Document; \ No newline at end of file diff --git a/test/fixtures/coverage/browser/pubspec.yaml b/test/fixtures/coverage/browser/pubspec.yaml new file mode 100644 index 00000000..7266965b --- /dev/null +++ b/test/fixtures/coverage/browser/pubspec.yaml @@ -0,0 +1,7 @@ +name: coverage_browser +version: 0.0.0 +dev_dependencies: + coverage: "^0.7.2" + dart_dev: + path: ../../../.. + test: "^0.12.0" \ No newline at end of file diff --git a/test/fixtures/coverage/browser/test/browser_custom_test.dart b/test/fixtures/coverage/browser/test/browser_custom_test.dart new file mode 100644 index 00000000..9be9809c --- /dev/null +++ b/test/fixtures/coverage/browser/test/browser_custom_test.dart @@ -0,0 +1,14 @@ +@TestOn('browser') +library coverage.browser.test.browser_custom_test; + +import 'dart:js' show context; + +import 'package:coverage_browser/coverage_browser.dart' as lib; +import 'package:test/test.dart'; + +main() { + test('browser test', () { + expect(lib.works(), isTrue); + expect(context['customScript'], isTrue); + }); +} \ No newline at end of file diff --git a/test/fixtures/coverage/browser/test/browser_custom_test.html b/test/fixtures/coverage/browser/test/browser_custom_test.html new file mode 100644 index 00000000..0c0a5718 --- /dev/null +++ b/test/fixtures/coverage/browser/test/browser_custom_test.html @@ -0,0 +1,5 @@ + + + \ No newline at end of file diff --git a/test/fixtures/coverage/browser/test/browser_test.dart b/test/fixtures/coverage/browser/test/browser_test.dart new file mode 100644 index 00000000..7ba91af9 --- /dev/null +++ b/test/fixtures/coverage/browser/test/browser_test.dart @@ -0,0 +1,11 @@ +@TestOn('browser') +library coverage.browser.test.browser_test; + +import 'package:coverage_browser/coverage_browser.dart' as lib; +import 'package:test/test.dart'; + +main() { + test('browser test', () { + expect(lib.works(), isTrue); + }); +} \ No newline at end of file diff --git a/test/fixtures/coverage/no_coverage_package/pubspec.yaml b/test/fixtures/coverage/no_coverage_package/pubspec.yaml new file mode 100644 index 00000000..caba1411 --- /dev/null +++ b/test/fixtures/coverage/no_coverage_package/pubspec.yaml @@ -0,0 +1,5 @@ +name: coverage_no_coverage_package +version: 0.0.0 +dev_dependencies: + dart_dev: + path: ../../../.. \ No newline at end of file diff --git a/test/fixtures/coverage/vm/lib/coverage_vm.dart b/test/fixtures/coverage/vm/lib/coverage_vm.dart new file mode 100644 index 00000000..30df5ca2 --- /dev/null +++ b/test/fixtures/coverage/vm/lib/coverage_vm.dart @@ -0,0 +1,9 @@ +library coverage_vm; + +import 'dart:io'; + +void notCovered() { + print('nope'); +} + +bool works() => stdout is IOSink; \ No newline at end of file diff --git a/test/fixtures/coverage/vm/pubspec.yaml b/test/fixtures/coverage/vm/pubspec.yaml new file mode 100644 index 00000000..ba900b44 --- /dev/null +++ b/test/fixtures/coverage/vm/pubspec.yaml @@ -0,0 +1,7 @@ +name: coverage_vm +version: 0.0.0 +dev_dependencies: + coverage: "^0.7.2" + dart_dev: + path: ../../../.. + test: "^0.12.0" \ No newline at end of file diff --git a/test/fixtures/coverage/vm/test/vm_test.dart b/test/fixtures/coverage/vm/test/vm_test.dart new file mode 100644 index 00000000..5d88726d --- /dev/null +++ b/test/fixtures/coverage/vm/test/vm_test.dart @@ -0,0 +1,11 @@ +@TestOn('vm') +library coverage.vm.test.vm_test; + +import 'package:coverage_vm/coverage_vm.dart' as lib; +import 'package:test/test.dart'; + +main() { + test('browser test', () { + expect(lib.works(), isTrue); + }); +} \ No newline at end of file diff --git a/test/integration/coverage_test.dart b/test/integration/coverage_test.dart new file mode 100644 index 00000000..ecf3628a --- /dev/null +++ b/test/integration/coverage_test.dart @@ -0,0 +1,48 @@ +@TestOn('vm') +library dart_dev.test.integration.coverage_test; + +import 'dart:async'; +import 'dart:io'; + +import 'package:dart_dev/util.dart' show TaskProcess; +import 'package:test/test.dart'; + +const String projectWithVmTests = 'test/fixtures/coverage/browser'; +const String projectWithBrowserTests = 'test/fixtures/coverage/vm'; +const String projectWithoutCoveragePackage = + 'test/fixtures/coverage/no_coverage_package'; + +Future runCoverage(String projectPath) async { + await Process.run('pub', ['get'], workingDirectory: projectPath); + Directory oldCoverage = new Directory('$projectPath/coverage'); + if (oldCoverage.existsSync()) { + oldCoverage.deleteSync(recursive: true); + } + + List args = ['run', 'dart_dev', 'coverage', '--no-open']; + TaskProcess process = + new TaskProcess('pub', args, workingDirectory: projectPath); + + await process.done; + return (await process.exitCode) == 0; +} + +void main() { + group('Coverage Task', () { + test('should generate coverage for Browser tests', () async { + expect(await runCoverage(projectWithBrowserTests), isTrue); + File lcov = new File('$projectWithBrowserTests/coverage/coverage.lcov'); + expect(lcov.existsSync(), isTrue); + }, timeout: new Timeout(new Duration(seconds: 60))); + + test('should generate coverage for VM tests', () async { + expect(await runCoverage(projectWithVmTests), isTrue); + File lcov = new File('$projectWithVmTests/coverage/coverage.lcov'); + expect(lcov.existsSync(), isTrue); + }, timeout: new Timeout(new Duration(seconds: 60))); + + test('should warn if "coverage" package is missing', () async { + expect(await runCoverage(projectWithoutCoveragePackage), isFalse); + }); + }); +} diff --git a/tool/dev.dart b/tool/dev.dart index ba4e0e43..8f9eacba 100644 --- a/tool/dev.dart +++ b/tool/dev.dart @@ -20,6 +20,7 @@ main(args) async { var directories = ['bin/', 'lib/', 'test/integration/', 'tool/']; config.analyze.entryPoints = directories; config.copyLicense.directories = directories; + config.coverage.reportOn = ['bin/', 'lib/']; config.format.directories = directories; config.test ..unitTests = []