Skip to content

Commit

Permalink
Merge pull request #1664 from dart-lang/merge-test_reflective_loader-…
Browse files Browse the repository at this point in the history
…package

Merge `package:test_reflective_loader`
mosuem authored Dec 12, 2024

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
2 parents b78c509 + bea4a35 commit febccb9
Showing 13 changed files with 617 additions and 0 deletions.
5 changes: 5 additions & 0 deletions .github/ISSUE_TEMPLATE/test_reflective_loader.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
name: "package:test_reflective_loader"
about: "Create a bug or file a feature request against package:test_reflective_loader."
labels: "package:test_reflective_loader"
---
4 changes: 4 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
@@ -108,6 +108,10 @@
- changed-files:
- any-glob-to-any-file: 'pkgs/sse/**'

'package:test_reflective_loader':
- changed-files:
- any-glob-to-any-file: 'pkgs/test_reflective_loader/**'

'package:timing':
- changed-files:
- any-glob-to-any-file: 'pkgs/timing/**'
43 changes: 43 additions & 0 deletions .github/workflows/test_reflective_loader.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: package:test_reflective_loader

on:
# Run on PRs and pushes to the default branch.
push:
branches: [ main ]
paths:
- '.github/workflows/test_reflective_loader.yaml'
- 'pkgs/test_reflective_loader/**'
pull_request:
branches: [ main ]
paths:
- '.github/workflows/test_reflective_loader.yaml'
- 'pkgs/test_reflective_loader/**'
schedule:
- cron: "0 0 * * 0"

env:
PUB_ENVIRONMENT: bot.github

defaults:
run:
working-directory: pkgs/test_reflective_loader/

jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
sdk: [dev, 3.1]

steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
- uses: dart-lang/setup-dart@e630b99d28a3b71860378cafdc2a067c71107f94
with:
sdk: ${{ matrix.sdk }}

- run: dart pub get
- name: dart format
run: dart format --output=none --set-exit-if-changed .
- run: dart analyze --fatal-infos
- run: dart test
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@ don't naturally belong to other topic monorepos (like
| [source_maps](pkgs/source_maps/) | A library to programmatically manipulate source map files. | [![package issues](https://img.shields.io/badge/package:source_maps-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Asource_maps) | [![pub package](https://img.shields.io/pub/v/source_maps.svg)](https://pub.dev/packages/source_maps) |
| [source_span](pkgs/source_span/) | Provides a standard representation for source code locations and spans. | [![package issues](https://img.shields.io/badge/package:source_span-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Asource_span) | [![pub package](https://img.shields.io/pub/v/source_span.svg)](https://pub.dev/packages/source_span) |
| [sse](pkgs/sse/) | Provides client and server functionality for setting up bi-directional communication through Server Sent Events (SSE) and corresponding POST requests. | [![package issues](https://img.shields.io/badge/package:sse-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Asse) | [![pub package](https://img.shields.io/pub/v/sse.svg)](https://pub.dev/packages/sse) |
| [test_reflective_loader](pkgs/test_reflective_loader/) | Support for discovering tests and test suites using reflection. | [![package issues](https://img.shields.io/badge/package:test_reflective_loader-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atest_reflective_loader) | [![pub package](https://img.shields.io/pub/v/test_reflective_loader.svg)](https://pub.dev/packages/test_reflective_loader) |
| [timing](pkgs/timing/) | A simple package for tracking the performance of synchronous and asynchronous actions. | [![package issues](https://img.shields.io/badge/package:timing-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atiming) | [![pub package](https://img.shields.io/pub/v/timing.svg)](https://pub.dev/packages/timing) |
| [unified_analytics](pkgs/unified_analytics/) | A package for logging analytics for all Dart and Flutter related tooling to Google Analytics. | [![package issues](https://img.shields.io/badge/package:unified_analytics-4774bc)](https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Aunified_analytics) | [![pub package](https://img.shields.io/pub/v/unified_analytics.svg)](https://pub.dev/packages/unified_analytics) |

11 changes: 11 additions & 0 deletions pkgs/test_reflective_loader/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
.buildlog
.DS_Store
.idea
.dart_tool/
.pub/
.project
.settings/
build/
packages
.packages
pubspec.lock
6 changes: 6 additions & 0 deletions pkgs/test_reflective_loader/AUTHORS
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Below is a list of people and organizations that have contributed
# to the project. Names should be added to the list like so:
#
# Name/Organization <email address>

Google Inc.
72 changes: 72 additions & 0 deletions pkgs/test_reflective_loader/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
## 0.2.3

- Require Dart `^3.1.0`.
- Move to `dart-lang/tools` monorepo.

## 0.2.2

- Update to package:lints 2.0.0 and move it to a dev dependency.

## 0.2.1

- Use package:lints for analysis.
- Populate the pubspec `repository` field.

## 0.2.0

- Stable null safety release.

## 0.2.0-nullsafety.0

- Migrate to the null safety language feature.

## 0.1.9

- Add `@SkippedTest` annotation and `skip_test` prefix.

## 0.1.8

- Update `FailingTest` to add named parameters `issue` and `reason`.

## 0.1.7

- Update documentation comments.
- Remove `@MirrorsUsed` annotation on `dart:mirrors`.

## 0.1.6

- Make `FailingTest` public, with the URI of the issue that causes
the test to break.

## 0.1.5

- Set max SDK version to `<3.0.0`, and adjust other dependencies.

## 0.1.3

- Fix `@failingTest` to fail when the test passes.

## 0.1.2

- Update the pubspec `dependencies` section to include `package:test`

## 0.1.1

- For `@failingTest` tests, properly handle when the test fails by throwing an
exception in a timer task
- Analyze this package in strong mode

## 0.1.0

- Switched from 'package:unittest' to 'package:test'.
- Since 'package:test' does not define 'solo_test', in order to keep this
functionality, `defineReflectiveSuite` must be used to wrap all
`defineReflectiveTests` invocations.

## 0.0.4

- Added @failingTest, @assertFailingTest and @soloTest annotations.

## 0.0.1

- Initial version
27 changes: 27 additions & 0 deletions pkgs/test_reflective_loader/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
Copyright 2015, the Dart project authors.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
* Neither the name of Google LLC nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 changes: 28 additions & 0 deletions pkgs/test_reflective_loader/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[![Build Status](https://github.com/dart-lang/tools/actions/workflows/test_reflective_loader.yaml/badge.svg)](https://github.com/dart-lang/tools/actions/workflows/test_reflective_loader.yaml)
[![pub package](https://img.shields.io/pub/v/test_reflective_loader.svg)](https://pub.dev/packages/test_reflective_loader)
[![package publisher](https://img.shields.io/pub/publisher/test_reflective_loader.svg)](https://pub.dev/packages/test_reflective_loader/publisher)

Support for discovering tests and test suites using reflection.

This package follows the xUnit style where each class is a test suite, and each
method with the name prefix `test_` is a single test.

Methods with names starting with `test_` are run using the `test()` function with
the corresponding name. If the class defines methods `setUp()` or `tearDown()`,
they are executed before / after each test correspondingly, even if the test fails.

Methods with names starting with `solo_test_` are run using the `solo_test()` function.

Methods with names starting with `fail_` are expected to fail.

Methods with names starting with `solo_fail_` are run using the `solo_test()` function
and expected to fail.

Method returning `Future` class instances are asynchronous, so `tearDown()` is
executed after the returned `Future` completes.

## Features and bugs

Please file feature requests and bugs at the [issue tracker][tracker].

[tracker]: https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atest_reflective_loader
5 changes: 5 additions & 0 deletions pkgs/test_reflective_loader/analysis_options.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
include: package:dart_flutter_team_lints/analysis_options.yaml

linter:
rules:
- public_member_api_docs
354 changes: 354 additions & 0 deletions pkgs/test_reflective_loader/lib/test_reflective_loader.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,354 @@
// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:mirrors';

import 'package:test/test.dart' as test_package;

/// A marker annotation used to annotate test methods which are expected to fail
/// when asserts are enabled.
const Object assertFailingTest = _AssertFailingTest();

/// A marker annotation used to annotate test methods which are expected to
/// fail.
const Object failingTest = FailingTest();

/// A marker annotation used to instruct dart2js to keep reflection information
/// for the annotated classes.
const Object reflectiveTest = _ReflectiveTest();

/// A marker annotation used to annotate test methods that should be skipped.
const Object skippedTest = SkippedTest();

/// A marker annotation used to annotate "solo" groups and tests.
const Object soloTest = _SoloTest();

final List<_Group> _currentGroups = <_Group>[];
int _currentSuiteLevel = 0;
String _currentSuiteName = '';

/// Is `true` the application is running in the checked mode.
final bool _isCheckedMode = () {
try {
assert(false);
return false;
} catch (_) {
return true;
}
}();

/// Run the [define] function parameter that calls [defineReflectiveTests] to
/// add normal and "solo" tests, and also calls [defineReflectiveSuite] to
/// create embedded suites. If the current suite is the top-level one, perform
/// check for "solo" groups and tests, and run all or only "solo" items.
void defineReflectiveSuite(void Function() define, {String name = ''}) {
var groupName = _currentSuiteName;
_currentSuiteLevel++;
try {
_currentSuiteName = _combineNames(_currentSuiteName, name);
define();
} finally {
_currentSuiteName = groupName;
_currentSuiteLevel--;
}
_addTestsIfTopLevelSuite();
}

/// Runs test methods existing in the given [type].
///
/// If there is a "solo" test method in the top-level suite, only "solo" methods
/// are run.
///
/// If there is a "solo" test type, only its test methods are run.
///
/// Otherwise all tests methods of all test types are run.
///
/// Each method is run with a new instance of [type].
/// So, [type] should have a default constructor.
///
/// If [type] declares method `setUp`, it methods will be invoked before any
/// test method invocation.
///
/// If [type] declares method `tearDown`, it will be invoked after any test
/// method invocation. If method returns [Future] to test some asynchronous
/// behavior, then `tearDown` will be invoked in `Future.complete`.
void defineReflectiveTests(Type type) {
var classMirror = reflectClass(type);
if (!classMirror.metadata.any((InstanceMirror annotation) =>
annotation.type.reflectedType == _ReflectiveTest)) {
var name = MirrorSystem.getName(classMirror.qualifiedName);
throw Exception('Class $name must have annotation "@reflectiveTest" '
'in order to be run by runReflectiveTests.');
}

_Group group;
{
var isSolo = _hasAnnotationInstance(classMirror, soloTest);
var className = MirrorSystem.getName(classMirror.simpleName);
group = _Group(isSolo, _combineNames(_currentSuiteName, className));
_currentGroups.add(group);
}

classMirror.instanceMembers
.forEach((Symbol symbol, MethodMirror memberMirror) {
// we need only methods
if (!memberMirror.isRegularMethod) {
return;
}
// prepare information about the method
var memberName = MirrorSystem.getName(symbol);
var isSolo = memberName.startsWith('solo_') ||
_hasAnnotationInstance(memberMirror, soloTest);
// test_
if (memberName.startsWith('test_')) {
if (_hasSkippedTestAnnotation(memberMirror)) {
group.addSkippedTest(memberName);
} else {
group.addTest(isSolo, memberName, memberMirror, () {
if (_hasFailingTestAnnotation(memberMirror) ||
_isCheckedMode && _hasAssertFailingTestAnnotation(memberMirror)) {
return _runFailingTest(classMirror, symbol);
} else {
return _runTest(classMirror, symbol);
}
});
}
return;
}
// solo_test_
if (memberName.startsWith('solo_test_')) {
group.addTest(true, memberName, memberMirror, () {
return _runTest(classMirror, symbol);
});
}
// fail_test_
if (memberName.startsWith('fail_')) {
group.addTest(isSolo, memberName, memberMirror, () {
return _runFailingTest(classMirror, symbol);
});
}
// solo_fail_test_
if (memberName.startsWith('solo_fail_')) {
group.addTest(true, memberName, memberMirror, () {
return _runFailingTest(classMirror, symbol);
});
}
// skip_test_
if (memberName.startsWith('skip_test_')) {
group.addSkippedTest(memberName);
}
});

// Support for the case of missing enclosing [defineReflectiveSuite].
_addTestsIfTopLevelSuite();
}

/// If the current suite is the top-level one, add tests to the `test` package.
void _addTestsIfTopLevelSuite() {
if (_currentSuiteLevel == 0) {
void runTests({required bool allGroups, required bool allTests}) {
for (var group in _currentGroups) {
if (allGroups || group.isSolo) {
for (var test in group.tests) {
if (allTests || test.isSolo) {
test_package.test(test.name, test.function,
timeout: test.timeout, skip: test.isSkipped);
}
}
}
}
}

if (_currentGroups.any((g) => g.hasSoloTest)) {
runTests(allGroups: true, allTests: false);
} else if (_currentGroups.any((g) => g.isSolo)) {
runTests(allGroups: false, allTests: true);
} else {
runTests(allGroups: true, allTests: true);
}
_currentGroups.clear();
}
}

/// Return the combination of the [base] and [addition] names.
/// If any other two is `null`, then the other one is returned.
String _combineNames(String base, String addition) {
if (base.isEmpty) {
return addition;
} else if (addition.isEmpty) {
return base;
} else {
return '$base | $addition';
}
}

Object? _getAnnotationInstance(DeclarationMirror declaration, Type type) {
for (var annotation in declaration.metadata) {
if ((annotation.reflectee as Object).runtimeType == type) {
return annotation.reflectee;
}
}
return null;
}

bool _hasAnnotationInstance(DeclarationMirror declaration, Object instance) =>
declaration.metadata.any((InstanceMirror annotation) =>
identical(annotation.reflectee, instance));

bool _hasAssertFailingTestAnnotation(MethodMirror method) =>
_hasAnnotationInstance(method, assertFailingTest);

bool _hasFailingTestAnnotation(MethodMirror method) =>
_hasAnnotationInstance(method, failingTest);

bool _hasSkippedTestAnnotation(MethodMirror method) =>
_hasAnnotationInstance(method, skippedTest);

Future<Object?> _invokeSymbolIfExists(
InstanceMirror instanceMirror, Symbol symbol) {
Object? invocationResult;
InstanceMirror? closure;
try {
closure = instanceMirror.getField(symbol);
// ignore: avoid_catching_errors
} on NoSuchMethodError {
// ignore
}

if (closure is ClosureMirror) {
invocationResult = closure.apply([]).reflectee;
}
return Future.value(invocationResult);
}

/// Run a test that is expected to fail, and confirm that it fails.
///
/// This properly handles the following cases:
/// - The test fails by throwing an exception
/// - The test returns a future which completes with an error.
/// - An exception is thrown to the zone handler from a timer task.
Future<Object?>? _runFailingTest(ClassMirror classMirror, Symbol symbol) {
var passed = false;
return runZonedGuarded(() {
// ignore: void_checks
return Future.sync(() => _runTest(classMirror, symbol)).then<void>((_) {
passed = true;
test_package.fail('Test passed - expected to fail.');
}).catchError((Object e) {
// if passed, and we call fail(), rethrow this exception
if (passed) {
// ignore: only_throw_errors
throw e;
}
// otherwise, an exception is not a failure for _runFailingTest
});
}, (e, st) {
// if passed, and we call fail(), rethrow this exception
if (passed) {
// ignore: only_throw_errors
throw e;
}
// otherwise, an exception is not a failure for _runFailingTest
});
}

Future<void> _runTest(ClassMirror classMirror, Symbol symbol) async {
var instanceMirror = classMirror.newInstance(const Symbol(''), []);
try {
await _invokeSymbolIfExists(instanceMirror, #setUp);
await instanceMirror.invoke(symbol, []).reflectee;
} finally {
await _invokeSymbolIfExists(instanceMirror, #tearDown);
}
}

typedef _TestFunction = dynamic Function();

/// A marker annotation used to annotate test methods which are expected to
/// fail.
class FailingTest {
/// Initialize this annotation with the given arguments.
///
/// [issue] is a full URI describing the failure and used for tracking.
/// [reason] is a free form textual description.
const FailingTest({String? issue, String? reason});
}

/// A marker annotation used to annotate test methods which are skipped.
class SkippedTest {
/// Initialize this annotation with the given arguments.
///
/// [issue] is a full URI describing the failure and used for tracking.
/// [reason] is a free form textual description.
const SkippedTest({String? issue, String? reason});
}

/// A marker annotation used to annotate test methods with additional timeout
/// information.
class TestTimeout {
final test_package.Timeout _timeout;

/// Initialize this annotation with the given timeout.
const TestTimeout(test_package.Timeout timeout) : _timeout = timeout;
}

/// A marker annotation used to annotate test methods which are expected to fail
/// when asserts are enabled.
class _AssertFailingTest {
const _AssertFailingTest();
}

/// Information about a type based test group.
class _Group {
final bool isSolo;
final String name;
final List<_Test> tests = <_Test>[];

_Group(this.isSolo, this.name);

bool get hasSoloTest => tests.any((test) => test.isSolo);

void addSkippedTest(String name) {
var fullName = _combineNames(this.name, name);
tests.add(_Test.skipped(isSolo, fullName));
}

void addTest(bool isSolo, String name, MethodMirror memberMirror,
_TestFunction function) {
var fullName = _combineNames(this.name, name);
var timeout =
_getAnnotationInstance(memberMirror, TestTimeout) as TestTimeout?;
tests.add(_Test(isSolo, fullName, function, timeout?._timeout));
}
}

/// A marker annotation used to instruct dart2js to keep reflection information
/// for the annotated classes.
class _ReflectiveTest {
const _ReflectiveTest();
}

/// A marker annotation used to annotate "solo" groups and tests.
class _SoloTest {
const _SoloTest();
}

/// Information about a test.
class _Test {
final bool isSolo;
final String name;
final _TestFunction function;
final test_package.Timeout? timeout;

final bool isSkipped;

_Test(this.isSolo, this.name, this.function, this.timeout)
: isSkipped = false;

_Test.skipped(this.isSolo, this.name)
: isSkipped = true,
function = (() {}),
timeout = null;
}
13 changes: 13 additions & 0 deletions pkgs/test_reflective_loader/pubspec.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: test_reflective_loader
version: 0.2.3
description: Support for discovering tests and test suites using reflection.
repository: https://github.com/dart-lang/tools/tree/main/pkgs/test_reflective_loader

environment:
sdk: ^3.1.0

dependencies:
test: ^1.16.0

dev_dependencies:
dart_flutter_team_lints: ^2.0.0
48 changes: 48 additions & 0 deletions pkgs/test_reflective_loader/test/test_reflective_loader_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// ignore_for_file: non_constant_identifier_names

import 'dart:async';

import 'package:test/test.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';

void main() {
defineReflectiveSuite(() {
defineReflectiveTests(TestReflectiveLoaderTest);
});
}

@reflectiveTest
class TestReflectiveLoaderTest {
void test_passes() {
expect(true, true);
}

@failingTest
void test_fails() {
expect(false, true);
}

@failingTest
void test_fails_throws_sync() {
throw StateError('foo');
}

@failingTest
Future test_fails_throws_async() {
return Future.error('foo');
}

@skippedTest
void test_fails_but_skipped() {
throw StateError('foo');
}

@skippedTest
void test_times_out_but_skipped() {
while (true) {}
}
}

0 comments on commit febccb9

Please sign in to comment.