diff --git a/.github/ISSUE_TEMPLATE/canary.md b/.github/ISSUE_TEMPLATE/canary.md new file mode 100644 index 00000000..f0262464 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/canary.md @@ -0,0 +1,5 @@ +--- +name: "package:canary" +about: "Create a bug or file a feature request against package:canary." +labels: "package:canary" +--- diff --git a/.github/test_repos/repos.json b/.github/test_repos/repos.json index 616587ff..bdbbb3f1 100644 --- a/.github/test_repos/repos.json +++ b/.github/test_repos/repos.json @@ -1,5 +1,5 @@ { - "$schema": "../../pkgs/quest/schema.json", + "$schema": "../../pkgs/canary/schema.json", "https://github.com/mosuem/my_app_old_web": { "level": "analyze" }, diff --git a/.github/workflows/ecosystem_test.yaml b/.github/workflows/canary.yaml similarity index 82% rename from .github/workflows/ecosystem_test.yaml rename to .github/workflows/canary.yaml index 1574ab57..22847b94 100644 --- a/.github/workflows/ecosystem_test.yaml +++ b/.github/workflows/canary.yaml @@ -1,4 +1,4 @@ -name: Ecosystem test +name: Canary on: workflow_call: @@ -14,7 +14,7 @@ on: required: false jobs: - update_and_test: + test: runs-on: ubuntu-latest steps: - name: Checkout repository @@ -28,16 +28,16 @@ jobs: - run: echo "${{ toJSON(github.event.pull_request.labels.*.name) }}" - - name: Install firehose - run: dart pub global activate -s path pkgs/quest + - name: Install local version of `package:canary` + run: dart pub global activate -s path pkgs/canary if: ${{ inputs.local_debug }} - - run: dart pub global activate -s git https://github.com/dart-lang/ecosystem.git --git-ref main --git-path pkgs/quest + - run: dart pub global activate -s git https://github.com/dart-lang/ecosystem.git --git-ref main --git-path pkgs/canary if: ${{ !inputs.local_debug }} - name: Update package and test run: | - dart pub global run quest ${{ inputs.repos_file }} ${{ github.repositoryUrl }} ${{ github.head_ref || github.ref_name }} "${{ toJSON(github.event.pull_request.labels.*.name) }}" + dart pub global run canary ${{ inputs.repos_file }} ${{ github.repositoryUrl }} ${{ github.head_ref || github.ref_name }} "${{ toJSON(github.event.pull_request.labels.*.name) }}" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ecosystem_test_internal.yaml b/.github/workflows/canary_internal.yaml similarity index 73% rename from .github/workflows/ecosystem_test_internal.yaml rename to .github/workflows/canary_internal.yaml index fb65bf3a..af0a6032 100644 --- a/.github/workflows/ecosystem_test_internal.yaml +++ b/.github/workflows/canary_internal.yaml @@ -1,4 +1,4 @@ -name: Ecosystem test:Internal +name: Canary:Internal on: pull_request: @@ -7,7 +7,7 @@ on: jobs: test_ecosystem: - uses: ./.github/workflows/ecosystem_test.yaml + uses: ./.github/workflows/canary.yaml with: repos_file: .github/test_repos/repos.json local_debug: true diff --git a/.github/workflows/quest.yml b/.github/workflows/canary_test.yml similarity index 77% rename from .github/workflows/quest.yml rename to .github/workflows/canary_test.yml index 0c63c576..05779e24 100644 --- a/.github/workflows/quest.yml +++ b/.github/workflows/canary_test.yml @@ -1,4 +1,4 @@ -name: package:quest +name: package:canary permissions: read-all @@ -6,19 +6,19 @@ on: pull_request: branches: [ main ] paths: - - '.github/workflows/quest.yml' - - 'pkgs/quest/**' + - '.github/workflows/canary.yml' + - 'pkgs/canary/**' push: branches: [ main ] paths: - - '.github/workflows/quest.yml' - - 'pkgs/quest/**' + - '.github/workflows/canary.yml' + - 'pkgs/canary/**' schedule: - cron: '0 0 * * 0' # weekly defaults: run: - working-directory: pkgs/quest + working-directory: pkgs/canary env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/post_summaries.yaml b/.github/workflows/post_summaries.yaml index 7030dd88..5c905af8 100644 --- a/.github/workflows/post_summaries.yaml +++ b/.github/workflows/post_summaries.yaml @@ -8,7 +8,7 @@ on: workflows: - Publish:Internal - Health:Internal - - Ecosystem test:Internal + - Canary:Internal types: - completed diff --git a/README.md b/README.md index 0fe25b38..202eccf8 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ This repository is home to general Dart Ecosystem tools and packages. | Package | Description | Version | | --- | --- | --- | | [blast_repo](pkgs/blast_repo/) | A tool to bulk validate and fix GitHub repos. | | +| [canary](pkgs/canary/) | Test package upgrades against the ecosystem. | | | [corpus](pkgs/corpus/) | A tool to calculate the API usage for a package. | | | [dart_flutter_team_lints](pkgs/dart_flutter_team_lints/) | An analysis rule set used by the Dart and Flutter teams. | [![pub package](https://img.shields.io/pub/v/dart_flutter_team_lints.svg)](https://pub.dev/packages/dart_flutter_team_lints) | | [firehose](pkgs/firehose/) | A tool to automate publishing of Pub packages from GitHub actions. | [![pub package](https://img.shields.io/pub/v/firehose.svg)](https://pub.dev/packages/firehose) | diff --git a/pkgs/quest/.gitignore b/pkgs/canary/.gitignore similarity index 100% rename from pkgs/quest/.gitignore rename to pkgs/canary/.gitignore diff --git a/pkgs/quest/LICENSE b/pkgs/canary/LICENSE similarity index 100% rename from pkgs/quest/LICENSE rename to pkgs/canary/LICENSE diff --git a/pkgs/quest/README.md b/pkgs/canary/README.md similarity index 77% rename from pkgs/quest/README.md rename to pkgs/canary/README.md index 69ade6b6..dedad8f2 100644 --- a/pkgs/quest/README.md +++ b/pkgs/canary/README.md @@ -1,5 +1,5 @@ -# Quest: Ecosystem Testing for Dart Packages -Embark your package on a quest of testing against a suite of applications. This helps identify potential breaking changes introduced by package updates, ensuring seamless integration across the ecosystem. +# Canary: Ecosystem Testing for Dart Packages +Before publishing, send a canary out to test a package against a suite of applications. This helps identify potential breaking changes introduced by package updates, ensuring seamless integration across the ecosystem. ## What does it do? It checks if your package upgrade would result in failures in the ecosystem. This is achieved by running the following pseudocode: @@ -35,9 +35,9 @@ for (final app in applicationSuite) { } ``` -2. Add a workflow file with the following contents: +2. Add a workflow file `canary.yaml` with the following contents: ```yaml -name: Ecosystem test +name: Canary on: pull_request: @@ -46,12 +46,12 @@ on: jobs: test_ecosystem: - uses: dart-lang/ecosystem/.github/workflows/ecosystem_test.yaml@main + uses: dart-lang/ecosystem/.github/workflows/canary.yaml@main with: repos_file: .github/test_repos/repos.json ``` -3. To show the markdown result as a comment, also add a workflow file +3. To show the markdown result as a comment, also add a workflow file `post_summaries.yaml` ```yaml name: Comment on the pull request @@ -60,9 +60,7 @@ on: # do things like create comments on the PR, even if the original workflow couldn't. workflow_run: workflows: - - Health - - Publish - - Ecosystem test + - Canary types: - completed diff --git a/pkgs/quest/analysis_options.yaml b/pkgs/canary/analysis_options.yaml similarity index 100% rename from pkgs/quest/analysis_options.yaml rename to pkgs/canary/analysis_options.yaml diff --git a/pkgs/quest/bin/quest.dart b/pkgs/canary/bin/canary.dart similarity index 64% rename from pkgs/quest/bin/quest.dart rename to pkgs/canary/bin/canary.dart index 252dbb44..1f75cdf7 100644 --- a/pkgs/quest/bin/quest.dart +++ b/pkgs/canary/bin/canary.dart @@ -29,82 +29,88 @@ Future main(List arguments) async { labels.any((label) => label == 'ecosystem-test-${package.name}'), ); if (package != null) { - print('Found $package. Embark on a quest!'); + print('Found $package to serve as a canary.'); final version = '${package.name}:${json.encode({ 'git': { 'url': gitUri, 'ref': branch, - 'path': - p.relative(package.directory.path, from: Directory.current.path) + 'path': p.relative( + package.directory.path, + from: Directory.current.path, + ) }, })}'; - final chronicles = - await Quest(package.name, version, repositoriesFile).embark(); - final comment = createComment(chronicles); + final mineAirQuality = + await Canary(package.name, version, repositoriesFile).intoTheMine(); + final comment = createComment(mineAirQuality); await writeComment(comment); - print(chronicles); - exitCode = chronicles.success ? 0 : 1; + print(mineAirQuality); + exitCode = mineAirQuality.success ? 0 : 1; } } enum Level { solve, analyze, test } -/// The result of embarking on a quest. Stores the [package] which was tested -/// with its new [version] as well as the [chapters] of the chronicles, each -/// storing the result of testing a single [Application]. -class Chronicles { +/// A mapping of a package to application test results. +/// +/// The result of sending a canary into the mine. Stores the [package] which was +/// tested with its new [version] as well as the [shaftAirQualities] holding the +/// information on each individual [Application] which was tested against. +class MineAirQuality { final String package; final String version; - final Map chapters; + final Map shaftAirQualities; - Chronicles(this.package, this.version, this.chapters); + MineAirQuality(this.package, this.version, this.shaftAirQualities); - bool get success => chapters.values.every((chapter) => chapter.success); + bool get success => shaftAirQualities.values.every((shaft) => shaft.success); @override - String toString() { - return ''' -Chronicles(package: $package, version: $version, chapters: $chapters)'''; - } + String toString() => ''' +MineAirQuality(package: $package, version: $version, shaftAirQualities: $shaftAirQualities)'''; } -/// An individual chapter in the [Chronicles]. This stores the result of testing -/// against an individual [Application]. -class Chapter { - final Map before; - final Map after; +/// Test results for a specific application. +/// +/// This stores the result of testing the canary against an individual +/// [Application]. +class ShaftAirQuality { + final Map before; + final Map after; - Chapter({required this.before, required this.after}); + ShaftAirQuality({required this.before, required this.after}); bool get success => failure == null; Level? get failure => Level.values.firstWhereOrNull( (level) => - before[level]?.success == true && after[level]?.success == false, + before[level]?.breathable == true && + after[level]?.breathable == false, ); String toRow(Application application) => ''' -| ${application.name} | ${Level.values.map((l) => '${before[l]?.success.toEmoji ?? '-'}/${after[l]?.success.toEmoji ?? '-'}').join(' | ')} |'''; +| ${application.name} | ${Level.values.map((l) => '${before[l]?.breathable.toEmoji ?? '-'}/${after[l]?.breathable.toEmoji ?? '-'}').join(' | ')} |'''; @override - String toString() => 'Chapter(before: $before, after: $after)'; + String toString() => 'ShaftAirQuality(before: $before, after: $after)'; } /// A success bool paired with the stdout and stderr for easier debugging. -class CheckResult { - final bool success; +class AirQuality { + /// Whether the check was a success. + final bool breathable; final String stdout; final String stderr; - CheckResult({ - required this.success, + AirQuality({ + required this.breathable, required this.stdout, required this.stderr, }); @override String toString() => - 'ChapterLevel(success: $success, stdout: $stdout, stderr: $stderr)'; + 'AirQuality(success: $breathable, stdout: $stdout, stderr: $stderr)'; } /// An application to test against, specified by the [url] where it can be @@ -137,26 +143,28 @@ class Application { } @override - String toString() => 'Repository(url: $url, name: $name, level: $level)'; + String toString() => 'Application(url: $url, name: $name, level: $level)'; } -/// Contains the logic to fill [Chronicles] with the [Chapter]s of testing the -/// [candidatePackage] at [version] against the [Application]s listed in the -/// [applicationFile]. -class Quest { - final String candidatePackage; +/// Contains the logic to determine the [MineAirQuality] and its individual +/// [ShaftAirQuality]s, by testing the [canaryPackage] at [version] against the +/// [Application]s listed in the [applicationFile]. +class Canary { + final String canaryPackage; final String version; + + /// The mine in the analogy, where we send our canary to test. final String applicationFile; - Quest(this.candidatePackage, this.version, this.applicationFile); + Canary(this.canaryPackage, this.version, this.applicationFile); /// For each package under test, this: /// * Does a pub get (and optionally analyze and test) /// * Upgrades to the new dep version /// * Again runs pub get (and optionally analyze and test) - Future embark() async { + Future intoTheMine() async { final tempDir = await Directory.systemTemp.createTemp(); - final chapters = {}; + final shaftAirQualities = {}; for (var application in await Application.listFromFile(applicationFile)) { final path = await cloneRepo(application.url, tempDir); print('Cloned $application into $path'); @@ -167,36 +175,36 @@ class Quest { as Map; final depsPackages = depsJson['packages'] as List; print(depsPackages); - if (depsPackages.any((p) => (p as Map)['name'] == candidatePackage)) { - print('Run checks for vanilla package'); + if (depsPackages.any((p) => (p as Map)['name'] == canaryPackage)) { + print('Test against the vanilla package'); final resultBefore = await runChecks(path, application.level); print('Clean repo'); await runFlutter(['clean'], path); - print('Rev package:$candidatePackage to version $version $application'); + print('Rev package:$canaryPackage to version $version $application'); final revSuccess = await runFlutter( ['pub', 'add', version], path, true, ); - print('Run checks for modified package'); + print('Test against the modified package'); final resultAfter = await runChecks(path, application.level); // flutter pub add runs an implicit pub get resultAfter[Level.solve] = revSuccess; - chapters[application] = Chapter( + shaftAirQualities[application] = ShaftAirQuality( before: resultBefore, after: resultAfter, ); } else { - print('No package:$candidatePackage found in $application'); + print('No package:$canaryPackage found in $application'); } } await tempDir.delete(recursive: true); - return Chronicles(candidatePackage, version, chapters); + return MineAirQuality(canaryPackage, version, shaftAirQualities); } /// Uses `gh` to clone the Github repo at [url]. @@ -219,21 +227,21 @@ class Quest { return fullPath; } - Future> runChecks(String path, Level level) async { - final result = {}; + Future> runChecks(String path, Level level) async { + final result = {}; result[Level.solve] = await runFlutter(['pub', 'get'], path); if (level.index >= Level.analyze.index && - result[Level.solve]?.success == true) { + result[Level.solve]?.breathable == true) { result[Level.analyze] = await runFlutter(['analyze'], path); } if (level.index >= Level.test.index && - result[Level.solve]?.success == true) { + result[Level.solve]?.breathable == true) { result[Level.test] = await runFlutter(['test'], path); } return result; } - Future runFlutter( + Future runFlutter( List arguments, String path, [ bool useDart = false, @@ -252,8 +260,8 @@ class Quest { print(stdout); print('stderr:'); print(stderr); - return CheckResult( - success: processResult.exitCode == 0, + return AirQuality( + breathable: processResult.exitCode == 0, stdout: stdout, stderr: stderr, ); @@ -266,37 +274,37 @@ Future writeComment(String content) async { await commentFile.writeAsString(content); } -String createComment(Chronicles chronicles) { +String createComment(MineAirQuality mine) { final contents = ''' ## Ecosystem testing | Package | Solve | Analyze | Test | | ------- | ----- | ------- | ---- | -${chronicles.chapters.entries.map((chapter) => chapter.value.toRow(chapter.key)).join('\n')} +${mine.shaftAirQualities.entries.map((shaft) => shaft.value.toRow(shaft.key)).join('\n')}
Details per app -${chronicles.chapters.entries.map((entry) { +${mine.shaftAirQualities.entries.map((entry) { final application = entry.key; - final chapter = entry.value; + final shaft = entry.value; return '''
-${application.name} ${chapter.success ? checkEmoji : crossEmoji} +${application.name} ${shaft.success ? checkEmoji : crossEmoji} -${chapter.success ? 'The app tests passed!' : ''' -The failure occured at the "${chapter.failure!.name}" step, this is the error output of that step: +${shaft.success ? 'The app tests passed!' : ''' +The failure occured at the "${shaft.failure!.name}" step, this is the error output of that step: ``` -${chapter.after[chapter.failure!]?.stderr} +${shaft.after[shaft.failure!]?.stderr} ``` '''} The complete list of logs is: -${chapter.before.keys.map((level) => ''' +${shaft.before.keys.map((level) => '''
Logs for step: ${level.name} @@ -307,24 +315,24 @@ ${chapter.before.keys.map((level) => ''' StdOut: ``` -${chapter.before[level]?.stdout} +${shaft.before[level]?.stdout} ``` StdErr: ``` -${chapter.before[level]?.stderr} +${shaft.before[level]?.stderr} ``` ### After: StdOut: ``` -${chapter.after[level]?.stdout} +${shaft.after[level]?.stdout} ``` StdErr: ``` -${chapter.after[level]?.stderr} +${shaft.after[level]?.stderr} ```
diff --git a/pkgs/quest/pubspec.yaml b/pkgs/canary/pubspec.yaml similarity index 93% rename from pkgs/quest/pubspec.yaml rename to pkgs/canary/pubspec.yaml index 07e09a2b..a6597dbb 100644 --- a/pkgs/quest/pubspec.yaml +++ b/pkgs/canary/pubspec.yaml @@ -1,6 +1,6 @@ -name: quest +name: canary description: Test package upgrades against the ecosystem. -repository: https://github.com/dart-lang/ecosystem/tree/main/pkgs/quest +repository: https://github.com/dart-lang/ecosystem/tree/main/pkgs/canary publish_to: none diff --git a/pkgs/quest/schema.json b/pkgs/canary/schema.json similarity index 100% rename from pkgs/quest/schema.json rename to pkgs/canary/schema.json diff --git a/pkgs/quest/test/quest_test.dart b/pkgs/canary/test/canary_test.dart similarity index 90% rename from pkgs/quest/test/quest_test.dart rename to pkgs/canary/test/canary_test.dart index e118ebd9..aed3ef74 100644 --- a/pkgs/quest/test/quest_test.dart +++ b/pkgs/canary/test/canary_test.dart @@ -8,7 +8,7 @@ import 'dart:io'; import 'package:path/path.dart' as p; import 'package:test/test.dart'; -import '../bin/quest.dart'; +import '../bin/canary.dart'; void main() { test('test name', () async { @@ -21,13 +21,13 @@ void main() { }), ); - final chronicles = await Quest( + final mineAirQuality = await Canary( 'intl', 'intl:{"git":{"url":"https://github.com/mosuem/i18n","ref":"pr","path":"pkgs/intl"}}', repoFile.path, - ).embark(); + ).intoTheMine(); - final comment = createComment(chronicles); + final comment = createComment(mineAirQuality); expect(comment, startsWith(goldenComment)); await temp.delete(recursive: true); }, timeout: const Timeout(Duration(minutes: 5))); diff --git a/pkgs/trebuchet/LICENSE b/pkgs/trebuchet/LICENSE new file mode 100644 index 00000000..b03a7886 --- /dev/null +++ b/pkgs/trebuchet/LICENSE @@ -0,0 +1,27 @@ +Copyright 2024, the Dart project authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkgs/trebuchet/pubspec.yaml b/pkgs/trebuchet/pubspec.yaml index 417a2d02..02413f3c 100644 --- a/pkgs/trebuchet/pubspec.yaml +++ b/pkgs/trebuchet/pubspec.yaml @@ -1,5 +1,6 @@ name: trebuchet description: A tool for hurling packages into monorepos. +repository: https://github.com/dart-lang/ecosystem/tree/main/pkgs/trebuchet publish_to: none