[flutter_plugin_tool] Add custom-test command (#5058)

script/tool/CHANGELOG.md

@@ -1,5 +1,6 @@
-## NEXT
+## 0.8.2
 
+- Adds a new `custom-test` command.
 - Switches from deprecated `flutter packages` alias to `flutter pub`.
 
 ## 0.8.1

script/tool/lib/src/custom_test_command.dart

@@ -0,0 +1,77 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file/file.dart';
+import 'package:platform/platform.dart';
+
+import 'common/package_looping_command.dart';
+import 'common/process_runner.dart';
+import 'common/repository_package.dart';
+
+const String _scriptName = 'run_tests.dart';
+const String _legacyScriptName = 'run_tests.sh';
+
+/// A command to run custom, package-local tests on packages.
+///
+/// This is an escape hatch for adding tests that this tooling doesn't support.
+/// It should be used sparingly; prefer instead to add functionality to this
+/// tooling to eliminate the need for bespoke tests.
+class CustomTestCommand extends PackageLoopingCommand {
+  /// Creates a custom test command instance.
+  CustomTestCommand(
+    Directory packagesDir, {
+    ProcessRunner processRunner = const ProcessRunner(),
+    Platform platform = const LocalPlatform(),
+  }) : super(packagesDir, processRunner: processRunner, platform: platform);
+
+  @override
+  final String name = 'custom-test';
+
+  @override
+  final String description = 'Runs package-specific custom tests defined in '
+      'a package\'s tool/$_scriptName file.\n\n'
+      'This command requires "dart" to be in your path.';
+
+  @override
+  Future<PackageResult> runForPackage(RepositoryPackage package) async {
+    final File script =
+        package.directory.childDirectory('tool').childFile(_scriptName);
+    final File legacyScript = package.directory.childFile(_legacyScriptName);
+    String? customSkipReason;
+    bool ranTests = false;
+
+    // Run the custom Dart script if presest.
+    if (script.existsSync()) {
+      final int exitCode = await processRunner.runAndStream(
+          'dart', <String>['run', 'tool/$_scriptName'],
+          workingDir: package.directory);
+      if (exitCode != 0) {
+        return PackageResult.fail();
+      }
+      ranTests = true;
+    }
+
+    // Run the legacy script if present.
+    if (legacyScript.existsSync()) {
+      if (platform.isWindows) {
+        customSkipReason = '$_legacyScriptName is not supported on Windows. '
+            'Please migrate to $_scriptName.';
+      } else {
+        final int exitCode = await processRunner.runAndStream(
+            legacyScript.path, <String>[],
+            workingDir: package.directory);
+        if (exitCode != 0) {
+          return PackageResult.fail();
+        }
+        ranTests = true;
+      }
+    }
+
+    if (!ranTests) {
+      return PackageResult.skip(customSkipReason ?? 'No custom tests');
+    }
+
+    return PackageResult.success();
+  }
+}

script/tool/lib/src/main.dart

@@ -12,6 +12,7 @@ import 'analyze_command.dart';
 import 'build_examples_command.dart';
 import 'common/core.dart';
 import 'create_all_plugins_app_command.dart';
+import 'custom_test_command.dart';
 import 'drive_examples_command.dart';
 import 'federation_safety_check_command.dart';
 import 'firebase_test_lab_command.dart';
@@ -50,6 +51,7 @@ void main(List<String> args) {
     ..addCommand(AnalyzeCommand(packagesDir))
     ..addCommand(BuildExamplesCommand(packagesDir))
     ..addCommand(CreateAllPluginsAppCommand(packagesDir))
+    ..addCommand(CustomTestCommand(packagesDir))
     ..addCommand(DriveExamplesCommand(packagesDir))
     ..addCommand(FederationSafetyCheckCommand(packagesDir))
     ..addCommand(FirebaseTestLabCommand(packagesDir))

script/tool/pubspec.yaml

@@ -1,7 +1,7 @@
 name: flutter_plugin_tools
 description: Productivity utils for flutter/plugins and flutter/packages
 repository: https://github.com/flutter/plugins/tree/main/script/tool
-version: 0.8.1
+version: 0.8.2
 
 dependencies:
   args: ^2.1.0

script/tool/test/custom_test_command_test.dart

@@ -0,0 +1,281 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+
+import 'package:args/command_runner.dart';
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+import 'package:flutter_plugin_tools/src/common/core.dart';
+import 'package:flutter_plugin_tools/src/custom_test_command.dart';
+import 'package:test/test.dart';
+
+import 'mocks.dart';
+import 'util.dart';
+
+void main() {
+  late FileSystem fileSystem;
+  late MockPlatform mockPlatform;
+  late Directory packagesDir;
+  late RecordingProcessRunner processRunner;
+  late CommandRunner<void> runner;
+
+  group('posix', () {
+    setUp(() {
+      fileSystem = MemoryFileSystem();
+      mockPlatform = MockPlatform();
+      packagesDir = createPackagesDirectory(fileSystem: fileSystem);
+      processRunner = RecordingProcessRunner();
+      final CustomTestCommand analyzeCommand = CustomTestCommand(
+        packagesDir,
+        processRunner: processRunner,
+        platform: mockPlatform,
+      );
+
+      runner = CommandRunner<void>(
+          'custom_test_command', 'Test for custom_test_command');
+      runner.addCommand(analyzeCommand);
+    });
+
+    test('runs both new and legacy when both are present', () async {
+      final Directory package =
+          createFakePlugin('a_package', packagesDir, extraFiles: <String>[
+        'tool/run_tests.dart',
+        'run_tests.sh',
+      ]);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(
+          processRunner.recordedCalls,
+          containsAll(<ProcessCall>[
+            ProcessCall(package.childFile('run_tests.sh').path,
+                const <String>[], package.path),
+            ProcessCall('dart', const <String>['run', 'tool/run_tests.dart'],
+                package.path),
+          ]));
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Ran for 1 package(s)'),
+          ]));
+    });
+
+    test('runs when only new is present', () async {
+      final Directory package = createFakePlugin('a_package', packagesDir,
+          extraFiles: <String>['tool/run_tests.dart']);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(
+          processRunner.recordedCalls,
+          containsAll(<ProcessCall>[
+            ProcessCall('dart', const <String>['run', 'tool/run_tests.dart'],
+                package.path),
+          ]));
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Ran for 1 package(s)'),
+          ]));
+    });
+
+    test('runs when only legacy is present', () async {
+      final Directory package = createFakePlugin('a_package', packagesDir,
+          extraFiles: <String>['run_tests.sh']);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(
+          processRunner.recordedCalls,
+          containsAll(<ProcessCall>[
+            ProcessCall(package.childFile('run_tests.sh').path,
+                const <String>[], package.path),
+          ]));
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Ran for 1 package(s)'),
+          ]));
+    });
+
+    test('skips when neither is present', () async {
+      createFakePlugin('a_package', packagesDir);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(processRunner.recordedCalls, isEmpty);
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Skipped 1 package(s)'),
+          ]));
+    });
+
+    test('fails if new fails', () async {
+      createFakePlugin('a_package', packagesDir, extraFiles: <String>[
+        'tool/run_tests.dart',
+        'run_tests.sh',
+      ]);
+
+      processRunner.mockProcessesForExecutable['dart'] = <io.Process>[
+        MockProcess(exitCode: 1),
+      ];
+
+      Error? commandError;
+      final List<String> output = await runCapturingPrint(
+          runner, <String>['custom-test'], errorHandler: (Error e) {
+        commandError = e;
+      });
+
+      expect(commandError, isA<ToolExit>());
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('The following packages had errors:'),
+            contains('a_package')
+          ]));
+    });
+
+    test('fails if legacy fails', () async {
+      final Directory package =
+          createFakePlugin('a_package', packagesDir, extraFiles: <String>[
+        'tool/run_tests.dart',
+        'run_tests.sh',
+      ]);
+
+      processRunner.mockProcessesForExecutable[
+          package.childFile('run_tests.sh').path] = <io.Process>[
+        MockProcess(exitCode: 1),
+      ];
+
+      Error? commandError;
+      final List<String> output = await runCapturingPrint(
+          runner, <String>['custom-test'], errorHandler: (Error e) {
+        commandError = e;
+      });
+
+      expect(commandError, isA<ToolExit>());
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('The following packages had errors:'),
+            contains('a_package')
+          ]));
+    });
+  });
+
+  group('Windows', () {
+    setUp(() {
+      fileSystem = MemoryFileSystem(style: FileSystemStyle.windows);
+      mockPlatform = MockPlatform(isWindows: true);
+      packagesDir = createPackagesDirectory(fileSystem: fileSystem);
+      processRunner = RecordingProcessRunner();
+      final CustomTestCommand analyzeCommand = CustomTestCommand(
+        packagesDir,
+        processRunner: processRunner,
+        platform: mockPlatform,
+      );
+
+      runner = CommandRunner<void>(
+          'custom_test_command', 'Test for custom_test_command');
+      runner.addCommand(analyzeCommand);
+    });
+
+    test('runs new and skips old when both are present', () async {
+      final Directory package =
+          createFakePlugin('a_package', packagesDir, extraFiles: <String>[
+        'tool/run_tests.dart',
+        'run_tests.sh',
+      ]);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(
+          processRunner.recordedCalls,
+          containsAll(<ProcessCall>[
+            ProcessCall('dart', const <String>['run', 'tool/run_tests.dart'],
+                package.path),
+          ]));
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Ran for 1 package(s)'),
+          ]));
+    });
+
+    test('runs when only new is present', () async {
+      final Directory package = createFakePlugin('a_package', packagesDir,
+          extraFiles: <String>['tool/run_tests.dart']);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(
+          processRunner.recordedCalls,
+          containsAll(<ProcessCall>[
+            ProcessCall('dart', const <String>['run', 'tool/run_tests.dart'],
+                package.path),
+          ]));
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('Ran for 1 package(s)'),
+          ]));
+    });
+
+    test('skips package when only legacy is present', () async {
+      createFakePlugin('a_package', packagesDir,
+          extraFiles: <String>['run_tests.sh']);
+
+      final List<String> output =
+          await runCapturingPrint(runner, <String>['custom-test']);
+
+      expect(processRunner.recordedCalls, isEmpty);
+
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('run_tests.sh is not supported on Windows'),
+            contains('Skipped 1 package(s)'),
+          ]));
+    });
+
+    test('fails if new fails', () async {
+      createFakePlugin('a_package', packagesDir, extraFiles: <String>[
+        'tool/run_tests.dart',
+        'run_tests.sh',
+      ]);
+
+      processRunner.mockProcessesForExecutable['dart'] = <io.Process>[
+        MockProcess(exitCode: 1),
+      ];
+
+      Error? commandError;
+      final List<String> output = await runCapturingPrint(
+          runner, <String>['custom-test'], errorHandler: (Error e) {
+        commandError = e;
+      });
+
+      expect(commandError, isA<ToolExit>());
+      expect(
+          output,
+          containsAllInOrder(<Matcher>[
+            contains('The following packages had errors:'),
+            contains('a_package')
+          ]));
+    });
+  });
+}