From bb5a25815cae4125f2a5903a2b376db58cff2e66 Mon Sep 17 00:00:00 2001 From: Jenn Magder Date: Tue, 5 Nov 2024 22:58:26 -0800 Subject: [PATCH] [ci] Upload screenshots, logs, and Xcode test results for drive and integration_test runs (#7430) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Native Xcode tests will output a helpful "xcresult" package on failure containing logs, screenshots, and screen recordings. Zip and upload these results when tests fail. 2. Pass `flutter test --debug-logs-dir` flag to upload logs like https://github.com/flutter/flutter/pull/142643. 3. Pass `flutter drive --screenshot` flag to upload screenshots on timeout like https://github.com/flutter/flutter/pull/96973. Example of [failing Xcode analyzer build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20ios_platform_tests_shard_5%20master/17374/overview) has the [zipped xcresult](https://storage.googleapis.com/flutter_logs/flutter/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcode%20analyze/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcodebuild-2024-10-25T09:56:46.440913.zip) attached as a log. ![Screenshot 2024-10-25 at 10 10 36 AM](https://github.com/user-attachments/assets/dd7ae9bc-6161-4381-8a4f-f10b8c981801) The unzipped xcresult looks like this in Xcode: ![Screenshot 2024-10-25 at 10 11 55 AM](https://github.com/user-attachments/assets/d4dd8420-f272-459c-9785-ab0c03887a74) A [failing "native test" step build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20macos_platform_tests%20master%20-%20packages/17315/overview): ![Screenshot 2024-10-25 at 10 19 55 AM](https://github.com/user-attachments/assets/76a86a15-2150-482a-8b15-e3e7ac90485e) Fixes https://github.com/flutter/flutter/issues/144795 --- script/tool/lib/src/common/core.dart | 11 ++ script/tool/lib/src/common/xcode.dart | 95 ++++++++++---- .../tool/lib/src/drive_examples_command.dart | 19 ++- script/tool/lib/src/native_test_command.dart | 16 +-- .../tool/lib/src/xcode_analyze_command.dart | 15 +-- script/tool/test/common/xcode_test.dart | 4 + .../test/drive_examples_command_test.dart | 119 ++++++++++++++++++ script/tool/test/mocks.dart | 3 + 8 files changed, 242 insertions(+), 40 deletions(-) diff --git a/script/tool/lib/src/common/core.dart b/script/tool/lib/src/common/core.dart index a26c8391596a..9fef751ab56d 100644 --- a/script/tool/lib/src/common/core.dart +++ b/script/tool/lib/src/common/core.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'package:file/file.dart'; +import 'package:platform/platform.dart'; import 'package:pub_semver/pub_semver.dart'; /// The signature for a print handler for commands that allow overriding the @@ -127,3 +128,13 @@ const int exitCommandFoundErrors = 1; /// A exit code for [ToolExit] for a failure to run due to invalid arguments. const int exitInvalidArguments = 2; + +/// The directory to which to write logs and other artifacts, if set in CI. +Directory? ciLogsDirectory(Platform platform, FileSystem fileSystem) { + final String? logsDirectoryPath = platform.environment['FLUTTER_LOGS_DIR']; + Directory? logsDirectory; + if (logsDirectoryPath != null) { + logsDirectory = fileSystem.directory(logsDirectoryPath); + } + return logsDirectory; +} diff --git a/script/tool/lib/src/common/xcode.dart b/script/tool/lib/src/common/xcode.dart index 4a4bf03ba549..7ba23fc000d3 100644 --- a/script/tool/lib/src/common/xcode.dart +++ b/script/tool/lib/src/common/xcode.dart @@ -6,7 +6,9 @@ import 'dart:convert'; import 'dart:io' as io; import 'package:file/file.dart'; +import 'package:platform/platform.dart'; +import 'core.dart'; import 'output_utils.dart'; import 'process_runner.dart'; @@ -33,36 +35,81 @@ class Xcode { /// Runs an `xcodebuild` in [directory] with the given parameters. Future runXcodeBuild( Directory exampleDirectory, - String platform, { + String targetPlatform, { List actions = const ['build'], required String workspace, required String scheme, String? configuration, List extraFlags = const [], - }) { - File? disabledSandboxEntitlementFile; - if (actions.contains('test') && platform.toLowerCase() == 'macos') { - disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile( - exampleDirectory.childDirectory(platform.toLowerCase()), - configuration ?? 'Debug', - ); - } - final List args = [ - _xcodeBuildCommand, - ...actions, - ...['-workspace', workspace], - ...['-scheme', scheme], - if (configuration != null) ...['-configuration', configuration], - ...extraFlags, - if (disabledSandboxEntitlementFile != null) - 'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}', - ]; - final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}'; - if (log) { - print(completeTestCommand); + required Platform hostPlatform, + }) async { + final FileSystem fileSystem = exampleDirectory.fileSystem; + String? resultBundlePath; + final Directory? logsDirectory = ciLogsDirectory(hostPlatform, fileSystem); + Directory? resultBundleTemp; + try { + if (logsDirectory != null) { + resultBundleTemp = + fileSystem.systemTempDirectory.createTempSync('flutter_xcresult.'); + resultBundlePath = resultBundleTemp.childDirectory('result').path; + } + File? disabledSandboxEntitlementFile; + if (actions.contains('test') && targetPlatform.toLowerCase() == 'macos') { + disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile( + exampleDirectory.childDirectory(targetPlatform.toLowerCase()), + configuration ?? 'Debug', + ); + } + final List args = [ + _xcodeBuildCommand, + ...actions, + ...['-workspace', workspace], + ...['-scheme', scheme], + if (resultBundlePath != null) ...[ + '-resultBundlePath', + resultBundlePath + ], + if (configuration != null) ...['-configuration', configuration], + ...extraFlags, + if (disabledSandboxEntitlementFile != null) + 'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}', + ]; + final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}'; + if (log) { + print(completeTestCommand); + } + final int resultExit = await processRunner + .runAndStream(_xcRunCommand, args, workingDir: exampleDirectory); + + if (resultExit != 0 && resultBundleTemp != null) { + final Directory xcresultBundle = + resultBundleTemp.childDirectory('result.xcresult'); + if (logsDirectory != null) { + if (xcresultBundle.existsSync()) { + // Zip the test results to the artifacts directory for upload. + final File zipPath = logsDirectory.childFile( + 'xcodebuild-${DateTime.now().toLocal().toIso8601String()}.zip'); + await processRunner.run( + 'zip', + [ + '-r', + '-9', + '-q', + zipPath.path, + xcresultBundle.basename, + ], + workingDir: resultBundleTemp, + ); + } else { + print( + 'xcresult bundle ${xcresultBundle.path} does not exist, skipping upload'); + } + } + } + return resultExit; + } finally { + resultBundleTemp?.deleteSync(recursive: true); } - return processRunner.runAndStream(_xcRunCommand, args, - workingDir: exampleDirectory); } /// Returns true if [project], which should be an .xcodeproj directory, diff --git a/script/tool/lib/src/drive_examples_command.dart b/script/tool/lib/src/drive_examples_command.dart index a5ddd3bd4a9d..8ae124856021 100644 --- a/script/tool/lib/src/drive_examples_command.dart +++ b/script/tool/lib/src/drive_examples_command.dart @@ -228,8 +228,12 @@ class DriveExamplesCommand extends PackageLoopingCommand { } for (final File driver in drivers) { final List failingTargets = await _driveTests( - example, driver, testTargets, - deviceFlags: deviceFlags); + example, + driver, + testTargets, + deviceFlags: deviceFlags, + exampleName: exampleName, + ); for (final File failingTarget in failingTargets) { errors.add( getRelativePosixPath(failingTarget, from: package.directory)); @@ -376,10 +380,16 @@ class DriveExamplesCommand extends PackageLoopingCommand { File driver, List targets, { required List deviceFlags, + required String exampleName, }) async { final List failures = []; final String enableExperiment = getStringArg(kEnableExperiment); + final String screenshotBasename = + '${exampleName.replaceAll(platform.pathSeparator, '_')}-drive'; + final Directory? screenshotDirectory = + ciLogsDirectory(platform, driver.fileSystem) + ?.childDirectory(screenshotBasename); for (final File target in targets) { final int exitCode = await processRunner.runAndStream( @@ -389,6 +399,8 @@ class DriveExamplesCommand extends PackageLoopingCommand { ...deviceFlags, if (enableExperiment.isNotEmpty) '--enable-experiment=$enableExperiment', + if (screenshotDirectory != null) + '--screenshot=${screenshotDirectory.path}', '--driver', getRelativePosixPath(driver, from: example.directory), '--target', @@ -416,6 +428,8 @@ class DriveExamplesCommand extends PackageLoopingCommand { required List testFiles, }) async { final String enableExperiment = getStringArg(kEnableExperiment); + final Directory? logsDirectory = + ciLogsDirectory(platform, testFiles.first.fileSystem); // Workaround for https://github.com/flutter/flutter/issues/135673 // Once that is fixed on stable, this logic can be removed and the command @@ -438,6 +452,7 @@ class DriveExamplesCommand extends PackageLoopingCommand { ...deviceFlags, if (enableExperiment.isNotEmpty) '--enable-experiment=$enableExperiment', + if (logsDirectory != null) '--debug-logs-dir=${logsDirectory.path}', target, ], workingDir: example.directory); diff --git a/script/tool/lib/src/native_test_command.dart b/script/tool/lib/src/native_test_command.dart index 1cd7afb6b4ae..47fd57998765 100644 --- a/script/tool/lib/src/native_test_command.dart +++ b/script/tool/lib/src/native_test_command.dart @@ -431,7 +431,7 @@ this command. /// usually at "example/{ios,macos}/Runner.xcworkspace". Future<_PlatformResult> _runXcodeTests( RepositoryPackage plugin, - String platform, + String targetPlatform, _TestMode mode, { List extraFlags = const [], }) async { @@ -456,7 +456,7 @@ this command. final String? targetToCheck = testTarget ?? (mode.unit ? unitTestTarget : null); final Directory xcodeProject = example.directory - .childDirectory(platform.toLowerCase()) + .childDirectory(targetPlatform.toLowerCase()) .childDirectory('Runner.xcodeproj'); if (targetToCheck != null) { final bool? hasTarget = @@ -473,16 +473,17 @@ this command. } } - _printRunningExampleTestsMessage(example, platform); + _printRunningExampleTestsMessage(example, targetPlatform); final int exitCode = await _xcode.runXcodeBuild( example.directory, - platform, + targetPlatform, // Clean before testing to remove cached swiftmodules from previous // runs, which can cause conflicts. actions: ['clean', 'test'], - workspace: '${platform.toLowerCase()}/Runner.xcworkspace', + workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace', scheme: 'Runner', configuration: 'Debug', + hostPlatform: platform, extraFlags: [ if (testTarget != null) '-only-testing:$testTarget', ...extraFlags, @@ -494,9 +495,10 @@ this command. const int xcodebuildNoTestExitCode = 66; switch (exitCode) { case xcodebuildNoTestExitCode: - _printNoExampleTestsMessage(example, platform); + _printNoExampleTestsMessage(example, targetPlatform); case 0: - printSuccess('Successfully ran $platform xctest for $exampleName'); + printSuccess( + 'Successfully ran $targetPlatform xctest for $exampleName'); // If this is the first test, assume success until something fails. if (overallResult == RunState.skipped) { overallResult = RunState.succeeded; diff --git a/script/tool/lib/src/xcode_analyze_command.dart b/script/tool/lib/src/xcode_analyze_command.dart index 707a3f08b6d6..ba4b3fb5e95b 100644 --- a/script/tool/lib/src/xcode_analyze_command.dart +++ b/script/tool/lib/src/xcode_analyze_command.dart @@ -97,10 +97,10 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand { multiplePlatformsRequested ? failures : []); } - /// Analyzes [plugin] for [platform], returning true if it passed analysis. + /// Analyzes [plugin] for [targetPlatform], returning true if it passed analysis. Future _analyzePlugin( RepositoryPackage plugin, - String platform, { + String targetPlatform, { List extraFlags = const [], }) async { bool passing = true; @@ -108,25 +108,26 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand { // Running tests and static analyzer. final String examplePath = getRelativePosixPath(example.directory, from: plugin.directory.parent); - print('Running $platform tests and analyzer for $examplePath...'); + print('Running $targetPlatform tests and analyzer for $examplePath...'); final int exitCode = await _xcode.runXcodeBuild( example.directory, - platform, + targetPlatform, // Clean before analyzing to remove cached swiftmodules from previous // runs, which can cause conflicts. actions: ['clean', 'analyze'], - workspace: '${platform.toLowerCase()}/Runner.xcworkspace', + workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace', scheme: 'Runner', configuration: 'Debug', + hostPlatform: platform, extraFlags: [ ...extraFlags, 'GCC_TREAT_WARNINGS_AS_ERRORS=YES', ], ); if (exitCode == 0) { - printSuccess('$examplePath ($platform) passed analysis.'); + printSuccess('$examplePath ($targetPlatform) passed analysis.'); } else { - printError('$examplePath ($platform) failed analysis.'); + printError('$examplePath ($targetPlatform) failed analysis.'); passing = false; } } diff --git a/script/tool/test/common/xcode_test.dart b/script/tool/test/common/xcode_test.dart index 8fd415160015..4d27372890fb 100644 --- a/script/tool/test/common/xcode_test.dart +++ b/script/tool/test/common/xcode_test.dart @@ -165,6 +165,7 @@ void main() { 'ios', workspace: 'A.xcworkspace', scheme: 'AScheme', + hostPlatform: MockPlatform(), ); expect(exitCode, 0); @@ -193,6 +194,7 @@ void main() { workspace: 'A.xcworkspace', scheme: 'AScheme', configuration: 'Debug', + hostPlatform: MockPlatform(), extraFlags: ['-a', '-b', 'c=d']); expect(exitCode, 0); @@ -230,6 +232,7 @@ void main() { 'ios', workspace: 'A.xcworkspace', scheme: 'AScheme', + hostPlatform: MockPlatform(), ); expect(exitCode, 1); @@ -264,6 +267,7 @@ void main() { 'macos', workspace: 'A.xcworkspace', scheme: 'AScheme', + hostPlatform: MockPlatform(), actions: ['test'], ); diff --git a/script/tool/test/drive_examples_command_test.dart b/script/tool/test/drive_examples_command_test.dart index 6bbd5e301ad5..a1d48865bc4a 100644 --- a/script/tool/test/drive_examples_command_test.dart +++ b/script/tool/test/drive_examples_command_test.dart @@ -41,6 +41,7 @@ void main() { // TODO(dit): Clean this up, https://github.com/flutter/flutter/issues/151869 mockPlatform.environment['CHANNEL'] = 'master'; + mockPlatform.environment['FLUTTER_LOGS_DIR'] = '/path/to/logs'; }); void setMockFlutterDevicesOutput({ @@ -318,6 +319,57 @@ void main() { ]), ); + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall(getFlutterCommand(mockPlatform), + const ['devices', '--machine'], null), + ProcessCall( + getFlutterCommand(mockPlatform), + const [ + 'test', + '-d', + _fakeIOSDevice, + '--debug-logs-dir=/path/to/logs', + 'integration_test', + ], + pluginExampleDirectory.path), + ])); + }); + + test('handles missing CI debug logs directory', () async { + mockPlatform.environment.remove('FLUTTER_LOGS_DIR'); + + final RepositoryPackage plugin = createFakePlugin( + 'plugin', + packagesDir, + extraFiles: [ + 'example/integration_test/bar_test.dart', + 'example/integration_test/foo_test.dart', + 'example/integration_test/ignore_me.dart', + 'example/android/android.java', + 'example/ios/ios.m', + ], + platformSupport: { + platformAndroid: const PlatformDetails(PlatformSupport.inline), + platformIOS: const PlatformDetails(PlatformSupport.inline), + }, + ); + + final Directory pluginExampleDirectory = getExampleDir(plugin); + + setMockFlutterDevicesOutput(); + final List output = + await runCapturingPrint(runner, ['drive-examples', '--ios']); + + expect( + output, + containsAllInOrder([ + contains('Running for plugin'), + contains('No issues found!'), + ]), + ); + expect( processRunner.recordedCalls, orderedEquals([ @@ -396,6 +448,7 @@ void main() { 'test', '-d', 'linux', + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -463,6 +516,7 @@ void main() { 'test', '-d', 'macos', + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -510,6 +564,7 @@ void main() { 'test', '-d', 'macos', + '--debug-logs-dir=/path/to/logs', 'integration_test/first_test.dart', ], pluginExampleDirectory.path), @@ -519,6 +574,7 @@ void main() { 'test', '-d', 'macos', + '--debug-logs-dir=/path/to/logs', 'integration_test/second_test.dart', ], pluginExampleDirectory.path), @@ -565,6 +621,7 @@ void main() { 'test', '-d', 'linux', + '--debug-logs-dir=/path/to/logs', 'integration_test/first_test.dart', ], pluginExampleDirectory.path), @@ -574,6 +631,7 @@ void main() { 'test', '-d', 'linux', + '--debug-logs-dir=/path/to/logs', 'integration_test/second_test.dart', ], pluginExampleDirectory.path), @@ -620,6 +678,7 @@ void main() { 'test', '-d', 'windows', + '--debug-logs-dir=/path/to/logs', 'integration_test/first_test.dart', ], pluginExampleDirectory.path), @@ -629,6 +688,7 @@ void main() { 'test', '-d', 'windows', + '--debug-logs-dir=/path/to/logs', 'integration_test/second_test.dart', ], pluginExampleDirectory.path), @@ -700,6 +760,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -751,6 +812,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--wasm', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -805,6 +867,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=html', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -854,6 +917,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -907,6 +971,7 @@ void main() { '--browser-name=chrome', '--web-renderer=canvaskit', '--chrome-binary=/path/to/chrome', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -977,6 +1042,7 @@ void main() { 'test', '-d', 'windows', + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -1023,6 +1089,7 @@ void main() { 'test', '-d', _fakeAndroidDevice, + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -1069,6 +1136,7 @@ void main() { 'test', '-d', _fakeAndroidDevice, + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -1197,6 +1265,7 @@ void main() { '-d', _fakeIOSDevice, '--enable-experiment=exp1', + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -1353,6 +1422,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -1368,6 +1438,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/plugin_example-drive', '--driver', 'test_driver/integration_test.dart', '--target', @@ -1425,6 +1496,7 @@ void main() { 'test', '-d', _fakeIOSDevice, + '--debug-logs-dir=/path/to/logs', 'integration_test', ], pluginExampleDirectory.path), @@ -1454,6 +1526,52 @@ void main() { ]), ); + expect( + processRunner.recordedCalls, + orderedEquals([ + ProcessCall( + getFlutterCommand(mockPlatform), + const [ + 'drive', + '-d', + 'web-server', + '--web-port=7357', + '--browser-name=chrome', + '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/a_package_example-drive', + '--driver', + 'test_driver/integration_test.dart', + '--target', + 'integration_test/foo_test.dart' + ], + exampleDirectory.path), + ])); + }); + + test('drive handles missing CI screenshot directory', () async { + mockPlatform.environment.remove('FLUTTER_LOGS_DIR'); + + final RepositoryPackage package = + createFakePackage('a_package', packagesDir, extraFiles: [ + 'example/integration_test/foo_test.dart', + 'example/test_driver/integration_test.dart', + 'example/web/index.html', + ]); + final Directory exampleDirectory = getExampleDir(package); + + final List output = await runCapturingPrint(runner, [ + 'drive-examples', + '--web', + ]); + + expect( + output, + containsAllInOrder([ + contains('Running for a_package'), + contains('No issues found!'), + ]), + ); + expect( processRunner.recordedCalls, orderedEquals([ @@ -1546,6 +1664,7 @@ void main() { '--web-port=7357', '--browser-name=chrome', '--web-renderer=canvaskit', + '--screenshot=/path/to/logs/a_package_example_with_web-drive', '--driver', 'test_driver/integration_test.dart', '--target', diff --git a/script/tool/test/mocks.dart b/script/tool/test/mocks.dart index 3ce9512ad018..2c84ecedd12c 100644 --- a/script/tool/test/mocks.dart +++ b/script/tool/test/mocks.dart @@ -33,6 +33,9 @@ class MockPlatform extends Mock implements Platform { @override Map environment = {}; + + @override + String get pathSeparator => isWindows ? r'\' : '/'; } class MockProcess extends Mock implements io.Process {