diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart index f0ad8276b0465..436cb421424ee 100644 --- a/testing/scenario_app/bin/run_android_tests.dart +++ b/testing/scenario_app/bin/run_android_tests.dart @@ -47,48 +47,52 @@ void main(List args) async { ..addOption( 'adb', help: 'Path to the adb tool', - defaultsTo: engine != null ? join( - engine.srcDir.path, - 'third_party', - 'android_tools', - 'sdk', - 'platform-tools', - 'adb', - ) : null, + defaultsTo: engine != null + ? join( + engine.srcDir.path, + 'third_party', + 'android_tools', + 'sdk', + 'platform-tools', + 'adb', + ) + : null, ) ..addOption( 'ndk-stack', help: 'Path to the ndk-stack tool', - defaultsTo: engine != null ? join( - engine.srcDir.path, - 'third_party', - 'android_tools', - 'ndk', - 'prebuilt', - () { - if (Platform.isLinux) { - return 'linux-x86_64'; - } else if (Platform.isMacOS) { - return 'darwin-x86_64'; - } else if (Platform.isWindows) { - return 'windows-x86_64'; - } else { - throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}'); - } - }(), - 'bin', - 'ndk-stack', - ) : null, + defaultsTo: engine != null + ? join( + engine.srcDir.path, + 'third_party', + 'android_tools', + 'ndk', + 'prebuilt', + () { + if (Platform.isLinux) { + return 'linux-x86_64'; + } else if (Platform.isMacOS) { + return 'darwin-x86_64'; + } else if (Platform.isWindows) { + return 'windows-x86_64'; + } else { + throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}'); + } + }(), + 'bin', + 'ndk-stack', + ) + : null, ) ..addOption( 'out-dir', help: 'Out directory', - defaultsTo: - engine?. - outputs(). - where((Output o) => basename(o.path.path).startsWith('android_')). - firstOrNull?. - path.path, + defaultsTo: engine + ?.outputs() + .where((Output o) => basename(o.path.path).startsWith('android_')) + .firstOrNull + ?.path + .path, ) ..addOption( 'smoke-test', @@ -106,14 +110,16 @@ void main(List args) async { ..addOption( 'output-contents-golden', help: 'Path to a file that contains the expected filenames of golden files.', - defaultsTo: engine != null ? join( - engine.srcDir.path, - 'flutter', - 'testing', - 'scenario_app', - 'android', - 'expected_golden_output.txt', - ) : null, + defaultsTo: engine != null + ? join( + engine.srcDir.path, + 'flutter', + 'testing', + 'scenario_app', + 'android', + 'expected_golden_output.txt', + ) + : null, ) ..addOption( 'impeller-backend', @@ -124,8 +130,8 @@ void main(List args) async { ..addOption( 'logs-dir', help: 'The directory to store the logs and screenshots. Defaults to ' - 'the value of the FLUTTER_LOGS_DIR environment variable, if set, ' - 'otherwise it defaults to a path within out-dir.', + 'the value of the FLUTTER_LOGS_DIR environment variable, if set, ' + 'otherwise it defaults to a path within out-dir.', defaultsTo: Platform.environment['FLUTTER_LOGS_DIR'], ); @@ -153,7 +159,10 @@ void main(List args) async { final String? contentsGolden = results['output-contents-golden'] as String?; final _ImpellerBackend? impellerBackend = _ImpellerBackend.tryParse(results['impeller-backend'] as String?); if (enableImpeller && impellerBackend == null) { - panic(['invalid graphics-backend', results['impeller-backend'] as String? ?? '']); + panic([ + 'invalid graphics-backend', + results['impeller-backend'] as String? ?? '' + ]); } final Directory logsDir = Directory(results['logs-dir'] as String? ?? join(outDir.path, 'scenario_app', 'logs')); final String? ndkStack = results['ndk-stack'] as String?; @@ -215,7 +224,10 @@ Future _run({ const ProcessManager pm = LocalProcessManager(); if (!outDir.existsSync()) { - panic(['out-dir does not exist: $outDir', 'make sure to build the selected engine variant']); + panic([ + 'out-dir does not exist: $outDir', + 'make sure to build the selected engine variant' + ]); } if (!adb.existsSync()) { @@ -236,11 +248,17 @@ Future _run({ log('writing logs and screenshots to ${logsDir.path}'); if (!testApk.existsSync()) { - panic(['test apk does not exist: ${testApk.path}', 'make sure to build the selected engine variant']); + panic([ + 'test apk does not exist: ${testApk.path}', + 'make sure to build the selected engine variant' + ]); } if (!appApk.existsSync()) { - panic(['app apk does not exist: ${appApk.path}', 'make sure to build the selected engine variant']); + panic([ + 'app apk does not exist: ${appApk.path}', + 'make sure to build the selected engine variant' + ]); } // Start a TCP socket in the host, and forward it to the device that runs the tests. @@ -248,7 +266,7 @@ Future _run({ // for the screenshots. // On LUCI, the host uploads the screenshots to Skia Gold. SkiaGoldClient? skiaGoldClient; - late ServerSocket server; + late ServerSocket server; final List> pendingComparisons = >[]; await step('Starting server...', () async { server = await ServerSocket.bind(InternetAddress.anyIPv4, _tcpPort); @@ -259,7 +277,8 @@ Future _run({ if (verbose) { stdout.writeln('client connected ${client.remoteAddress.address}:${client.remotePort}'); } - client.transform(const ScreenshotBlobTransformer()).listen((Screenshot screenshot) { + client.transform(const ScreenshotBlobTransformer()).listen( + (Screenshot screenshot) { final String fileName = screenshot.filename; final Uint8List fileContent = screenshot.fileContent; if (verbose) { @@ -277,18 +296,15 @@ Future _run({ } if (isSkiaGoldClientAvailable) { final Future comparison = skiaGoldClient! - .addImg(fileName, goldenFile, - screenshotSize: screenshot.pixelCount) - .catchError((dynamic err) { - panic(['skia gold comparison failed: $err']); - }); + .addImg(fileName, goldenFile, screenshotSize: screenshot.pixelCount) + .catchError((dynamic err) { + panic(['skia gold comparison failed: $err']); + }); pendingComparisons.add(comparison); } - }, - onError: (dynamic err) { + }, onError: (dynamic err) { panic(['error while receiving bytes: $err']); - }, - cancelOnError: true); + }, cancelOnError: true); }); }); @@ -311,27 +327,38 @@ Future _run({ final (Future logcatExitCode, Stream logcatOutput) = getProcessStreams(logcatProcess); logcatProcessExitCode = logcatExitCode; + String? filterProcessId; + logcatOutput.listen((String line) { // Always write to the full log. logcat.writeln(line); // Conditionally parse and write to stderr. final AdbLogLine? adbLogLine = AdbLogLine.tryParse(line); - switch (adbLogLine?.process) { - case null: - break; - case 'ActivityManager': - // These are mostly noise, i.e. "D ActivityManager: freezing 24632 com.blah". - if (adbLogLine!.severity == 'D') { - break; - } - // TODO(matanlurey): Figure out why this isn't 'flutter.scenario' or similar. - // Also, why is there two different names? - case 'utter.scenario': - case 'utter.scenarios': - case 'flutter': - case 'FlutterJNI': - log('[adb] $line'); + if (verbose || adbLogLine == null) { + log(line); + return; + } + + // If we haven't already found a process ID, try to find one. + // The process ID will help us filter out logs from other processes. + filterProcessId ??= adbLogLine.tryParseProcess(); + + // If this is a "verbose" log, possibly skip it. + final bool isVerbose = adbLogLine.isVerbose(filterProcessId: filterProcessId); + if (isVerbose || filterProcessId == null) { + // We've requested verbose output, so print everything. + if (verbose) { + adbLogLine.printFormatted(); + } + return; + } + + // It's a non-verbose log, so print it. + adbLogLine.printFormatted(); + }, onError: (Object? err) { + if (verbose) { + logWarning('logcat stream error: $err'); } }); }); @@ -364,10 +391,7 @@ Future _run({ log('using dimensions: ${json.encode(dimensions)}'); skiaGoldClient = SkiaGoldClient( outDir, - dimensions: { - 'AndroidAPILevel': connectedDeviceAPILevel, - 'GraphicsBackend': enableImpeller ? 'impeller-${impellerBackend!.name}' : 'skia', - }, + dimensions: dimensions, ); }); @@ -412,11 +436,9 @@ Future _run({ 'am', 'instrument', '-w', - if (smokeTestFullPath != null) - '-e class $smokeTestFullPath', + if (smokeTestFullPath != null) '-e class $smokeTestFullPath', 'dev.flutter.scenarios.test/dev.flutter.TestRunner', - if (enableImpeller) - '-e enable-impeller', + if (enableImpeller) '-e enable-impeller', if (impellerBackend != null) '-e impeller-backend ${impellerBackend.name}', ]); @@ -465,7 +487,8 @@ Future _run({ final int exitCode = await pm.runAndForward([ adb.path, 'reverse', - '--remove', 'tcp:3000', + '--remove', + 'tcp:3000', ]); if (exitCode != 0) { panic(['could not unforward port']); @@ -473,14 +496,16 @@ Future _run({ }); await step('Uninstalling app APK...', () async { - final int exitCode = await pm.runAndForward([adb.path, 'uninstall', 'dev.flutter.scenarios']); + final int exitCode = await pm.runAndForward( + [adb.path, 'uninstall', 'dev.flutter.scenarios']); if (exitCode != 0) { panic(['could not uninstall app apk']); } }); await step('Uninstalling test APK...', () async { - final int exitCode = await pm.runAndForward([adb.path, 'uninstall', 'dev.flutter.scenarios.test']); + final int exitCode = await pm.runAndForward( + [adb.path, 'uninstall', 'dev.flutter.scenarios.test']); if (exitCode != 0) { panic(['could not uninstall app apk']); } diff --git a/testing/scenario_app/bin/utils/adb_logcat_filtering.dart b/testing/scenario_app/bin/utils/adb_logcat_filtering.dart index ca67ab17df46d..97852630c0a6a 100644 --- a/testing/scenario_app/bin/utils/adb_logcat_filtering.dart +++ b/testing/scenario_app/bin/utils/adb_logcat_filtering.dart @@ -1,3 +1,7 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + /// Some notes about filtering `adb logcat` output, especially as a result of /// running `adb shell` to instrument the app and test scripts, as it's /// non-trivial and error-prone. @@ -26,6 +30,8 @@ /// See also: . library; +import 'logs.dart'; + /// Represents a line of `adb logcat` output parsed into a structured form. /// /// For example the line: @@ -40,14 +46,15 @@ library; /// with lazy parsing. extension type const AdbLogLine._(Match _match) { // RegEx that parses into the following groups: - // 1. Everything up to the severity (I, W, E, etc.). - // In other words, any whitespace, numbers, hyphens, colons, and periods. - // 2. The severity (a single uppercase letter). - // 3. The name of the process (up to the colon). - // 4. The message (after the colon). + // 1. The time of the log message, such as `02-22 13:54:39.839`. + // 2. The process ID. + // 3. The thread ID. + // 4. The character representing the severity of the log message, such as `I`. + // 5. The tag, such as `ActivityManager`. + // 6. The actual log message. // // This regex is simple versus being more precise. Feel free to improve it. - static final RegExp _pattern = RegExp(r'([^A-Z]*)([A-Z])\s([^:]*)\:\s(.*)'); + static final RegExp _pattern = RegExp(r'(\d+-\d+\s[\d|:]+\.\d+)\s+(\d+)\s+(\d+)\s(\w)\s(\S+)\s*:\s*(.*)'); /// Parses the given [adbLogCatLine] into a structured form. /// @@ -57,15 +64,91 @@ extension type const AdbLogLine._(Match _match) { return match == null ? null : AdbLogLine._(match); } + /// Tries to parse the process that was started, if the log line is about it. + String? tryParseProcess() { + if (name == 'ActivityManager' && message.startsWith('Start proc')) { + // Start proc 6840:d + final RegExpMatch? match = RegExp(r'Start proc (\d+):').firstMatch(message); + return match?.group(1); + } + return null; + } + + /// Returns `true` if the log line is verbose. + bool isVerbose({String? filterProcessId}) => !_isRelevant(filterProcessId: filterProcessId); + bool _isRelevant({String? filterProcessId}) { + // Fatal errors are always useful. + if (severity == 'F') { + return true; + } + + // Debug logs are rarely useful. + if (severity == 'D') { + return false; + } + + // These are "known" noise tags. + if (const { + 'MonitoringInstr', + 'ResourceExtractor', + 'THREAD_STATE', + 'ziparchive', + }.contains(name)) { + return false; + } + + // These are "known" tags useful for debugging. + if (const { + 'utter.scenario', + 'utter.scenarios', + 'TestRunner', + }.contains(name)) { + return true; + } + + // If a process ID is specified, exclude logs _not_ from that process. + if (filterProcessId != null && process != filterProcessId) { + return false; + } + + // And... whatever, include anything with the word "flutter". + return name.toLowerCase().contains('flutter') || message.toLowerCase().contains('flutter'); + } + + /// Logs the line to the console. + void printFormatted() { + final String formatted = '$time [$severity] $name: $message'; + if (severity == 'W' || severity == 'E' || severity == 'F') { + logWarning(formatted); + } else if (name == 'TestRunner') { + logImportant(formatted); + } else { + log(formatted); + } + } + /// The full line of `adb logcat` output. String get line => _match.group(0)!; + /// The time of the log message, such as `02-22 13:54:39.839`. + String get time => _match.group(1)!; + + /// The process ID. + String get process => _match.group(2)!; + + /// The thread ID. + String get thread => _match.group(3)!; + /// The character representing the severity of the log message, such as `I`. - String get severity => _match.group(2)!; + String get severity => _match.group(4)!; - /// The process name, such as `ActivityManager`. - String get process => _match.group(3)!; + /// The tag, such as `ActivityManager`. + String get name => _match.group(5)!; /// The actual log message. - String get message => _match.group(4)!; + String get message => _match.group(6)!; + + String toDebugString() { + return 'AdbLogLine(time: $time, process: $process, thread: $thread, severity: $severity, name: $name, message: $message)'; + } } diff --git a/testing/scenario_app/bin/utils/logs.dart b/testing/scenario_app/bin/utils/logs.dart index b84e8127f29fe..4d883d340e261 100644 --- a/testing/scenario_app/bin/utils/logs.dart +++ b/testing/scenario_app/bin/utils/logs.dart @@ -7,6 +7,7 @@ import 'dart:io'; bool _supportsAnsi = stdout.supportsAnsiEscapes; String _green = _supportsAnsi ? '\u001b[1;32m' : ''; String _red = _supportsAnsi ? '\u001b[31m' : ''; +String _yellow = _supportsAnsi ? '\u001b[33m' : ''; String _gray = _supportsAnsi ? '\u001b[90m' : ''; String _reset = _supportsAnsi? '\u001B[0m' : ''; @@ -22,15 +23,27 @@ Future step(String msg, Future Function() fn) async { } } +void _logWithColor(String color, String msg) { + stdout.writeln('$color$msg$_reset'); +} + void log(String msg) { - stdout.writeln('$_gray$msg$_reset'); + _logWithColor(_gray, msg); +} + +void logImportant(String msg) { + stdout.writeln(msg); +} + +void logWarning(String msg) { + _logWithColor(_yellow, msg); } final class Panic extends Error {} Never panic(List messages) { for (final String message in messages) { - stderr.writeln('$_red$message$_reset'); + _logWithColor(_red, message); } throw Panic(); } diff --git a/testing/scenario_app/tool/logcat_reader.dart b/testing/scenario_app/tool/logcat_reader.dart new file mode 100644 index 0000000000000..909204a1af342 --- /dev/null +++ b/testing/scenario_app/tool/logcat_reader.dart @@ -0,0 +1,43 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +// It's bad to import a file from `bin` into `tool`. +// However this tool is not very important, so delete it if necessary. +import '../bin/utils/adb_logcat_filtering.dart'; + +/// A tiny tool to read saved `adb logcat` output and perform some analysis. +/// +/// This tool is not meant to be a full-fledged logcat reader. It's just a +/// simple tool that uses the [AdbLogLine] extension type to parse results of +/// `adb logcat` and explain what log tag names are most common. +void main(List args) { + if (args case [final String path]) { + final List parsed = io.File(path) + .readAsLinesSync() + .map(AdbLogLine.tryParse) + .whereType() + // Filter out all debug logs. + .where((AdbLogLine line) => line.severity != 'D') + .toList(); + + final Map tagCounts = {}; + for (final AdbLogLine line in parsed) { + tagCounts[line.name] = (tagCounts[line.name] ?? 0) + 1; + } + + // Print in order of most common to least common. + final List> sorted = tagCounts.entries.toList() + ..sort((MapEntry a, MapEntry b) => b.value.compareTo(a.value)); + for (final MapEntry entry in sorted) { + print("'${entry.key}', // ${entry.value}"); + } + + return; + } + + print('Usage: logcat_reader.dart '); + io.exitCode = 1; +}