diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml new file mode 100644 index 0000000000..f87bf2734c --- /dev/null +++ b/.github/workflows/metrics.yml @@ -0,0 +1,81 @@ +name: SDK metrics + +on: + push: + branches: [main] + pull_request: + paths: + - .github/workflows/metrics.yml + - dart/** + - flutter/** + - metrics/** + +jobs: + metrics: + name: ${{ matrix.name }} + runs-on: ${{ matrix.host }} + strategy: + # We want that the matrix keeps running, default is to cancel all jobs if a single fails. + fail-fast: false + matrix: + include: + - platform: ios + name: iOS + appPlain: test-app-plain.ipa + host: macos-latest + - platform: android + name: Android + appPlain: metrics/perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk + host: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + # Let's stick to an explicit version and update manually because a version change may affect results. + # If it would update implicitly it could confuse people to think the change is actually caused by the PR. + # Instead, we use Updater (update-deps.yml) to create PRs for explicit Flutter SDK update. + - name: Read configured Flutter SDK version + id: conf + run: | + version=$(grep "version" metrics/flutter.properties | cut -d'=' -f2 | xargs) + echo "::set-output name=flutter::$version" + + - name: Install Flutter v${{ steps.conf.outputs.flutter }} + uses: subosito/flutter-action@1e6ee87cb840500837bcd50a667fb28815d8e310 # pin@v2 + with: + flutter-version: ${{ steps.conf.outputs.flutter }} + + - uses: actions/setup-java@v3 + if: ${{ matrix.platform == 'android' }} + with: + java-version: '11' + distribution: 'adopt' + + - run: ./metrics/prepare.sh + + - uses: actions/cache@v3 + id: app-plain-cache + with: + path: ${{ matrix.appPlain }} + key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ matrix.platform }}-${{ hashFiles('metrics/perf-test-app-plain/pubspec.yaml') }} + + - name: Build + run: ./metrics/build.sh ${{ matrix.platform }} + env: + # Necessary to build an iOS .ipa (using fastlane) + APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} + APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} + APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} + FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} + MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} + MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} + MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} + APP_PLAIN: ${{ matrix.appPlain }} + + - name: Collect apps metrics + uses: getsentry/action-app-sdk-overhead-metrics@v1 + with: + name: ${{ matrix.name }} + config: ./metrics/metrics-${{ matrix.platform }}.yml + sauce-user: ${{ secrets.SAUCE_USERNAME }} + sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} diff --git a/.github/workflows/update-deps.yml b/.github/workflows/update-deps.yml index 4973480c69..408dd27125 100644 --- a/.github/workflows/update-deps.yml +++ b/.github/workflows/update-deps.yml @@ -25,3 +25,11 @@ jobs: name: Cocoa SDK secrets: api-token: ${{ secrets.CI_DEPLOY_KEY }} + + metrics-flutter: + uses: getsentry/github-workflows/.github/workflows/updater.yml@v2 + with: + path: metrics/flutter.properties + name: Flutter SDK (metrics) + secrets: + api-token: ${{ secrets.CI_DEPLOY_KEY }} diff --git a/metrics/.gitignore b/metrics/.gitignore new file mode 100644 index 0000000000..f69bea8ddc --- /dev/null +++ b/metrics/.gitignore @@ -0,0 +1 @@ +perf-test-app* diff --git a/metrics/build.sh b/metrics/build.sh new file mode 100755 index 0000000000..017619c7b8 --- /dev/null +++ b/metrics/build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +targetDir=$( + cd $(dirname $0) + pwd +) +[[ "$targetDir" != "" ]] || exit 1 + +if [ "$#" -gt 2 ]; then + shift + args="$@" + target=$1 +else + # Use default args if no were given + case "$1" in + ios) + target="ipa" + args="--release --no-codesign" + ;; + + android) + target="apk" + args="--release --target-platform android-arm64 --split-per-abi" + ;; + + *) + echo "Unknown platform: '$1'" + exit 1 + ;; + esac +fi + +for dir in "$targetDir/perf-test-app-"*; do + app="$(basename "$dir")" + echo "Building $app" + + # Support caching if this is running in CI. + if [[ -n ${CI+x} && $app == *-plain && -f $APP_PLAIN ]]; then + echo "Cached app exists: $APP_PLAIN - skipping build" + continue + fi + + ( + echo "::group::Flutter build $app" + cd $dir + flutter pub get + if [[ "$1" == "ios" && -f "ios/Podfile" ]]; then + cd ios + pod install --repo-update + cd - + fi + flutter build $target $args + echo '::endgroup::' + ) + + if [[ "$1" == "ios" ]]; then + flJob=${app//-/_} + ( + echo "::group::Fastlane build $app" + cd $targetDir + fastlane "build_$flJob" --verbose + echo '::endgroup::' + ) + fi +done diff --git a/metrics/fastlane/.gitignore b/metrics/fastlane/.gitignore new file mode 100644 index 0000000000..f6c97a67c4 --- /dev/null +++ b/metrics/fastlane/.gitignore @@ -0,0 +1,5 @@ +test_output/ +README.md +report.xml +.env +cobertura.xml diff --git a/metrics/fastlane/Appfile b/metrics/fastlane/Appfile new file mode 100644 index 0000000000..73db52a804 --- /dev/null +++ b/metrics/fastlane/Appfile @@ -0,0 +1,6 @@ +itc_team_id("96157806") # App Store Connect Team ID +team_id("97JCY7859U") # Developer Portal Team ID + +# For more information about the Appfile, see: +# https://docs.fastlane.tools/advanced/#appfile + diff --git a/metrics/fastlane/Fastfile b/metrics/fastlane/Fastfile new file mode 100644 index 0000000000..389468c5f1 --- /dev/null +++ b/metrics/fastlane/Fastfile @@ -0,0 +1,54 @@ +default_platform(:ios) + +platform :ios do + desc "Build perf-test app without Sentry" + lane :build_perf_test_app_plain do + setup_ci + + sync_code_signing( + type: "development", + readonly: true, + app_identifier: ["io.sentry.dart.perfTestAppPlain"] + ) + + build_app( + workspace: "perf-test-app-plain/ios/Runner.xcworkspace", + scheme: "Runner", + include_bitcode: false, + include_symbols: false, + export_method: "development", + export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id), + output_name: "test-app-plain.ipa", + skip_build_archive: true, + archive_path: "perf-test-app-plain/build/ios/archive/Runner.xcarchive", + ) + + delete_keychain(name: "fastlane_tmp_keychain") unless is_ci + end + + desc "Build perf-test app with Sentry" + lane :build_perf_test_app_with_sentry do + setup_ci + + sync_code_signing( + type: "development", + readonly: true, + app_identifier: ["io.sentry.dart.perfTestAppWithSentry"] + ) + + build_app( + workspace: "perf-test-app-with-sentry/ios/Runner.xcworkspace", + scheme: "Runner", + include_bitcode: false, + include_symbols: false, + export_method: "development", + export_team_id: CredentialsManager::AppfileConfig.try_fetch_value(:team_id), + output_name: "test-app-sentry.ipa", + skip_build_archive: true, + archive_path: "perf-test-app-with-sentry/build/ios/archive/Runner.xcarchive", + ) + + delete_keychain(name: "fastlane_tmp_keychain") unless is_ci + end + +end diff --git a/metrics/fastlane/Matchfile b/metrics/fastlane/Matchfile new file mode 100644 index 0000000000..054fef1c37 --- /dev/null +++ b/metrics/fastlane/Matchfile @@ -0,0 +1,5 @@ +git_url("git@github.com:getsentry/codesigning.git") +storage_mode("git") +username("bot@getsentry.com") # Your Apple Developer Portal username + +# The docs are available on https://docs.fastlane.tools/actions/match diff --git a/metrics/fastlane/Pluginfile b/metrics/fastlane/Pluginfile new file mode 100644 index 0000000000..a152560f42 --- /dev/null +++ b/metrics/fastlane/Pluginfile @@ -0,0 +1,5 @@ +# Autogenerated by fastlane +# +# Ensure this file is checked in to source control! + +gem 'fastlane-plugin-sentry' diff --git a/metrics/flutter.properties b/metrics/flutter.properties new file mode 100644 index 0000000000..5f3755f0a2 --- /dev/null +++ b/metrics/flutter.properties @@ -0,0 +1,2 @@ +version = 3.3.3 +repo = https://github.com/flutter/flutter diff --git a/metrics/metrics-android.yml b/metrics/metrics-android.yml new file mode 100644 index 0000000000..931613a494 --- /dev/null +++ b/metrics/metrics-android.yml @@ -0,0 +1,16 @@ +apps: + - name: io.sentry.dart.perf_test_app_plain + activity: MainActivity + path: perf-test-app-plain/build/app/outputs/apk/release/app-arm64-v8a-release.apk + - name: io.sentry.dart.perf_test_app_with_sentry + activity: MainActivity + path: perf-test-app-with-sentry/build/app/outputs/apk/release/app-arm64-v8a-release.apk + +startupTimeTest: + runs: 50 + diffMin: 0 + diffMax: 150 + +binarySizeTest: + diffMin: 900 KiB + diffMax: 1100 KiB diff --git a/metrics/metrics-ios.yml b/metrics/metrics-ios.yml new file mode 100644 index 0000000000..4cc7e147a1 --- /dev/null +++ b/metrics/metrics-ios.yml @@ -0,0 +1,14 @@ +apps: + - name: io.sentry.dart.perfTestAppPlain + path: test-app-plain.ipa + - name: io.sentry.dart.perfTestAppWithSentry + path: test-app-sentry.ipa + +startupTimeTest: + runs: 50 + diffMin: 0 + diffMax: 150 + +binarySizeTest: + diffMin: 900 KiB + diffMax: 1100 KiB diff --git a/metrics/prepare.sh b/metrics/prepare.sh new file mode 100755 index 0000000000..edb87bcae6 --- /dev/null +++ b/metrics/prepare.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +targetDir=$( + cd $(dirname $0) + pwd +) +[[ "$targetDir" != "" ]] || exit 1 + +flutterCreate() { + name=${1//-/_} + dir=$targetDir/$1 + rm -rf $dir + echo "::group::Flutter create $1" + flutter create --template=app --no-pub --org 'io.sentry.dart' --project-name $name "$dir" + echo '::endgroup::' +} + +flutterCreate 'perf-test-app-plain' +flutterCreate 'perf-test-app-with-sentry' + +echo '::group::Patch perf-test-app-with-sentry' +pubspec="$targetDir/perf-test-app-with-sentry/pubspec.yaml" +echo "Adding dependencies to $pubspec" +cat <>"$pubspec" + +dependency_overrides: + sentry: + path: ../../dart + sentry_flutter: + path: ../../flutter + +EOF + +patch -p0 "$targetDir/perf-test-app-with-sentry/lib/main.dart" <<'EOF' +@@ -1,7 +1,12 @@ + import 'package:flutter/material.dart'; ++import 'package:sentry_flutter/sentry_flutter.dart'; + +-void main() { +- runApp(const MyApp()); ++Future main() async { ++ await SentryFlutter.init( ++ (options) => options.dsn = ++ 'https://e85b375ffb9f43cf8bdf9787768149e0@o447951.ingest.sentry.io/5428562', ++ appRunner: () => runApp(const MyApp()), ++ ); + } + + class MyApp extends StatelessWidget { +EOF +echo '::endgroup::'