From 9e87fe8ce0ff8d95c761fa1ec2fb110b98816048 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 19 Mar 2024 15:43:22 +0100 Subject: [PATCH 01/19] Add replay integration --- .../api/sentry-android-core.api | 4 + sentry-android-core/build.gradle.kts | 1 + .../core/AndroidOptionsInitializer.java | 8 +- .../sentry/android/core/LifecycleWatcher.java | 62 ++-- .../io/sentry/android/core/SentryAndroid.java | 80 ++++- .../api/sentry-android-replay.api | 28 +- sentry-android-replay/build.gradle.kts | 4 + .../io/sentry/android/replay/ReplayCache.kt | 171 ++++++++++ .../android/replay/ReplayIntegration.kt | 313 ++++++++++++++++++ .../android/replay/ScreenshotRecorder.kt | 145 ++++---- .../sentry/android/replay/WindowRecorder.kt | 83 ++--- .../java/io/sentry/android/replay/Windows.kt | 10 +- .../replay/video/SimpleMp4FrameMuxer.kt | 7 +- .../replay/video/SimpleVideoEncoder.kt | 143 ++++---- .../sentry/android/replay/ReplayCacheTest.kt | 86 +++++ sentry-android/build.gradle.kts | 1 + sentry/api/sentry.api | 10 +- sentry/src/main/java/io/sentry/IScope.java | 18 + sentry/src/main/java/io/sentry/NoOpScope.java | 9 + sentry/src/main/java/io/sentry/Scope.java | 17 + .../java/io/sentry/SentryEnvelopeItem.java | 5 +- .../java/io/sentry/rrweb/RRWebVideoEvent.java | 8 +- 22 files changed, 1005 insertions(+), 208 deletions(-) create mode 100644 sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt create mode 100644 sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt create mode 100644 sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 51eb48f1b2..55278c6356 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -245,6 +245,10 @@ public final class io/sentry/android/core/SentryAndroid { public static fun init (Landroid/content/Context;Lio/sentry/ILogger;)V public static fun init (Landroid/content/Context;Lio/sentry/ILogger;Lio/sentry/Sentry$OptionsConfiguration;)V public static fun init (Landroid/content/Context;Lio/sentry/Sentry$OptionsConfiguration;)V + public static fun pauseReplay ()V + public static fun resumeReplay ()V + public static fun startReplay ()V + public static fun stopReplay ()V } public final class io/sentry/android/core/SentryAndroidDateProvider : io/sentry/SentryDateProvider { diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 4ab0aec423..da4851c92a 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -76,6 +76,7 @@ dependencies { api(projects.sentry) compileOnly(projects.sentryAndroidFragment) compileOnly(projects.sentryAndroidTimber) + compileOnly(projects.sentryAndroidReplay) compileOnly(projects.sentryCompose) // lifecycle processor, session tracking diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java index 41d0dec6b2..b58051cee7 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java @@ -22,6 +22,7 @@ import io.sentry.android.core.internal.util.SentryFrameMetricsCollector; import io.sentry.android.core.performance.AppStartMetrics; import io.sentry.android.fragment.FragmentLifecycleIntegration; +import io.sentry.android.replay.ReplayIntegration; import io.sentry.android.timber.SentryTimberIntegration; import io.sentry.cache.PersistingOptionsObserver; import io.sentry.cache.PersistingScopeObserver; @@ -29,6 +30,7 @@ import io.sentry.compose.viewhierarchy.ComposeViewHierarchyExporter; import io.sentry.internal.gestures.GestureTargetLocator; import io.sentry.internal.viewhierarchy.ViewHierarchyExporter; +import io.sentry.transport.CurrentDateProvider; import io.sentry.transport.NoOpEnvelopeCache; import io.sentry.util.LazyEvaluator; import io.sentry.util.Objects; @@ -230,7 +232,8 @@ static void installDefaultIntegrations( final @NotNull LoadClass loadClass, final @NotNull ActivityFramesTracker activityFramesTracker, final boolean isFragmentAvailable, - final boolean isTimberAvailable) { + final boolean isTimberAvailable, + final boolean isReplayAvailable) { // Integration MUST NOT cache option values in ctor, as they will be configured later by the // user @@ -295,6 +298,9 @@ static void installDefaultIntegrations( new NetworkBreadcrumbsIntegration(context, buildInfoProvider, options.getLogger())); options.addIntegration(new TempSensorBreadcrumbsIntegration(context)); options.addIntegration(new PhoneStateBreadcrumbsIntegration(context)); + if (isReplayAvailable) { + options.addIntegration(new ReplayIntegration(context, CurrentDateProvider.getInstance())); + } } /** diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java b/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java index 7b38bcd9c2..ff281d2beb 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java @@ -11,6 +11,7 @@ import io.sentry.transport.ICurrentDateProvider; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -19,11 +20,12 @@ final class LifecycleWatcher implements DefaultLifecycleObserver { private final AtomicLong lastUpdatedSession = new AtomicLong(0L); + private final AtomicBoolean isFreshSession = new AtomicBoolean(false); private final long sessionIntervalMillis; private @Nullable TimerTask timerTask; - private final @Nullable Timer timer; + private final @NotNull Timer timer = new Timer(true); private final @NotNull Object timerLock = new Object(); private final @NotNull IHub hub; private final boolean enableSessionTracking; @@ -55,11 +57,6 @@ final class LifecycleWatcher implements DefaultLifecycleObserver { this.enableAppLifecycleBreadcrumbs = enableAppLifecycleBreadcrumbs; this.hub = hub; this.currentDateProvider = currentDateProvider; - if (enableSessionTracking) { - timer = new Timer(true); - } else { - timer = null; - } } // App goes to foreground @@ -74,41 +71,45 @@ public void onStart(final @NotNull LifecycleOwner owner) { } private void startSession() { - if (enableSessionTracking) { - cancelTask(); + cancelTask(); - final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); + final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); - hub.configureScope( - scope -> { - if (lastUpdatedSession.get() == 0L) { - final @Nullable Session currentSession = scope.getSession(); - if (currentSession != null && currentSession.getStarted() != null) { - lastUpdatedSession.set(currentSession.getStarted().getTime()); - } + hub.configureScope( + scope -> { + if (lastUpdatedSession.get() == 0L) { + final @Nullable Session currentSession = scope.getSession(); + if (currentSession != null && currentSession.getStarted() != null) { + lastUpdatedSession.set(currentSession.getStarted().getTime()); + isFreshSession.set(true); } - }); + } + }); - final long lastUpdatedSession = this.lastUpdatedSession.get(); - if (lastUpdatedSession == 0L - || (lastUpdatedSession + sessionIntervalMillis) <= currentTimeMillis) { + final long lastUpdatedSession = this.lastUpdatedSession.get(); + if (lastUpdatedSession == 0L + || (lastUpdatedSession + sessionIntervalMillis) <= currentTimeMillis) { + if (enableSessionTracking) { addSessionBreadcrumb("start"); hub.startSession(); } - this.lastUpdatedSession.set(currentTimeMillis); + SentryAndroid.startReplay(); + } else if (!isFreshSession.getAndSet(false)) { + // only resume if it's not a fresh session, which has been started in SentryAndroid.init + SentryAndroid.resumeReplay(); } + this.lastUpdatedSession.set(currentTimeMillis); } // App went to background and triggered this callback after 700ms // as no new screen was shown @Override public void onStop(final @NotNull LifecycleOwner owner) { - if (enableSessionTracking) { - final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); - this.lastUpdatedSession.set(currentTimeMillis); + final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); + this.lastUpdatedSession.set(currentTimeMillis); - scheduleEndSession(); - } + SentryAndroid.pauseReplay(); + scheduleEndSession(); AppState.getInstance().setInBackground(true); addAppBreadcrumb("background"); @@ -122,8 +123,11 @@ private void scheduleEndSession() { new TimerTask() { @Override public void run() { - addSessionBreadcrumb("end"); - hub.endSession(); + if (enableSessionTracking) { + addSessionBreadcrumb("end"); + hub.endSession(); + } + SentryAndroid.stopReplay(); } }; @@ -164,7 +168,7 @@ TimerTask getTimerTask() { } @TestOnly - @Nullable + @NotNull Timer getTimer() { return timer; } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java index af68a026fb..39a2019d1d 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java @@ -15,6 +15,8 @@ import io.sentry.android.core.performance.AppStartMetrics; import io.sentry.android.core.performance.TimeSpan; import io.sentry.android.fragment.FragmentLifecycleIntegration; +import io.sentry.android.replay.ReplayIntegration; +import io.sentry.android.replay.ReplayIntegrationKt; import io.sentry.android.timber.SentryTimberIntegration; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; @@ -33,6 +35,11 @@ public final class SentryAndroid { static final String SENTRY_TIMBER_INTEGRATION_CLASS_NAME = "io.sentry.android.timber.SentryTimberIntegration"; + static final String SENTRY_REPLAY_INTEGRATION_CLASS_NAME = + "io.sentry.android.replay.ReplayIntegration"; + + private static boolean isReplayAvailable = false; + private static final String TIMBER_CLASS_NAME = "timber.log.Timber"; private static final String FRAGMENT_CLASS_NAME = "androidx.fragment.app.FragmentManager$FragmentLifecycleCallbacks"; @@ -99,6 +106,8 @@ public static synchronized void init( final boolean isTimberAvailable = (isTimberUpstreamAvailable && classLoader.isClassAvailable(SENTRY_TIMBER_INTEGRATION_CLASS_NAME, options)); + isReplayAvailable = + classLoader.isClassAvailable(SENTRY_REPLAY_INTEGRATION_CLASS_NAME, options); final BuildInfoProvider buildInfoProvider = new BuildInfoProvider(logger); final LoadClass loadClass = new LoadClass(); @@ -118,7 +127,8 @@ public static synchronized void init( loadClass, activityFramesTracker, isFragmentAvailable, - isTimberAvailable); + isTimberAvailable, + isReplayAvailable); configuration.configure(options); @@ -145,9 +155,12 @@ public static synchronized void init( true); final @NotNull IHub hub = Sentry.getCurrentHub(); - if (hub.getOptions().isEnableAutoSessionTracking() && ContextUtils.isForegroundImportance()) { - hub.addBreadcrumb(BreadcrumbFactory.forSession("session.start")); - hub.startSession(); + if (ContextUtils.isForegroundImportance()) { + if (hub.getOptions().isEnableAutoSessionTracking()) { + hub.addBreadcrumb(BreadcrumbFactory.forSession("session.start")); + hub.startSession(); + } + startReplay(); } } catch (IllegalAccessException e) { logger.log(SentryLevel.FATAL, "Fatal error during SentryAndroid.init(...)", e); @@ -212,4 +225,63 @@ private static void deduplicateIntegrations( } } } + + public static synchronized void startReplay() { + performReplayAction( + "starting", + (replay) -> { + replay.start(); + }); + } + + public static synchronized void stopReplay() { + performReplayAction( + "stopping", + (replay) -> { + replay.stop(); + }); + } + + public static synchronized void resumeReplay() { + performReplayAction( + "resuming", + (replay) -> { + replay.resume(); + }); + } + + public static synchronized void pauseReplay() { + performReplayAction( + "pausing", + (replay) -> { + replay.pause(); + }); + } + + private static void performReplayAction( + final @NotNull String actionName, final @NotNull ReplayCallable action) { + final @NotNull IHub hub = Sentry.getCurrentHub(); + if (isReplayAvailable) { + final ReplayIntegration replay = ReplayIntegrationKt.getReplayIntegration(hub); + if (replay != null) { + action.call(replay); + } else { + hub.getOptions() + .getLogger() + .log( + SentryLevel.INFO, + "Session Replay wasn't registered yet, not " + actionName + " the replay"); + } + } else { + hub.getOptions() + .getLogger() + .log( + SentryLevel.INFO, + "Session Replay wasn't found on classpath, not " + actionName + " the replay"); + } + } + + private interface ReplayCallable { + void call(final @NotNull ReplayIntegration replay); + } } diff --git a/sentry-android-replay/api/sentry-android-replay.api b/sentry-android-replay/api/sentry-android-replay.api index e81c5840ea..5af2ca943b 100644 --- a/sentry-android-replay/api/sentry-android-replay.api +++ b/sentry-android-replay/api/sentry-android-replay.api @@ -6,10 +6,30 @@ public final class io/sentry/android/replay/BuildConfig { public fun ()V } -public final class io/sentry/android/replay/WindowRecorder { - public fun ()V - public final fun startRecording (Landroid/content/Context;)V - public final fun stopRecording ()V +public final class io/sentry/android/replay/ReplayIntegration : io/sentry/Integration, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable { + public static final field Companion Lio/sentry/android/replay/ReplayIntegration$Companion; + public static final field VIDEO_BUFFER_DURATION J + public static final field VIDEO_SEGMENT_DURATION J + public fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V + public fun close ()V + public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V + public final fun pause ()V + public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V + public final fun resume ()V + public final fun start ()V + public final fun stop ()V +} + +public final class io/sentry/android/replay/ReplayIntegration$Companion { +} + +public final class io/sentry/android/replay/ReplayIntegrationKt { + public static final fun getReplayIntegration (Lio/sentry/IHub;)Lio/sentry/android/replay/ReplayIntegration; + public static final fun gracefullyShutdown (Ljava/util/concurrent/ExecutorService;Lio/sentry/SentryOptions;)V +} + +public abstract interface class io/sentry/android/replay/ScreenshotRecorderCallback { + public abstract fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V } public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer { diff --git a/sentry-android-replay/build.gradle.kts b/sentry-android-replay/build.gradle.kts index 2314960f5e..319386ee2b 100644 --- a/sentry-android-replay/build.gradle.kts +++ b/sentry-android-replay/build.gradle.kts @@ -19,6 +19,8 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersionReplay + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner + // for AGP 4.1 buildConfigField("String", "VERSION_NAME", "\"${project.version}\"") } @@ -67,7 +69,9 @@ dependencies { // tests testImplementation(projects.sentryTestSupport) + testImplementation(Config.TestLibs.robolectric) testImplementation(Config.TestLibs.kotlinTestJunit) + testImplementation(Config.TestLibs.androidxRunner) testImplementation(Config.TestLibs.androidxJunit) testImplementation(Config.TestLibs.mockitoKotlin) testImplementation(Config.TestLibs.mockitoInline) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt new file mode 100644 index 0000000000..cf6d6a93aa --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -0,0 +1,171 @@ +package io.sentry.android.replay + +import android.graphics.Bitmap +import android.graphics.Bitmap.CompressFormat.JPEG +import android.graphics.BitmapFactory +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.ERROR +import io.sentry.SentryLevel.WARNING +import io.sentry.SentryOptions +import io.sentry.android.replay.video.MuxerConfig +import io.sentry.android.replay.video.SimpleVideoEncoder +import io.sentry.protocol.SentryId +import io.sentry.util.FileUtils +import java.io.Closeable +import java.io.File + +internal class ReplayCache( + private val options: SentryOptions, + private val replayId: SentryId, + private val recorderConfig: ScreenshotRecorderConfig +) : Closeable { + + private val encoderLock = Any() + private var encoder: SimpleVideoEncoder? = null + + private val replayCacheDir: File? by lazy { + if (options.cacheDirPath.isNullOrEmpty()) { + options.logger.log( + WARNING, + "SentryOptions.cacheDirPath is not set, session replay is no-op" + ) + null + } else { + File(options.cacheDirPath!!, "replay_$replayId").also { it.mkdirs() } + } + } + + // TODO: maybe account for multi-threaded access + private val frames = mutableListOf() + + fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { + if (replayCacheDir == null) { + return + } + + val screenshot = File(replayCacheDir, "$frameTimestamp.jpg").also { + it.createNewFile() + } + screenshot.outputStream().use { + bitmap.compress(JPEG, 80, it) + it.flush() + } + + val frame = ReplayFrame(screenshot, frameTimestamp) + frames += frame + } + + fun createVideoOf(duration: Long, from: Long, segmentId: Int): GeneratedVideo? { + if (frames.isEmpty()) { + options.logger.log( + DEBUG, + "No captured frames, skipping generating a video segment" + ) + return null + } + + // TODO: reuse instance of encoder and just change file path to create a different muxer + val videoFile = File(replayCacheDir, "$segmentId.mp4") + encoder = synchronized(encoderLock) { + SimpleVideoEncoder( + options, + MuxerConfig( + file = videoFile, + recorderConfig = recorderConfig, + frameRate = recorderConfig.frameRate.toFloat(), + bitrate = 20 * 1000 + ) + ) + }.also { it.start() } + + val step = 1000 / recorderConfig.frameRate.toLong() + var frameCount = 0 + var lastFrame: ReplayFrame = frames.first() + for (timestamp in from until (from + (duration)) step step) { + val iter = frames.iterator() + val frameCountBefore = frameCount + while (iter.hasNext()) { + val frame = iter.next() + if (frame.timestamp in (timestamp..timestamp + step)) { + frameCount++ + encode(frame) + lastFrame = frame + break // we only support 1 frame per given interval + } + } + + // if the frame count hasn't changed we just replicate the last known frame to respect + // the video duration. + if (frameCountBefore == frameCount) { + frameCount++ + encode(lastFrame) + } + } + + if (frameCount == 0) { + options.logger.log( + DEBUG, + "Generated a video with no frames, not capturing a replay segment" + ) + deleteFile(videoFile) + return null + } + + var videoDuration: Long + synchronized(encoderLock) { + encoder?.release() + videoDuration = encoder?.duration ?: 0 + encoder = null + } + + frames.removeAll { + if (it.timestamp < (from + duration)) { + deleteFile(it.screenshot) + return@removeAll true + } + return@removeAll false + } + + return GeneratedVideo(videoFile, frameCount, videoDuration) + } + + private fun encode(frame: ReplayFrame) { + val bitmap = BitmapFactory.decodeFile(frame.screenshot.absolutePath) + synchronized(encoderLock) { + encoder?.encode(bitmap) + } + bitmap.recycle() + } + + private fun deleteFile(file: File) { + try { + if (!file.delete()) { + options.logger.log(ERROR, "Failed to delete replay frame: %s", file.absolutePath) + } + } catch (e: Throwable) { + options.logger.log(ERROR, e, "Failed to delete replay frame: %s", file.absolutePath) + } + } + + fun cleanup() { + FileUtils.deleteRecursively(replayCacheDir) + } + + override fun close() { + synchronized(encoderLock) { + encoder?.release() + encoder = null + } + } +} + +internal data class ReplayFrame( + val screenshot: File, + val timestamp: Long +) + +internal data class GeneratedVideo( + val video: File, + val frameCount: Int, + val duration: Long +) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt new file mode 100644 index 0000000000..26b35f9a15 --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -0,0 +1,313 @@ +package io.sentry.android.replay + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Point +import android.graphics.Rect +import android.os.Build +import android.os.Build.VERSION +import android.os.Build.VERSION_CODES +import android.view.WindowManager +import io.sentry.DateUtils +import io.sentry.Hint +import io.sentry.IHub +import io.sentry.Integration +import io.sentry.ReplayRecording +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.INFO +import io.sentry.SentryOptions +import io.sentry.SentryReplayEvent +import io.sentry.protocol.SentryId +import io.sentry.rrweb.RRWebMetaEvent +import io.sentry.rrweb.RRWebVideoEvent +import io.sentry.transport.ICurrentDateProvider +import java.io.Closeable +import java.io.File +import java.util.Date +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.ThreadFactory +import java.util.concurrent.TimeUnit.MILLISECONDS +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import kotlin.LazyThreadSafetyMode.NONE +import kotlin.math.roundToInt + +class ReplayIntegration( + private val context: Context, + private val dateProvider: ICurrentDateProvider +) : Integration, Closeable, ScreenshotRecorderCallback { + + companion object { + const val VIDEO_SEGMENT_DURATION = 5_000L + const val VIDEO_BUFFER_DURATION = 30_000L + } + + private lateinit var options: SentryOptions + private var hub: IHub? = null + private var recorder: WindowRecorder? = null + private var cache: ReplayCache? = null + + // TODO: probably not everything has to be thread-safe here + private val isEnabled = AtomicBoolean(false) + private val isRecording = AtomicBoolean(false) + private val currentReplayId = AtomicReference() + private val segmentTimestamp = AtomicReference() + private val currentSegment = AtomicInteger(0) + private val saver = + Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory()) + + private val screenBounds by lazy(NONE) { + // PixelCopy takes screenshots including system bars, so we have to get the real size here + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + if (VERSION.SDK_INT >= VERSION_CODES.R) { + wm.currentWindowMetrics.bounds + } else { + val screenBounds = Point() + @Suppress("DEPRECATION") + wm.defaultDisplay.getRealSize(screenBounds) + Rect(0, 0, screenBounds.x, screenBounds.y) + } + } + + private val aspectRatio by lazy(NONE) { + screenBounds.bottom.toFloat() / screenBounds.right.toFloat() + } + + private val recorderConfig by lazy(NONE) { + ScreenshotRecorderConfig( + recordingWidth = (720 / aspectRatio).roundToInt(), + recordingHeight = 720, + scaleFactor = 720f / screenBounds.bottom + ) + } + + override fun register(hub: IHub, options: SentryOptions) { + this.options = options + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + options.logger.log(INFO, "Session replay is only supported on API 26 and above") + return + } + + // TODO: check for replaysSessionSampleRate and replaysOnErrorSampleRate + + this.hub = hub + recorder = WindowRecorder(options, recorderConfig, this) + isEnabled.set(true) + } + + fun start() { + if (!isEnabled.get()) { + options.logger.log( + DEBUG, + "Session replay is disabled due to conditions not met in Integration.register" + ) + return + } + + if (isRecording.getAndSet(true)) { + options.logger.log( + DEBUG, + "Session replay is already being recorded, not starting a new one" + ) + return + } + + currentSegment.set(0) + currentReplayId.set(SentryId()) + hub?.configureScope { it.replayId = currentReplayId.get() } + cache = ReplayCache(options, currentReplayId.get(), recorderConfig) + + recorder?.startRecording() + segmentTimestamp.set(DateUtils.getCurrentDateTime()) + // TODO: finalize old recording if there's some left on disk and send it using the replayId from persisted scope (e.g. for ANRs) + } + + fun resume() { + segmentTimestamp.set(DateUtils.getCurrentDateTime()) + recorder?.resume() + } + + fun pause() { + val now = dateProvider.currentTimeMillis + recorder?.pause() + + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val duration = now - currentSegmentTimestamp.time + val replayId = currentReplayId.get() + saver.submit { + val videoDuration = + createAndCaptureSegment(duration, currentSegmentTimestamp, replayId, segmentId) + if (videoDuration != null) { + currentSegment.getAndIncrement() + } + } + } + + fun stop() { + if (!isEnabled.get()) { + options.logger.log( + DEBUG, + "Session replay is disabled due to conditions not met in Integration.register" + ) + return + } + + val now = dateProvider.currentTimeMillis + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val duration = now - currentSegmentTimestamp.time + val replayId = currentReplayId.get() + saver.submit { + createAndCaptureSegment(duration, currentSegmentTimestamp, replayId, segmentId) + cache?.cleanup() + } + + recorder?.stopRecording() + cache?.close() + currentSegment.set(0) + segmentTimestamp.set(null) + currentReplayId.set(null) + hub?.configureScope { it.replayId = null } + isRecording.set(false) + } + + override fun onScreenshotRecorded(bitmap: Bitmap) { + // have to do it before submitting, otherwise if the queue is busy, the timestamp won't be + // reflecting the exact time of when it was captured + val frameTimestamp = dateProvider.currentTimeMillis + saver.submit { + cache?.addFrame(bitmap, frameTimestamp) + + val now = dateProvider.currentTimeMillis + if (now - segmentTimestamp.get().time >= VIDEO_SEGMENT_DURATION) { + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val replayId = currentReplayId.get() + + val videoDuration = + createAndCaptureSegment( + VIDEO_SEGMENT_DURATION, + currentSegmentTimestamp, + replayId, + segmentId + ) + if (videoDuration != null) { + currentSegment.getAndIncrement() + // set next segment timestamp as close to the previous one as possible to avoid gaps + segmentTimestamp.set(DateUtils.getDateTime(currentSegmentTimestamp.time + videoDuration)) + } + } + } + } + + private fun createAndCaptureSegment( + duration: Long, + currentSegmentTimestamp: Date, + replayId: SentryId, + segmentId: Int + ): Long? { + val generatedVideo = cache?.createVideoOf( + duration, + currentSegmentTimestamp.time, + segmentId + ) ?: return null + + val (video, frameCount, videoDuration) = generatedVideo + captureReplay( + video, + replayId, + currentSegmentTimestamp, + segmentId, + frameCount, + videoDuration + ) + return videoDuration + } + + private fun captureReplay( + video: File, + currentReplayId: SentryId, + segmentTimestamp: Date, + segmentId: Int, + frameCount: Int, + duration: Long + ) { + val replay = SentryReplayEvent().apply { + eventId = currentReplayId + replayId = currentReplayId + this.segmentId = segmentId + this.timestamp = DateUtils.getDateTime(segmentTimestamp.time + duration) + if (segmentId == 0) { + replayStartTimestamp = segmentTimestamp + } + videoFile = video + } + + val recording = ReplayRecording().apply { + this.segmentId = segmentId + payload = listOf( + RRWebMetaEvent().apply { + this.timestamp = segmentTimestamp.time + height = recorderConfig.recordingHeight + width = recorderConfig.recordingWidth + }, + RRWebVideoEvent().apply { + this.timestamp = segmentTimestamp.time + this.segmentId = segmentId + this.duration = duration + this.frameCount = frameCount + size = video.length() + frameRate = recorderConfig.frameRate + height = recorderConfig.recordingHeight + width = recorderConfig.recordingWidth + // TODO: support non-fullscreen windows later + left = 0 + top = 0 + } + ) + } + + val hint = Hint().apply { replayRecording = recording } + hub?.captureReplay(replay, hint) + } + + override fun close() { + stop() + saver.gracefullyShutdown(options) + } + + private class ReplayExecutorServiceThreadFactory : ThreadFactory { + private var cnt = 0 + override fun newThread(r: Runnable): Thread { + val ret = Thread(r, "SentryReplayIntegration-" + cnt++) + ret.setDaemon(true) + return ret + } + } +} + +/** + * Retrieves the [ReplayIntegration] from the list of integrations in [SentryOptions] + */ +fun IHub.getReplayIntegration(): ReplayIntegration? = + options.integrations.find { it is ReplayIntegration } as? ReplayIntegration + +fun ExecutorService.gracefullyShutdown(options: SentryOptions) { + synchronized(this) { + if (!isShutdown) { + shutdown() + } + try { + if (!awaitTermination(options.shutdownTimeoutMillis, MILLISECONDS)) { + shutdownNow() + } + } catch (e: InterruptedException) { + shutdownNow() + Thread.currentThread().interrupt() + } + } +} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 3b3a6758fc..767c3614ba 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -2,34 +2,35 @@ package io.sentry.android.replay import android.annotation.TargetApi import android.graphics.Bitmap +import android.graphics.Bitmap.Config.ARGB_8888 import android.graphics.Canvas import android.graphics.Matrix import android.graphics.Paint -import android.graphics.Rect -import android.graphics.RectF import android.os.Handler import android.os.HandlerThread import android.os.Looper -import android.os.SystemClock -import android.util.Log import android.view.PixelCopy import android.view.View import android.view.ViewGroup import android.view.ViewTreeObserver -import io.sentry.android.replay.video.SimpleVideoEncoder +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.INFO +import io.sentry.SentryOptions import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode import java.lang.ref.WeakReference import java.util.WeakHashMap +import java.util.concurrent.atomic.AtomicBoolean import kotlin.system.measureTimeMillis -// TODO: use ILogger of Sentry and change level @TargetApi(26) internal class ScreenshotRecorder( - val encoder: SimpleVideoEncoder + val config: ScreenshotRecorderConfig, + val options: SentryOptions, + private val screenshotRecorderCallback: ScreenshotRecorderCallback ) : ViewTreeObserver.OnDrawListener { private var rootView: WeakReference? = null - private val thread = HandlerThread("SentryReplay").also { it.start() } + private val thread = HandlerThread("SentryReplayRecorder").also { it.start() } private val handler = Handler(thread.looper) private val bitmapToVH = WeakHashMap() private val maskingPaint = Paint() @@ -40,24 +41,32 @@ internal class ScreenshotRecorder( ) private val singlePixelBitmapCanvas: Canvas = Canvas(singlePixelBitmap) private val prescaledMatrix = Matrix().apply { - preScale(encoder.muxerConfig.scaleFactor, encoder.muxerConfig.scaleFactor) + preScale(config.scaleFactor, config.scaleFactor) } + private val contentChanged = AtomicBoolean(false) + private val isCapturing = AtomicBoolean(true) + private var lastScreenshot: Bitmap? = null - companion object { - const val TAG = "ScreenshotRecorder" - } + fun capture() { + if (!isCapturing.get()) { + options.logger.log(DEBUG, "ScreenshotRecorder is paused, not capturing screenshot") + return + } - private var lastCapturedAtMs: Long? = null - override fun onDraw() { - // TODO: replace with Debouncer from sentry-core - val now = SystemClock.uptimeMillis() - if (lastCapturedAtMs != null && (now - lastCapturedAtMs!!) < 500L) { + if (!contentChanged.get() && lastScreenshot != null) { + options.logger.log(DEBUG, "Content hasn't changed, repeating last known frame") + + lastScreenshot?.let { + screenshotRecorderCallback.onScreenshotRecorded( + it.copy(ARGB_8888, false) + ) + } return } - lastCapturedAtMs = now val root = rootView?.get() if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) { + options.logger.log(DEBUG, "Root view is invalid, not capturing screenshot") return } @@ -68,76 +77,75 @@ internal class ScreenshotRecorder( Bitmap.Config.ARGB_8888 ) - val time = measureTimeMillis { - val rootNode = ViewHierarchyNode.fromView(root) - root.traverse(rootNode) - bitmapToVH[bitmap] = rootNode - } - Log.e("TIME", time.toString()) - // postAtFrontOfQueue to ensure the view hierarchy and bitmap are ase close in-sync as possible Handler(Looper.getMainLooper()).postAtFrontOfQueue { + val time = measureTimeMillis { + val rootNode = ViewHierarchyNode.fromView(root) + root.traverse(rootNode) + bitmapToVH[bitmap] = rootNode + } + options.logger.log(DEBUG, "Took %d ms to capture view hierarchy", time) + PixelCopy.request( window, bitmap, { copyResult: Int -> - Log.d(TAG, "PixelCopy result: $copyResult") if (copyResult != PixelCopy.SUCCESS) { - Log.e(TAG, "Failed to capture screenshot") + options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult) return@request } - Log.e("BITMAP CAPTURED", bitmap.toString()) val viewHierarchy = bitmapToVH[bitmap] - - var scaledBitmap: Bitmap? = null + val scaledBitmap: Bitmap if (viewHierarchy == null) { - Log.e(TAG, "Failed to determine view hierarchy, not capturing") + options.logger.log(INFO, "Failed to determine view hierarchy, not capturing") return@request } else { scaledBitmap = Bitmap.createScaledBitmap( bitmap, - encoder.muxerConfig.videoWidth, - encoder.muxerConfig.videoHeight, + config.recordingWidth, + config.recordingHeight, true ) val canvas = Canvas(scaledBitmap) canvas.setMatrix(prescaledMatrix) viewHierarchy.traverse { - if (it.shouldRedact && (it.width > 0 && it.height > 0)) { - it.visibleRect ?: return@traverse - - // TODO: check for view type rather than rely on absence of dominantColor here - val color = if (it.dominantColor == null) { - singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) - singlePixelBitmap.getPixel(0, 0) - } else { - it.dominantColor - } - - maskingPaint.setColor(color) - canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) - } +// if (it.shouldRedact && (it.width > 0 && it.height > 0)) { +// it.visibleRect ?: return@traverse +// +// // TODO: check for view type rather than rely on absence of dominantColor here +// val color = if (it.dominantColor == null) { +// singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) +// singlePixelBitmap.getPixel(0, 0) +// } else { +// it.dominantColor +// } +// +// maskingPaint.setColor(color) +// canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) +// } } } -// val baos = ByteArrayOutputStream() -// scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 75, baos) -// val bmp = BitmapFactory.decodeByteArray(baos.toByteArray(), 0, baos.size()) - scaledBitmap?.let { - encoder.encode(it) - it.recycle() - } -// bmp.recycle() + val screenshot = scaledBitmap.copy(ARGB_8888, false) + screenshotRecorderCallback.onScreenshotRecorded(screenshot) + lastScreenshot = screenshot + contentChanged.set(false) + + scaledBitmap.recycle() bitmap.recycle() - Log.i(TAG, "Captured a screenshot") + bitmapToVH.remove(bitmap) }, handler ) } } + override fun onDraw() { + contentChanged.set(true) + } + fun bind(root: View) { // first unbind the current root unbind(rootView?.get()) @@ -152,9 +160,23 @@ internal class ScreenshotRecorder( root?.viewTreeObserver?.removeOnDrawListener(this) } + fun pause() { + isCapturing.set(false) + unbind(rootView?.get()) + } + + fun resume() { + // can't use bind() as it will invalidate the weakref + rootView?.get()?.viewTreeObserver?.addOnDrawListener(this) + isCapturing.set(true) + } + fun close() { unbind(rootView?.get()) rootView?.clear() + lastScreenshot?.recycle() + bitmapToVH.clear() + isCapturing.set(false) thread.quitSafely() } @@ -188,3 +210,14 @@ internal class ScreenshotRecorder( parentNode.children = childNodes } } + +internal data class ScreenshotRecorderConfig( + val recordingWidth: Int, + val recordingHeight: Int, + val scaleFactor: Float, + val frameRate: Int = 2 +) + +interface ScreenshotRecorderCallback { + fun onScreenshotRecorded(bitmap: Bitmap) +} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt index 1a60d686b4..d23222368f 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt @@ -1,31 +1,34 @@ package io.sentry.android.replay import android.annotation.TargetApi -import android.content.Context -import android.graphics.Point -import android.os.Build.VERSION -import android.os.Build.VERSION_CODES import android.view.View -import android.view.WindowManager -import io.sentry.android.replay.video.MuxerConfig -import io.sentry.android.replay.video.SimpleVideoEncoder -import java.io.File +import io.sentry.SentryLevel.ERROR +import io.sentry.SentryOptions +import java.io.Closeable import java.lang.ref.WeakReference +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.ThreadFactory +import java.util.concurrent.TimeUnit.MILLISECONDS import java.util.concurrent.atomic.AtomicBoolean import kotlin.LazyThreadSafetyMode.NONE -import kotlin.math.roundToInt @TargetApi(26) -class WindowRecorder { +internal class WindowRecorder( + private val options: SentryOptions, + private val recorderConfig: ScreenshotRecorderConfig, + private val screenshotRecorderCallback: ScreenshotRecorderCallback +) : Closeable { private val rootViewsSpy by lazy(NONE) { RootViewsSpy.install() } - private var encoder: SimpleVideoEncoder? = null private val isRecording = AtomicBoolean(false) private val rootViews = ArrayList>() private var recorder: ScreenshotRecorder? = null + private var capturingTask: ScheduledFuture<*>? = null + private val capturer = Executors.newSingleThreadScheduledExecutor(RecorderExecutorServiceThreadFactory()) private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added -> if (added) { @@ -42,53 +45,53 @@ class WindowRecorder { } } - fun startRecording(context: Context) { + fun startRecording() { if (isRecording.getAndSet(true)) { return } - val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager // val (height, width) = (wm.currentWindowMetrics.bounds.bottom / // context.resources.displayMetrics.density).roundToInt() to // (wm.currentWindowMetrics.bounds.right / // context.resources.displayMetrics.density).roundToInt() - // TODO: API level check - // PixelCopy takes screenshots including system bars, so we have to get the real size here - val height: Int - val aspectRatio = if (VERSION.SDK_INT >= VERSION_CODES.R) { - height = wm.currentWindowMetrics.bounds.bottom - height.toFloat() / wm.currentWindowMetrics.bounds.right.toFloat() - } else { - val screenResolution = Point() - @Suppress("DEPRECATION") - wm.defaultDisplay.getRealSize(screenResolution) - height = screenResolution.y - height.toFloat() / screenResolution.x.toFloat() - } - val videoFile = File(context.cacheDir, "sentry-sr.mp4") - encoder = SimpleVideoEncoder( - MuxerConfig( - videoFile, - videoWidth = (720 / aspectRatio).roundToInt(), - videoHeight = 720, - scaleFactor = 720f / height, - frameRate = 2f, - bitrate = 500 * 1000 - ) - ).also { it.start() } - recorder = ScreenshotRecorder(encoder!!) + recorder = ScreenshotRecorder(recorderConfig, options, screenshotRecorderCallback) rootViewsSpy.listeners += onRootViewsChangedListener + capturingTask = capturer.scheduleAtFixedRate({ + try { + recorder?.capture() + } catch (e: Throwable) { + options.logger.log(ERROR, "Failed to capture a screenshot with exception:", e) + // TODO: I guess schedule the capturer again, cause it will stop executing the runnable? + } + }, 0L, 1000L / recorderConfig.frameRate, MILLISECONDS) } + fun resume() = recorder?.resume() + fun pause() = recorder?.pause() + fun stopRecording() { rootViewsSpy.listeners -= onRootViewsChangedListener rootViews.forEach { recorder?.unbind(it.get()) } recorder?.close() rootViews.clear() recorder = null - encoder?.startRelease() - encoder = null + capturingTask?.cancel(false) + capturingTask = null isRecording.set(false) } + + private class RecorderExecutorServiceThreadFactory : ThreadFactory { + private var cnt = 0 + override fun newThread(r: Runnable): Thread { + val ret = Thread(r, "SentryWindowRecorder-" + cnt++) + ret.setDaemon(true) + return ret + } + } + + override fun close() { + stopRecording() + capturer.gracefullyShutdown(options) + } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt index 86ff440d02..ece684c46e 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt @@ -20,6 +20,8 @@ package io.sentry.android.replay import android.annotation.SuppressLint import android.os.Build.VERSION.SDK_INT +import android.os.Handler +import android.os.Looper import android.util.Log import android.view.View import android.view.Window @@ -152,8 +154,12 @@ internal class RootViewsSpy private constructor() { companion object { fun install(): RootViewsSpy { return RootViewsSpy().apply { - WindowManagerSpy.swapWindowManagerGlobalMViews { mViews -> - delegatingViewList.apply { addAll(mViews) } + // had to do this as a first message of the main thread queue, otherwise if this is + // called from ContentProvider, it might be too early and the listener won't be installed + Handler(Looper.getMainLooper()).postAtFrontOfQueue { + WindowManagerSpy.swapWindowManagerGlobalMViews { mViews -> + delegatingViewList.apply { addAll(mViews) } + } } } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt index bdedb888cd..db34b2dbea 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt @@ -32,9 +32,10 @@ package io.sentry.android.replay.video import android.media.MediaCodec import android.media.MediaFormat import android.media.MediaMuxer -import android.util.Log import java.nio.ByteBuffer import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeUnit.MICROSECONDS +import java.util.concurrent.TimeUnit.MILLISECONDS class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMuxer { private val frameUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() @@ -50,7 +51,6 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux override fun start(videoFormat: MediaFormat) { videoTrackIndex = muxer.addTrack(videoFormat) - Log.i("SimpleMp4FrameMuxer", "start() videoFormat=$videoFormat videoTrackIndex=$videoTrackIndex") muxer.start() started = true } @@ -74,6 +74,7 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux } override fun getVideoTime(): Long { - return finalVideoTime + // have to add one sec as we calculate it 0-based above + return MILLISECONDS.convert(finalVideoTime + frameUsec, MICROSECONDS) } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt index 4046eec37b..cdb15fc11f 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt @@ -34,20 +34,25 @@ import android.graphics.Bitmap import android.media.MediaCodec import android.media.MediaCodecInfo import android.media.MediaFormat -import android.os.Handler -import android.os.Looper import android.view.Surface +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryOptions +import io.sentry.android.replay.ScreenshotRecorderConfig import java.io.File +import java.nio.ByteBuffer + +private const val TIMEOUT_USEC = 100_000L @TargetApi(26) internal class SimpleVideoEncoder( + val options: SentryOptions, val muxerConfig: MuxerConfig ) { private val mediaFormat: MediaFormat = run { val format = MediaFormat.createVideoFormat( muxerConfig.mimeType, - muxerConfig.videoWidth, - muxerConfig.videoHeight + muxerConfig.recorderConfig.recordingWidth, + muxerConfig.recorderConfig.recordingHeight ) // Set some properties. Failing to specify some of these can cause the MediaCodec @@ -72,84 +77,102 @@ internal class SimpleVideoEncoder( codec } + private val bufferInfo: MediaCodec.BufferInfo = MediaCodec.BufferInfo() private val frameMuxer = muxerConfig.frameMuxer + val duration get() = frameMuxer.getVideoTime() private var surface: Surface? = null fun start() { - mediaCodec.setCallback(createMediaCodecCallback(), Handler(Looper.getMainLooper())) - mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) surface = mediaCodec.createInputSurface() mediaCodec.start() + drainCodec(false) } - private fun createMediaCodecCallback(): MediaCodec.Callback { - return object : MediaCodec.Callback() { - override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { - } - - override fun onOutputBufferAvailable( - codec: MediaCodec, - index: Int, - info: MediaCodec.BufferInfo - ) { - val encodedData = codec.getOutputBuffer(index)!! + fun encode(image: Bitmap) { + // NOTE do not use `lockCanvas` like what is done in bitmap2video + // This is because https://developer.android.com/reference/android/media/MediaCodec#createInputSurface() + // says that, "Surface.lockCanvas(android.graphics.Rect) may fail or produce unexpected results." + val canvas = surface?.lockHardwareCanvas() + canvas?.drawBitmap(image, 0f, 0f, null) + surface?.unlockCanvasAndPost(canvas) + drainCodec(false) + } - var effectiveSize = info.size + /** + * Extracts all pending data from the encoder. + * + * + * If endOfStream is not set, this returns when there is no more data to drain. If it + * is set, we send EOS to the encoder, and then iterate until we see EOS on the output. + * Calling this with endOfStream set should be done once, right before stopping the muxer. + * + * Borrows heavily from https://bigflake.com/mediacodec/EncodeAndMuxTest.java.txt + */ + private fun drainCodec(endOfStream: Boolean) { + options.logger.log(DEBUG, "[Encoder]: drainCodec($endOfStream)") + if (endOfStream) { + options.logger.log(DEBUG, "[Encoder]: sending EOS to encoder") + mediaCodec.signalEndOfInputStream() + } + var encoderOutputBuffers: Array? = mediaCodec.outputBuffers + while (true) { + val encoderStatus: Int = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC) + if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { + // no output available yet + if (!endOfStream) { + break // out of while + } else { + options.logger.log(DEBUG, "[Encoder]: no output available, spinning to await EOS") + } + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + // not expected for an encoder + encoderOutputBuffers = mediaCodec.outputBuffers + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + // should happen before receiving buffers, and should only happen once + if (frameMuxer.isStarted()) { + throw RuntimeException("format changed twice") + } + val newFormat: MediaFormat = mediaCodec.outputFormat + options.logger.log(DEBUG, "[Encoder]: encoder output format changed: $newFormat") - if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + // now that we have the Magic Goodies, start the muxer + frameMuxer.start(newFormat) + } else if (encoderStatus < 0) { + options.logger.log(DEBUG, "[Encoder]: unexpected result from encoder.dequeueOutputBuffer: $encoderStatus") + // let's ignore it + } else { + val encodedData = encoderOutputBuffers?.get(encoderStatus) + ?: throw RuntimeException("encoderOutputBuffer $encoderStatus was null") + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. - effectiveSize = 0 + options.logger.log(DEBUG, "[Encoder]: ignoring BUFFER_FLAG_CODEC_CONFIG") + bufferInfo.size = 0 } - - if (effectiveSize != 0) { + if (bufferInfo.size != 0) { if (!frameMuxer.isStarted()) { throw RuntimeException("muxer hasn't started") } - frameMuxer.muxVideoFrame(encodedData, info) + frameMuxer.muxVideoFrame(encodedData, bufferInfo) + options.logger.log(DEBUG, "[Encoder]: sent ${bufferInfo.size} bytes to muxer") } - - mediaCodec.releaseOutputBuffer(index, false) - - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { - actualRelease() - } - } - - override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) { - } - - override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - // should happen before receiving buffers, and should only happen once - if (frameMuxer.isStarted()) { - throw RuntimeException("format changed twice") + mediaCodec.releaseOutputBuffer(encoderStatus, false) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { + if (!endOfStream) { + options.logger.log(DEBUG, "[Encoder]: reached end of stream unexpectedly") + } else { + options.logger.log(DEBUG, "[Encoder]: end of stream reached") + } + break // out of while } - val newFormat: MediaFormat = mediaCodec.outputFormat - // now that we have the Magic Goodies, start the muxer - frameMuxer.start(newFormat) } } } - fun encode(image: Bitmap) { - // NOTE do not use `lockCanvas` like what is done in bitmap2video - // This is because https://developer.android.com/reference/android/media/MediaCodec#createInputSurface() - // says that, "Surface.lockCanvas(android.graphics.Rect) may fail or produce unexpected results." - val canvas = surface?.lockHardwareCanvas() - canvas?.drawBitmap(image, 0f, 0f, null) - surface?.unlockCanvasAndPost(canvas) - } - - /** - * can only *start* releasing, since it is asynchronous - */ - fun startRelease() { - mediaCodec.signalEndOfInputStream() - } - - private fun actualRelease() { + fun release() { + drainCodec(true) mediaCodec.stop() mediaCodec.release() surface?.release() @@ -161,9 +184,7 @@ internal class SimpleVideoEncoder( @TargetApi(24) internal data class MuxerConfig( val file: File, - val videoWidth: Int, - val videoHeight: Int, - val scaleFactor: Float, + val recorderConfig: ScreenshotRecorderConfig, val mimeType: String = MediaFormat.MIMETYPE_VIDEO_AVC, val frameRate: Float, val bitrate: Int, diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt new file mode 100644 index 0000000000..6a03291105 --- /dev/null +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt @@ -0,0 +1,86 @@ +package io.sentry.android.replay + +import android.graphics.Bitmap +import android.graphics.Bitmap.Config.ARGB_8888 +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.SentryOptions +import io.sentry.protocol.SentryId +import io.sentry.transport.ICurrentDateProvider +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.annotation.Config +import kotlin.test.Test + +@RunWith(AndroidJUnit4::class) +@Config(sdk = [26]) +class ReplayCacheTest { + + @get:Rule + val tmpDir = TemporaryFolder() + + internal class Fixture { + val options = SentryOptions() + fun getSut( + dir: TemporaryFolder?, + replayId: SentryId, + frameRate: Int, + dateProvider: ICurrentDateProvider + ): ReplayCache { + val recorderConfig = ScreenshotRecorderConfig(100, 200, 1f, frameRate) + options.run { + cacheDirPath = dir?.newFolder()?.absolutePath + } + return ReplayCache(options, replayId, recorderConfig, dateProvider) + } + } + + private val fixture = Fixture() + + @Test + fun `test`() { + val replayId = SentryId() + val replayCache = fixture.getSut( + tmpDir, + replayId, + frameRate = 1, + dateProvider = object : ICurrentDateProvider { + override fun getCurrentTimeMillis(): Long { + return 1 + } + } + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap) + + replayCache.createVideoOf(5000L, 0, 0) + replayCache.createVideoOf(5000L, 5000L, 1) + } + + @Test + fun `test2`() { + val replayId = SentryId() + val replayCache = fixture.getSut( + tmpDir, + replayId, + frameRate = 1, + dateProvider = object : ICurrentDateProvider { + var counter = 0 + override fun getCurrentTimeMillis(): Long { + return when (counter++) { + 0 -> 1 + 1 -> 1001 + else -> 1001 + } + } + } + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap) + replayCache.addFrame(bitmap) + + replayCache.createVideoOf(5000L, 0, 0) + } +} diff --git a/sentry-android/build.gradle.kts b/sentry-android/build.gradle.kts index 47b873ac49..81619b736f 100644 --- a/sentry-android/build.gradle.kts +++ b/sentry-android/build.gradle.kts @@ -35,4 +35,5 @@ android { dependencies { api(projects.sentryAndroidCore) api(projects.sentryAndroidNdk) + api(projects.sentryAndroidReplay) } diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index b6acae0cde..9e3a90946f 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -664,6 +664,7 @@ public abstract interface class io/sentry/IScope { public abstract fun getLevel ()Lio/sentry/SentryLevel; public abstract fun getOptions ()Lio/sentry/SentryOptions; public abstract fun getPropagationContext ()Lio/sentry/PropagationContext; + public abstract fun getReplayId ()Lio/sentry/protocol/SentryId; public abstract fun getRequest ()Lio/sentry/protocol/Request; public abstract fun getScreen ()Ljava/lang/String; public abstract fun getSession ()Lio/sentry/Session; @@ -686,6 +687,7 @@ public abstract interface class io/sentry/IScope { public abstract fun setFingerprint (Ljava/util/List;)V public abstract fun setLevel (Lio/sentry/SentryLevel;)V public abstract fun setPropagationContext (Lio/sentry/PropagationContext;)V + public abstract fun setReplayId (Lio/sentry/protocol/SentryId;)V public abstract fun setRequest (Lio/sentry/protocol/Request;)V public abstract fun setScreen (Ljava/lang/String;)V public abstract fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -1215,6 +1217,7 @@ public final class io/sentry/NoOpScope : io/sentry/IScope { public fun getLevel ()Lio/sentry/SentryLevel; public fun getOptions ()Lio/sentry/SentryOptions; public fun getPropagationContext ()Lio/sentry/PropagationContext; + public fun getReplayId ()Lio/sentry/protocol/SentryId; public fun getRequest ()Lio/sentry/protocol/Request; public fun getScreen ()Ljava/lang/String; public fun getSession ()Lio/sentry/Session; @@ -1237,6 +1240,7 @@ public final class io/sentry/NoOpScope : io/sentry/IScope { public fun setFingerprint (Ljava/util/List;)V public fun setLevel (Lio/sentry/SentryLevel;)V public fun setPropagationContext (Lio/sentry/PropagationContext;)V + public fun setReplayId (Lio/sentry/protocol/SentryId;)V public fun setRequest (Lio/sentry/protocol/Request;)V public fun setScreen (Ljava/lang/String;)V public fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -1639,6 +1643,7 @@ public final class io/sentry/Scope : io/sentry/IScope { public fun getLevel ()Lio/sentry/SentryLevel; public fun getOptions ()Lio/sentry/SentryOptions; public fun getPropagationContext ()Lio/sentry/PropagationContext; + public fun getReplayId ()Lio/sentry/protocol/SentryId; public fun getRequest ()Lio/sentry/protocol/Request; public fun getScreen ()Ljava/lang/String; public fun getSession ()Lio/sentry/Session; @@ -1661,6 +1666,7 @@ public final class io/sentry/Scope : io/sentry/IScope { public fun setFingerprint (Ljava/util/List;)V public fun setLevel (Lio/sentry/SentryLevel;)V public fun setPropagationContext (Lio/sentry/PropagationContext;)V + public fun setReplayId (Lio/sentry/protocol/SentryId;)V public fun setRequest (Lio/sentry/protocol/Request;)V public fun setScreen (Ljava/lang/String;)V public fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -4767,7 +4773,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun equals (Ljava/lang/Object;)Z public fun getContainer ()Ljava/lang/String; public fun getDataUnknown ()Ljava/util/Map; - public fun getDuration ()I + public fun getDuration ()J public fun getEncoding ()Ljava/lang/String; public fun getFrameCount ()I public fun getFrameRate ()I @@ -4785,7 +4791,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setContainer (Ljava/lang/String;)V public fun setDataUnknown (Ljava/util/Map;)V - public fun setDuration (I)V + public fun setDuration (J)V public fun setEncoding (Ljava/lang/String;)V public fun setFrameCount (I)V public fun setFrameRate (I)V diff --git a/sentry/src/main/java/io/sentry/IScope.java b/sentry/src/main/java/io/sentry/IScope.java index 3842fb2c3a..2d38371ead 100644 --- a/sentry/src/main/java/io/sentry/IScope.java +++ b/sentry/src/main/java/io/sentry/IScope.java @@ -2,6 +2,7 @@ import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.User; import java.util.Collection; import java.util.List; @@ -84,6 +85,23 @@ public interface IScope { @ApiStatus.Internal void setScreen(final @Nullable String screen); + /** + * Returns the Scope's current replay_id, previously set by {@link IScope#setReplayId(SentryId)} + * + * @return the id of the current session replay + */ + @ApiStatus.Internal + @Nullable + SentryId getReplayId(); + + /** + * Sets the Scope's current replay_id + * + * @param replayId the id of the current session replay + */ + @ApiStatus.Internal + void setReplayId(final @Nullable SentryId replayId); + /** * Returns the Scope's request * diff --git a/sentry/src/main/java/io/sentry/NoOpScope.java b/sentry/src/main/java/io/sentry/NoOpScope.java index c756fb49a3..660dca0b69 100644 --- a/sentry/src/main/java/io/sentry/NoOpScope.java +++ b/sentry/src/main/java/io/sentry/NoOpScope.java @@ -2,6 +2,7 @@ import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.User; import java.util.ArrayDeque; import java.util.ArrayList; @@ -68,6 +69,14 @@ public void setUser(@Nullable User user) {} @Override public void setScreen(@Nullable String screen) {} + @Override + public @Nullable SentryId getReplayId() { + return null; + } + + @Override + public void setReplayId(@Nullable SentryId replayId) {} + @Override public @Nullable Request getRequest() { return null; diff --git a/sentry/src/main/java/io/sentry/Scope.java b/sentry/src/main/java/io/sentry/Scope.java index 91c9fcd8cf..164d52dc2b 100644 --- a/sentry/src/main/java/io/sentry/Scope.java +++ b/sentry/src/main/java/io/sentry/Scope.java @@ -3,6 +3,7 @@ import io.sentry.protocol.App; import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.TransactionNameSource; import io.sentry.protocol.User; import io.sentry.util.CollectionUtils; @@ -80,6 +81,9 @@ public final class Scope implements IScope { private @NotNull PropagationContext propagationContext; + /** Scope's session replay id */ + private @Nullable SentryId replayId; + /** * Scope's ctor * @@ -101,6 +105,7 @@ private Scope(final @NotNull Scope scope) { final User userRef = scope.user; this.user = userRef != null ? new User(userRef) : null; this.screen = scope.screen; + this.replayId = scope.replayId; final Request requestRef = scope.request; this.request = requestRef != null ? new Request(requestRef) : null; @@ -312,6 +317,18 @@ public void setScreen(final @Nullable String screen) { } } + @Override + public @Nullable SentryId getReplayId() { + return replayId; + } + + @Override + public void setReplayId(final @Nullable SentryId replayId) { + this.replayId = replayId; + + // TODO: set to contexts and notify observers to persist this as well + } + /** * Returns the Scope's request * diff --git a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java index ff162f4464..728b28d0e2 100644 --- a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java +++ b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java @@ -23,7 +23,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.Callable; import org.jetbrains.annotations.ApiStatus; @@ -360,7 +360,8 @@ public static SentryEnvelopeItem fromReplay( try (final ByteArrayOutputStream stream = new ByteArrayOutputStream(); final Writer writer = new BufferedWriter(new OutputStreamWriter(stream, UTF_8))) { - final Map replayPayload = new HashMap<>(); + // relay expects the payload to be in this exact order: [event,rrweb,video] + final Map replayPayload = new LinkedHashMap<>(); // first serialize replay event json bytes serializer.serialize(replayEvent, writer); replayPayload.put(SentryItemType.ReplayEvent.getItemType(), stream.toByteArray()); diff --git a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java index 5bea9e3c47..532177ff9f 100644 --- a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java +++ b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java @@ -26,7 +26,7 @@ public final class RRWebVideoEvent extends RRWebEvent implements JsonUnknown, Js private @NotNull String tag; private int segmentId; private long size; - private int duration; + private long duration; private @NotNull String encoding = REPLAY_ENCODING; private @NotNull String container = REPLAY_CONTAINER; private int height; @@ -72,11 +72,11 @@ public void setSize(final long size) { this.size = size; } - public int getDuration() { + public long getDuration() { return duration; } - public void setDuration(final int duration) { + public void setDuration(final long duration) { this.duration = duration; } @@ -380,7 +380,7 @@ private void deserializePayload( event.size = size == null ? 0 : size; break; case JsonKeys.DURATION: - event.duration = reader.nextInt(); + event.duration = reader.nextLong(); break; case JsonKeys.CONTAINER: final String container = reader.nextStringOrNull(); From 2e954f700be7c161ca4a47c45d27d72f045c8d6f Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 19 Mar 2024 16:23:56 +0100 Subject: [PATCH 02/19] Uncomment redacting --- .../android/replay/ScreenshotRecorder.kt | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 767c3614ba..87915f8505 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -6,6 +6,8 @@ import android.graphics.Bitmap.Config.ARGB_8888 import android.graphics.Canvas import android.graphics.Matrix import android.graphics.Paint +import android.graphics.Rect +import android.graphics.RectF import android.os.Handler import android.os.HandlerThread import android.os.Looper @@ -111,20 +113,20 @@ internal class ScreenshotRecorder( val canvas = Canvas(scaledBitmap) canvas.setMatrix(prescaledMatrix) viewHierarchy.traverse { -// if (it.shouldRedact && (it.width > 0 && it.height > 0)) { -// it.visibleRect ?: return@traverse -// -// // TODO: check for view type rather than rely on absence of dominantColor here -// val color = if (it.dominantColor == null) { -// singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) -// singlePixelBitmap.getPixel(0, 0) -// } else { -// it.dominantColor -// } -// -// maskingPaint.setColor(color) -// canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) -// } + if (it.shouldRedact && (it.width > 0 && it.height > 0)) { + it.visibleRect ?: return@traverse + + // TODO: check for view type rather than rely on absence of dominantColor here + val color = if (it.dominantColor == null) { + singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) + singlePixelBitmap.getPixel(0, 0) + } else { + it.dominantColor + } + + maskingPaint.setColor(color) + canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) + } } } From 8bc6219d87c40673be0e210a2932b1865bd82e8e Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Wed, 20 Mar 2024 11:37:43 +0100 Subject: [PATCH 03/19] Update proguard rules --- sentry-android-core/proguard-rules.pro | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sentry-android-core/proguard-rules.pro b/sentry-android-core/proguard-rules.pro index 67d7e7691d..a78a5a14a1 100644 --- a/sentry-android-core/proguard-rules.pro +++ b/sentry-android-core/proguard-rules.pro @@ -72,3 +72,9 @@ -keepnames class io.sentry.exception.SentryHttpClientException ##---------------End: proguard configuration for sentry-okhttp ---------- + +##---------------Begin: proguard configuration for sentry-android-replay ---------- +-dontwarn io.sentry.android.replay.ReplayIntegration +-dontwarn io.sentry.android.replay.ReplayIntegrationKt +-keepnames class io.sentry.android.replay.ReplayIntegration +##---------------End: proguard configuration for sentry-android-replay ---------- From a5aa4bee6954015ce440d0b9ec1a7ca64e9ca7ed Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Wed, 20 Mar 2024 12:07:26 +0100 Subject: [PATCH 04/19] Add missing rule for AndroidTest --- .../sentry-uitest-android/proguard-rules.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro index 49f7f0749d..02f5e80ba3 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro +++ b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro @@ -39,4 +39,4 @@ -dontwarn org.opentest4j.AssertionFailedError -dontwarn org.mockito.internal.** -dontwarn org.jetbrains.annotations.** - +-dontwarn io.sentry.android.replay.ReplayIntegration From f72e45ff601ad9ba8221c496c5ae079362fbb6bf Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Thu, 21 Mar 2024 22:38:59 +0100 Subject: [PATCH 05/19] Add ReplayCache tests --- .../io/sentry/android/replay/ReplayCache.kt | 36 ++-- .../android/replay/ReplayIntegration.kt | 4 +- .../replay/video/SimpleMp4FrameMuxer.kt | 5 +- .../replay/video/SimpleVideoEncoder.kt | 10 +- .../sentry/android/replay/ReplayCacheTest.kt | 180 +++++++++++++++--- 5 files changed, 180 insertions(+), 55 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt index cf6d6a93aa..da62dfb6b1 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -10,20 +10,30 @@ import io.sentry.SentryOptions import io.sentry.android.replay.video.MuxerConfig import io.sentry.android.replay.video.SimpleVideoEncoder import io.sentry.protocol.SentryId -import io.sentry.util.FileUtils import java.io.Closeable import java.io.File internal class ReplayCache( private val options: SentryOptions, private val replayId: SentryId, - private val recorderConfig: ScreenshotRecorderConfig + private val recorderConfig: ScreenshotRecorderConfig, + private val encoderCreator: (File) -> SimpleVideoEncoder = { videoFile -> + SimpleVideoEncoder( + options, + MuxerConfig( + file = videoFile, + recorderConfig = recorderConfig, + frameRate = recorderConfig.frameRate.toFloat(), + bitrate = 20 * 1000 + ) + ).also { it.start() } + } ) : Closeable { private val encoderLock = Any() - private var encoder: SimpleVideoEncoder? = null + internal var encoder: SimpleVideoEncoder? = null - private val replayCacheDir: File? by lazy { + internal val replayCacheDir: File? by lazy { if (options.cacheDirPath.isNullOrEmpty()) { options.logger.log( WARNING, @@ -36,7 +46,7 @@ internal class ReplayCache( } // TODO: maybe account for multi-threaded access - private val frames = mutableListOf() + internal val frames = mutableListOf() fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { if (replayCacheDir == null) { @@ -66,17 +76,7 @@ internal class ReplayCache( // TODO: reuse instance of encoder and just change file path to create a different muxer val videoFile = File(replayCacheDir, "$segmentId.mp4") - encoder = synchronized(encoderLock) { - SimpleVideoEncoder( - options, - MuxerConfig( - file = videoFile, - recorderConfig = recorderConfig, - frameRate = recorderConfig.frameRate.toFloat(), - bitrate = 20 * 1000 - ) - ) - }.also { it.start() } + encoder = synchronized(encoderLock) { encoderCreator(videoFile) } val step = 1000 / recorderConfig.frameRate.toLong() var frameCount = 0 @@ -147,10 +147,6 @@ internal class ReplayCache( } } - fun cleanup() { - FileUtils.deleteRecursively(replayCacheDir) - } - override fun close() { synchronized(encoderLock) { encoder?.release() diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index 26b35f9a15..756b8638d0 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -21,6 +21,7 @@ import io.sentry.protocol.SentryId import io.sentry.rrweb.RRWebMetaEvent import io.sentry.rrweb.RRWebVideoEvent import io.sentry.transport.ICurrentDateProvider +import io.sentry.util.FileUtils import java.io.Closeable import java.io.File import java.util.Date @@ -161,9 +162,10 @@ class ReplayIntegration( val segmentId = currentSegment.get() val duration = now - currentSegmentTimestamp.time val replayId = currentReplayId.get() + val replayCacheDir = cache?.replayCacheDir saver.submit { createAndCaptureSegment(duration, currentSegmentTimestamp, replayId, segmentId) - cache?.cleanup() + FileUtils.deleteRecursively(replayCacheDir) } recorder?.stopRecording() diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt index db34b2dbea..8a21b0bec0 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt @@ -37,7 +37,7 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit.MICROSECONDS import java.util.concurrent.TimeUnit.MILLISECONDS -class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMuxer { +class SimpleMp4FrameMuxer(path: String, fps: Float) : SimpleFrameMuxer { private val frameUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() private val muxer: MediaMuxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) @@ -74,6 +74,9 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux } override fun getVideoTime(): Long { + if (videoFrames == 0) { + return 0 + } // have to add one sec as we calculate it 0-based above return MILLISECONDS.convert(finalVideoTime + frameUsec, MICROSECONDS) } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt index cdb15fc11f..e2561faa1b 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt @@ -46,7 +46,8 @@ private const val TIMEOUT_USEC = 100_000L @TargetApi(26) internal class SimpleVideoEncoder( val options: SentryOptions, - val muxerConfig: MuxerConfig + val muxerConfig: MuxerConfig, + val onClose: (() -> Unit)? = null ) { private val mediaFormat: MediaFormat = run { val format = MediaFormat.createVideoFormat( @@ -68,7 +69,7 @@ internal class SimpleVideoEncoder( format } - private val mediaCodec: MediaCodec = run { + internal val mediaCodec: MediaCodec = run { // val codecs = MediaCodecList(REGULAR_CODECS) // val codecName = codecs.findEncoderForFormat(mediaFormat) // val codec = MediaCodec.createByCodecName(codecName) @@ -172,6 +173,7 @@ internal class SimpleVideoEncoder( } fun release() { + onClose?.invoke() drainCodec(true) mediaCodec.stop() mediaCodec.release() @@ -185,8 +187,8 @@ internal class SimpleVideoEncoder( internal data class MuxerConfig( val file: File, val recorderConfig: ScreenshotRecorderConfig, + val bitrate: Int = 20_000, + val frameRate: Float = recorderConfig.frameRate.toFloat(), val mimeType: String = MediaFormat.MIMETYPE_VIDEO_AVC, - val frameRate: Float, - val bitrate: Int, val frameMuxer: SimpleFrameMuxer = SimpleMp4FrameMuxer(file.absolutePath, frameRate) ) diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt index 6a03291105..88d410fab1 100644 --- a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt @@ -2,15 +2,23 @@ package io.sentry.android.replay import android.graphics.Bitmap import android.graphics.Bitmap.Config.ARGB_8888 +import android.media.MediaCodec import androidx.test.ext.junit.runners.AndroidJUnit4 import io.sentry.SentryOptions +import io.sentry.android.replay.video.MuxerConfig +import io.sentry.android.replay.video.SimpleVideoEncoder import io.sentry.protocol.SentryId -import io.sentry.transport.ICurrentDateProvider import org.junit.Rule import org.junit.rules.TemporaryFolder import org.junit.runner.RunWith import org.robolectric.annotation.Config +import java.io.File +import java.util.concurrent.TimeUnit.MICROSECONDS +import java.util.concurrent.TimeUnit.MILLISECONDS import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue @RunWith(AndroidJUnit4::class) @Config(sdk = [26]) @@ -21,66 +29,180 @@ class ReplayCacheTest { internal class Fixture { val options = SentryOptions() + var encoder: SimpleVideoEncoder? = null fun getSut( dir: TemporaryFolder?, - replayId: SentryId, + replayId: SentryId = SentryId(), frameRate: Int, - dateProvider: ICurrentDateProvider + framesToEncode: Int = 0 ): ReplayCache { val recorderConfig = ScreenshotRecorderConfig(100, 200, 1f, frameRate) options.run { cacheDirPath = dir?.newFolder()?.absolutePath } - return ReplayCache(options, replayId, recorderConfig, dateProvider) + return ReplayCache(options, replayId, recorderConfig, encoderCreator = { videoFile -> + encoder = SimpleVideoEncoder( + options, + MuxerConfig( + file = videoFile, + recorderConfig = recorderConfig, + frameRate = recorderConfig.frameRate.toFloat(), + bitrate = 20 * 1000 + ), + onClose = { + encodeFrame(framesToEncode, frameRate, size = 0, flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM) + } + ).also { it.start() } + repeat(framesToEncode) { encodeFrame(it, frameRate) } + + encoder!! + }) + } + + fun encodeFrame(index: Int, frameRate: Int, size: Int = 10, flags: Int = 0) { + val presentationTime = MICROSECONDS.convert(index * (1000L / frameRate), MILLISECONDS) + encoder!!.mediaCodec.dequeueInputBuffer(0) + encoder!!.mediaCodec.queueInputBuffer(index, index * size, size, presentationTime, flags) } } private val fixture = Fixture() @Test - fun `test`() { + fun `when no cacheDirPath specified, does not store screenshots`() { val replayId = SentryId() val replayCache = fixture.getSut( - tmpDir, + null, replayId, - frameRate = 1, - dateProvider = object : ICurrentDateProvider { - override fun getCurrentTimeMillis(): Long { - return 1 - } - } + frameRate = 1 ) val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) - replayCache.addFrame(bitmap) + replayCache.addFrame(bitmap, 1) - replayCache.createVideoOf(5000L, 0, 0) - replayCache.createVideoOf(5000L, 5000L, 1) + assertTrue(replayCache.frames.isEmpty()) } @Test - fun `test2`() { + fun `stores screenshots with timestamp as name`() { val replayId = SentryId() val replayCache = fixture.getSut( tmpDir, replayId, + frameRate = 1 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + val expectedScreenshotFile = File(replayCache.replayCacheDir, "1.jpg") + assertTrue(expectedScreenshotFile.exists()) + assertEquals(replayCache.frames.first().timestamp, 1) + assertEquals(replayCache.frames.first().screenshot, expectedScreenshotFile) + } + + @Test + fun `when no frames are provided, returns nothing`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1 + ) + + val video = replayCache.createVideoOf(5000L, 0, 0) + + assertNull(video) + } + + @Test + fun `deletes frames after creating a video`() { + val replayCache = fixture.getSut( + tmpDir, frameRate = 1, - dateProvider = object : ICurrentDateProvider { - var counter = 0 - override fun getCurrentTimeMillis(): Long { - return when (counter++) { - 0 -> 1 - 1 -> 1001 - else -> 1001 - } - } - } + framesToEncode = 3 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + replayCache.createVideoOf(3000L, 0, 0) + assertTrue(replayCache.frames.isEmpty()) + assertTrue(replayCache.replayCacheDir!!.listFiles()!!.none { it.extension == "jpg" }) + } + + @Test + fun `repeats last known frame for the segment duration`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + } + + @Test + fun `repeats last known frame for the segment duration for each timespan`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 3001) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + } + + @Test + fun `repeats last known frame for each segment`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 5001) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + + val segment1 = replayCache.createVideoOf(5000L, 5000L, 1) + assertEquals(5, segment1!!.frameCount) + assertEquals(5000, segment1.duration) + assertEquals(File(replayCache.replayCacheDir, "1.mp4"), segment1.video) + } + + @Test + fun `respects frameRate`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 2, + framesToEncode = 6 ) val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) - replayCache.addFrame(bitmap) - replayCache.addFrame(bitmap) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 1001) + replayCache.addFrame(bitmap, 1501) - replayCache.createVideoOf(5000L, 0, 0) + val segment0 = replayCache.createVideoOf(3000L, 0, 0) + assertEquals(6, segment0!!.frameCount) + assertEquals(3000, segment0.duration) + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) } } From 545712ca5d298eca2e265f6a711bc95068e80846 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Fri, 22 Mar 2024 00:28:02 +0100 Subject: [PATCH 06/19] Add tests --- sentry-android-core/build.gradle.kts | 1 + .../io/sentry/android/core/SentryAndroid.java | 50 +++++++++---------- .../core/AndroidOptionsInitializerTest.kt | 27 +++++++++- .../android/core/AndroidProfilerTest.kt | 1 + .../core/AndroidTransactionProfilerTest.kt | 1 + .../android/core/LifecycleWatcherTest.kt | 8 --- .../sentry/android/core/SentryAndroidTest.kt | 25 ++++++++-- .../android/core/SentryInitProviderTest.kt | 1 + .../api/sentry-android-replay.api | 1 + .../android/replay/ReplayIntegration.kt | 2 + 10 files changed, 76 insertions(+), 41 deletions(-) diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index da4851c92a..6ea33c7b74 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -104,6 +104,7 @@ dependencies { testImplementation(projects.sentryTestSupport) testImplementation(projects.sentryAndroidFragment) testImplementation(projects.sentryAndroidTimber) + testImplementation(projects.sentryAndroidReplay) testImplementation(projects.sentryComposeHelper) testImplementation(projects.sentryAndroidNdk) testRuntimeOnly(Config.Libs.composeUi) diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java index 39a2019d1d..d6e11d15f5 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java @@ -227,44 +227,43 @@ private static void deduplicateIntegrations( } public static synchronized void startReplay() { - performReplayAction( - "starting", - (replay) -> { - replay.start(); - }); + if (!ensureReplayIntegration("starting")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).start(); } public static synchronized void stopReplay() { - performReplayAction( - "stopping", - (replay) -> { - replay.stop(); - }); + if (!ensureReplayIntegration("stopping")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).stop(); } public static synchronized void resumeReplay() { - performReplayAction( - "resuming", - (replay) -> { - replay.resume(); - }); + if (!ensureReplayIntegration("resuming")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).resume(); } public static synchronized void pauseReplay() { - performReplayAction( - "pausing", - (replay) -> { - replay.pause(); - }); + if (!ensureReplayIntegration("pausing")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).pause(); } - private static void performReplayAction( - final @NotNull String actionName, final @NotNull ReplayCallable action) { + private static boolean ensureReplayIntegration(final @NotNull String actionName) { final @NotNull IHub hub = Sentry.getCurrentHub(); if (isReplayAvailable) { final ReplayIntegration replay = ReplayIntegrationKt.getReplayIntegration(hub); if (replay != null) { - action.call(replay); + return true; } else { hub.getOptions() .getLogger() @@ -279,9 +278,6 @@ private static void performReplayAction( SentryLevel.INFO, "Session Replay wasn't found on classpath, not " + actionName + " the replay"); } - } - - private interface ReplayCallable { - void call(final @NotNull ReplayIntegration replay); + return false; } } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt index 6353e9dde8..94b0490f17 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt @@ -15,6 +15,7 @@ import io.sentry.android.core.internal.gestures.AndroidViewGestureTargetLocator import io.sentry.android.core.internal.modules.AssetsModulesLoader import io.sentry.android.core.internal.util.AndroidMainThreadChecker import io.sentry.android.fragment.FragmentLifecycleIntegration +import io.sentry.android.replay.ReplayIntegration import io.sentry.android.timber.SentryTimberIntegration import io.sentry.cache.PersistingOptionsObserver import io.sentry.cache.PersistingScopeObserver @@ -83,6 +84,7 @@ class AndroidOptionsInitializerTest { loadClass, activityFramesTracker, false, + false, false ) @@ -99,7 +101,8 @@ class AndroidOptionsInitializerTest { minApi: Int = Build.VERSION_CODES.KITKAT, classesToLoad: List = emptyList(), isFragmentAvailable: Boolean = false, - isTimberAvailable: Boolean = false + isTimberAvailable: Boolean = false, + isReplayAvailable: Boolean = false ) { mockContext = ContextUtilsTestHelper.mockMetaData( mockContext = ContextUtilsTestHelper.createMockContext(hasAppContext = true), @@ -126,7 +129,8 @@ class AndroidOptionsInitializerTest { loadClass, activityFramesTracker, isFragmentAvailable, - isTimberAvailable + isTimberAvailable, + isReplayAvailable ) AndroidOptionsInitializer.initializeIntegrationsAndProcessors( @@ -478,6 +482,24 @@ class AndroidOptionsInitializerTest { assertNull(actual) } + @Test + fun `ReplayIntegration added to the integration list if available on classpath`() { + fixture.initSutWithClassLoader(isReplayAvailable = true) + + val actual = + fixture.sentryOptions.integrations.firstOrNull { it is ReplayIntegration } + assertNotNull(actual) + } + + @Test + fun `ReplayIntegration won't be enabled, it throws class not found`() { + fixture.initSutWithClassLoader(isReplayAvailable = false) + + val actual = + fixture.sentryOptions.integrations.firstOrNull { it is ReplayIntegration } + assertNull(actual) + } + @Test fun `AndroidEnvelopeCache is set to options`() { fixture.initSut() @@ -634,6 +656,7 @@ class AndroidOptionsInitializerTest { mock(), mock(), false, + false, false ) verify(mockOptions, never()).outboxPath diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt index a726b2c55b..2efb602075 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt @@ -118,6 +118,7 @@ class AndroidProfilerTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt index 405aa6dc98..9376ea79fb 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt @@ -125,6 +125,7 @@ class AndroidTransactionProfilerTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt index be30993142..4b620813bf 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt @@ -123,7 +123,6 @@ class LifecycleWatcherTest { fun `When session tracking is disabled, do not end session`() { val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) watcher.onStop(fixture.ownerMock) - assertNull(watcher.timerTask) verify(fixture.hub, never()).endSession() } @@ -167,7 +166,6 @@ class LifecycleWatcherTest { fun `When session tracking is disabled, do not add breadcrumb on stop`() { val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) watcher.onStop(fixture.ownerMock) - assertNull(watcher.timerTask) verify(fixture.hub, never()).addBreadcrumb(any()) } @@ -219,12 +217,6 @@ class LifecycleWatcherTest { assertNotNull(watcher.timer) } - @Test - fun `timer is not created if session tracking is disabled`() { - val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) - assertNull(watcher.timer) - } - @Test fun `if the hub has already a fresh session running, don't start new one`() { val watcher = fixture.getSUT( diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt index bd5b3695fb..b543ae318a 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt @@ -26,6 +26,8 @@ import io.sentry.UncaughtExceptionHandlerIntegration import io.sentry.android.core.cache.AndroidEnvelopeCache import io.sentry.android.core.performance.AppStartMetrics import io.sentry.android.fragment.FragmentLifecycleIntegration +import io.sentry.android.replay.ReplayIntegration +import io.sentry.android.replay.getReplayIntegration import io.sentry.android.timber.SentryTimberIntegration import io.sentry.cache.IEnvelopeCache import io.sentry.cache.PersistingOptionsObserver @@ -313,12 +315,26 @@ class SentryAndroidTest { } } + @Test + @Config(sdk = [26]) + fun `init starts session replay if app is in foreground`() { + initSentryWithForegroundImportance(true) { _ -> + assertTrue(Sentry.getCurrentHub().getReplayIntegration()!!.isRecording()) + } + } + + @Test + @Config(sdk = [26]) + fun `init does not start session replay if the app is in background`() { + initSentryWithForegroundImportance(false) { _ -> + assertFalse(Sentry.getCurrentHub().getReplayIntegration()!!.isRecording()) + } + } + private fun initSentryWithForegroundImportance( inForeground: Boolean, callback: (session: Session?) -> Unit ) { - val context = ContextUtilsTestHelper.createMockContext() - Mockito.mockStatic(ContextUtils::class.java).use { mockedContextUtils -> mockedContextUtils.`when` { ContextUtils.isForegroundImportance() } .thenReturn(inForeground) @@ -412,7 +428,7 @@ class SentryAndroidTest { fixture.initSut(context = mock()) { options -> optionsRef = options options.dsn = "https://key@sentry.io/123" - assertEquals(19, options.integrations.size) + assertEquals(20, options.integrations.size) options.integrations.removeAll { it is UncaughtExceptionHandlerIntegration || it is ShutdownHookIntegration || @@ -431,7 +447,8 @@ class SentryAndroidTest { it is SystemEventsBreadcrumbsIntegration || it is NetworkBreadcrumbsIntegration || it is TempSensorBreadcrumbsIntegration || - it is PhoneStateBreadcrumbsIntegration + it is PhoneStateBreadcrumbsIntegration || + it is ReplayIntegration } } assertEquals(0, optionsRef.integrations.size) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt index a83076efb0..5b546523d0 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt @@ -143,6 +143,7 @@ class SentryInitProviderTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-replay/api/sentry-android-replay.api b/sentry-android-replay/api/sentry-android-replay.api index 5af2ca943b..b3bc98e15b 100644 --- a/sentry-android-replay/api/sentry-android-replay.api +++ b/sentry-android-replay/api/sentry-android-replay.api @@ -12,6 +12,7 @@ public final class io/sentry/android/replay/ReplayIntegration : io/sentry/Integr public static final field VIDEO_SEGMENT_DURATION J public fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V public fun close ()V + public final fun isRecording ()Z public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V public final fun pause ()V public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index 756b8638d0..e1f0d2f7e3 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -99,6 +99,8 @@ class ReplayIntegration( isEnabled.set(true) } + fun isRecording() = isRecording.get() + fun start() { if (!isEnabled.get()) { options.logger.log( From bee240b8608a9ee08c6920e4dd38eed79d4294ac Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Mon, 25 Mar 2024 15:01:05 +0100 Subject: [PATCH 07/19] Call listeners when installing RootViewsSpy --- .../src/main/java/io/sentry/android/replay/Windows.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt index ece684c46e..e9c6761c75 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt @@ -139,6 +139,15 @@ internal class RootViewsSpy private constructor() { val listeners = CopyOnWriteArrayList() private val delegatingViewList = object : ArrayList() { + override fun addAll(elements: Collection): Boolean { + listeners.forEach { listener -> + elements.forEach { element -> + listener.onRootViewsChanged(element, true) + } + } + return super.addAll(elements) + } + override fun add(element: View): Boolean { listeners.forEach { it.onRootViewsChanged(element, true) } return super.add(element) From 5faeb4ef0594c501af51bcb54d9147c3ff2ae0c2 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Thu, 28 Mar 2024 14:00:37 +0100 Subject: [PATCH 08/19] Better error handling --- .../android/replay/ScreenshotRecorder.kt | 118 ++++++++++-------- 1 file changed, 67 insertions(+), 51 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 87915f8505..df161551a4 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -17,6 +17,7 @@ import android.view.ViewGroup import android.view.ViewTreeObserver import io.sentry.SentryLevel.DEBUG import io.sentry.SentryLevel.INFO +import io.sentry.SentryLevel.WARNING import io.sentry.SentryOptions import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode import java.lang.ref.WeakReference @@ -72,7 +73,12 @@ internal class ScreenshotRecorder( return } - val window = root.phoneWindow ?: return + val window = root.phoneWindow + if (window == null) { + options.logger.log(DEBUG, "Window is invalid, not capturing screenshot") + return + } + val bitmap = Bitmap.createBitmap( root.width, root.height, @@ -88,59 +94,69 @@ internal class ScreenshotRecorder( } options.logger.log(DEBUG, "Took %d ms to capture view hierarchy", time) - PixelCopy.request( - window, - bitmap, - { copyResult: Int -> - if (copyResult != PixelCopy.SUCCESS) { - options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult) - return@request - } - - val viewHierarchy = bitmapToVH[bitmap] - val scaledBitmap: Bitmap - - if (viewHierarchy == null) { - options.logger.log(INFO, "Failed to determine view hierarchy, not capturing") - return@request - } else { - scaledBitmap = Bitmap.createScaledBitmap( - bitmap, - config.recordingWidth, - config.recordingHeight, - true - ) - val canvas = Canvas(scaledBitmap) - canvas.setMatrix(prescaledMatrix) - viewHierarchy.traverse { - if (it.shouldRedact && (it.width > 0 && it.height > 0)) { - it.visibleRect ?: return@traverse - - // TODO: check for view type rather than rely on absence of dominantColor here - val color = if (it.dominantColor == null) { - singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) - singlePixelBitmap.getPixel(0, 0) - } else { - it.dominantColor - } + try { + PixelCopy.request( + window, + bitmap, + { copyResult: Int -> + if (copyResult != PixelCopy.SUCCESS) { + options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult) + bitmap.recycle() + bitmapToVH.remove(bitmap) + return@request + } - maskingPaint.setColor(color) - canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) + val viewHierarchy = bitmapToVH[bitmap] + val scaledBitmap: Bitmap + + if (viewHierarchy == null) { + options.logger.log(INFO, "Failed to determine view hierarchy, not capturing") + bitmap.recycle() + bitmapToVH.remove(bitmap) + return@request + } else { + scaledBitmap = Bitmap.createScaledBitmap( + bitmap, + config.recordingWidth, + config.recordingHeight, + true + ) + val canvas = Canvas(scaledBitmap) + canvas.setMatrix(prescaledMatrix) + viewHierarchy.traverse { + if (it.shouldRedact && (it.width > 0 && it.height > 0)) { + it.visibleRect ?: return@traverse + + // TODO: check for view type rather than rely on absence of dominantColor here + val color = if (it.dominantColor == null) { + singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) + singlePixelBitmap.getPixel(0, 0) + } else { + it.dominantColor + } + + maskingPaint.setColor(color) + canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) + } } } - } - - val screenshot = scaledBitmap.copy(ARGB_8888, false) - screenshotRecorderCallback.onScreenshotRecorded(screenshot) - lastScreenshot = screenshot - contentChanged.set(false) - - scaledBitmap.recycle() - bitmap.recycle() - bitmapToVH.remove(bitmap) - }, - handler - ) + + val screenshot = scaledBitmap.copy(ARGB_8888, false) + screenshotRecorderCallback.onScreenshotRecorded(screenshot) + lastScreenshot = screenshot + contentChanged.set(false) + + scaledBitmap.recycle() + bitmap.recycle() + bitmapToVH.remove(bitmap) + }, + handler + ) + } catch (e: Throwable) { + options.logger.log(WARNING, "Failed to capture replay recording", e) + bitmap.recycle() + bitmapToVH.remove(bitmap) + } } } From 65d35ececa849a9a268431632b7555a3f183b8fa Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Thu, 28 Mar 2024 14:03:01 +0100 Subject: [PATCH 09/19] recycler lastScreenshot before re-assigning --- .../src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index df161551a4..66f1ede82d 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -143,6 +143,7 @@ internal class ScreenshotRecorder( val screenshot = scaledBitmap.copy(ARGB_8888, false) screenshotRecorderCallback.onScreenshotRecorded(screenshot) + lastScreenshot?.recycle() lastScreenshot = screenshot contentChanged.set(false) From 0f4e718fc20c2001efed3ac4ec1f1947f4366a46 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Thu, 28 Mar 2024 16:59:32 +0100 Subject: [PATCH 10/19] Expose ReplayCache as public api --- .../api/sentry-android-replay.api | 41 +++++++++++++++++++ .../io/sentry/android/replay/ReplayCache.kt | 32 +++++++++++---- .../android/replay/ScreenshotRecorder.kt | 2 +- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/sentry-android-replay/api/sentry-android-replay.api b/sentry-android-replay/api/sentry-android-replay.api index b3bc98e15b..7f45c6d8f7 100644 --- a/sentry-android-replay/api/sentry-android-replay.api +++ b/sentry-android-replay/api/sentry-android-replay.api @@ -6,6 +6,29 @@ public final class io/sentry/android/replay/BuildConfig { public fun ()V } +public final class io/sentry/android/replay/GeneratedVideo { + public fun (Ljava/io/File;IJ)V + public final fun component1 ()Ljava/io/File; + public final fun component2 ()I + public final fun component3 ()J + public final fun copy (Ljava/io/File;IJ)Lio/sentry/android/replay/GeneratedVideo; + public static synthetic fun copy$default (Lio/sentry/android/replay/GeneratedVideo;Ljava/io/File;IJILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo; + public fun equals (Ljava/lang/Object;)Z + public final fun getDuration ()J + public final fun getFrameCount ()I + public final fun getVideo ()Ljava/io/File; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class io/sentry/android/replay/ReplayCache : java/io/Closeable { + public fun (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;Lio/sentry/android/replay/ScreenshotRecorderConfig;)V + public final fun addFrame (Ljava/io/File;J)V + public fun close ()V + public final fun createVideoOf (JJILjava/io/File;)Lio/sentry/android/replay/GeneratedVideo; + public static synthetic fun createVideoOf$default (Lio/sentry/android/replay/ReplayCache;JJILjava/io/File;ILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo; +} + public final class io/sentry/android/replay/ReplayIntegration : io/sentry/Integration, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable { public static final field Companion Lio/sentry/android/replay/ReplayIntegration$Companion; public static final field VIDEO_BUFFER_DURATION J @@ -33,6 +56,24 @@ public abstract interface class io/sentry/android/replay/ScreenshotRecorderCallb public abstract fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V } +public final class io/sentry/android/replay/ScreenshotRecorderConfig { + public fun (IIFI)V + public synthetic fun (IIFIILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()I + public final fun component2 ()I + public final fun component3 ()F + public final fun component4 ()I + public final fun copy (IIFI)Lio/sentry/android/replay/ScreenshotRecorderConfig; + public static synthetic fun copy$default (Lio/sentry/android/replay/ScreenshotRecorderConfig;IIFIILjava/lang/Object;)Lio/sentry/android/replay/ScreenshotRecorderConfig; + public fun equals (Ljava/lang/Object;)Z + public final fun getFrameRate ()I + public final fun getRecordingHeight ()I + public final fun getRecordingWidth ()I + public final fun getScaleFactor ()F + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer { public abstract fun getVideoTime ()J public abstract fun isStarted ()Z diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt index da62dfb6b1..7be5e901dc 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -13,11 +13,18 @@ import io.sentry.protocol.SentryId import java.io.Closeable import java.io.File -internal class ReplayCache( +public class ReplayCache internal constructor( private val options: SentryOptions, private val replayId: SentryId, private val recorderConfig: ScreenshotRecorderConfig, - private val encoderCreator: (File) -> SimpleVideoEncoder = { videoFile -> + private val encoderCreator: (File) -> SimpleVideoEncoder +) : Closeable { + + public constructor( + options: SentryOptions, + replayId: SentryId, + recorderConfig: ScreenshotRecorderConfig + ) : this(options, replayId, recorderConfig, encoderCreator = { videoFile -> SimpleVideoEncoder( options, MuxerConfig( @@ -27,11 +34,10 @@ internal class ReplayCache( bitrate = 20 * 1000 ) ).also { it.start() } - } -) : Closeable { + }) private val encoderLock = Any() - internal var encoder: SimpleVideoEncoder? = null + private var encoder: SimpleVideoEncoder? = null internal val replayCacheDir: File? by lazy { if (options.cacheDirPath.isNullOrEmpty()) { @@ -48,7 +54,7 @@ internal class ReplayCache( // TODO: maybe account for multi-threaded access internal val frames = mutableListOf() - fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { + internal fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { if (replayCacheDir == null) { return } @@ -61,11 +67,20 @@ internal class ReplayCache( it.flush() } + addFrame(screenshot, frameTimestamp) + } + + public fun addFrame(screenshot: File, frameTimestamp: Long) { val frame = ReplayFrame(screenshot, frameTimestamp) frames += frame } - fun createVideoOf(duration: Long, from: Long, segmentId: Int): GeneratedVideo? { + public fun createVideoOf( + duration: Long, + from: Long, + segmentId: Int, + videoFile: File = File(replayCacheDir, "$segmentId.mp4") + ): GeneratedVideo? { if (frames.isEmpty()) { options.logger.log( DEBUG, @@ -75,7 +90,6 @@ internal class ReplayCache( } // TODO: reuse instance of encoder and just change file path to create a different muxer - val videoFile = File(replayCacheDir, "$segmentId.mp4") encoder = synchronized(encoderLock) { encoderCreator(videoFile) } val step = 1000 / recorderConfig.frameRate.toLong() @@ -160,7 +174,7 @@ internal data class ReplayFrame( val timestamp: Long ) -internal data class GeneratedVideo( +public data class GeneratedVideo( val video: File, val frameCount: Int, val duration: Long diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 66f1ede82d..16f2544918 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -230,7 +230,7 @@ internal class ScreenshotRecorder( } } -internal data class ScreenshotRecorderConfig( +public data class ScreenshotRecorderConfig( val recordingWidth: Int, val recordingHeight: Int, val scaleFactor: Float, From fd6e6333944bf921fd056ef00fb568688fd806ce Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Thu, 28 Mar 2024 19:40:30 +0100 Subject: [PATCH 11/19] Fix redacting out of sync --- .../android/replay/ScreenshotRecorder.kt | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 16f2544918..b403ceb4fa 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -21,8 +21,8 @@ import io.sentry.SentryLevel.WARNING import io.sentry.SentryOptions import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode import java.lang.ref.WeakReference -import java.util.WeakHashMap import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference import kotlin.system.measureTimeMillis @TargetApi(26) @@ -35,7 +35,7 @@ internal class ScreenshotRecorder( private var rootView: WeakReference? = null private val thread = HandlerThread("SentryReplayRecorder").also { it.start() } private val handler = Handler(thread.looper) - private val bitmapToVH = WeakHashMap() + private val pendingViewHierarchy = AtomicReference() private val maskingPaint = Paint() private val singlePixelBitmap: Bitmap = Bitmap.createBitmap( 1, @@ -51,6 +51,8 @@ internal class ScreenshotRecorder( private var lastScreenshot: Bitmap? = null fun capture() { + val viewHierarchy = pendingViewHierarchy.get() + if (!isCapturing.get()) { options.logger.log(DEBUG, "ScreenshotRecorder is paused, not capturing screenshot") return @@ -87,13 +89,6 @@ internal class ScreenshotRecorder( // postAtFrontOfQueue to ensure the view hierarchy and bitmap are ase close in-sync as possible Handler(Looper.getMainLooper()).postAtFrontOfQueue { - val time = measureTimeMillis { - val rootNode = ViewHierarchyNode.fromView(root) - root.traverse(rootNode) - bitmapToVH[bitmap] = rootNode - } - options.logger.log(DEBUG, "Took %d ms to capture view hierarchy", time) - try { PixelCopy.request( window, @@ -102,17 +97,14 @@ internal class ScreenshotRecorder( if (copyResult != PixelCopy.SUCCESS) { options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult) bitmap.recycle() - bitmapToVH.remove(bitmap) return@request } - val viewHierarchy = bitmapToVH[bitmap] val scaledBitmap: Bitmap if (viewHierarchy == null) { options.logger.log(INFO, "Failed to determine view hierarchy, not capturing") bitmap.recycle() - bitmapToVH.remove(bitmap) return@request } else { scaledBitmap = Bitmap.createScaledBitmap( @@ -149,19 +141,30 @@ internal class ScreenshotRecorder( scaledBitmap.recycle() bitmap.recycle() - bitmapToVH.remove(bitmap) }, handler ) } catch (e: Throwable) { options.logger.log(WARNING, "Failed to capture replay recording", e) bitmap.recycle() - bitmapToVH.remove(bitmap) } } } override fun onDraw() { + val root = rootView?.get() + if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) { + options.logger.log(DEBUG, "Root view is invalid, not capturing screenshot") + return + } + + val time = measureTimeMillis { + val rootNode = ViewHierarchyNode.fromView(root) + root.traverse(rootNode) + pendingViewHierarchy.set(rootNode) + } + options.logger.log(DEBUG, "Took %d ms to capture view hierarchy", time) + contentChanged.set(true) } @@ -194,7 +197,7 @@ internal class ScreenshotRecorder( unbind(rootView?.get()) rootView?.clear() lastScreenshot?.recycle() - bitmapToVH.clear() + pendingViewHierarchy.set(null) isCapturing.set(false) thread.quitSafely() } From 18eb67ebf6a7ea188175fb819fe67c7a76d7a561 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 22:13:32 +0200 Subject: [PATCH 12/19] Add more tests --- .../sentry/android/replay/ReplayCacheTest.kt | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt index 88d410fab1..fab5f28ac8 100644 --- a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt @@ -1,6 +1,7 @@ package io.sentry.android.replay import android.graphics.Bitmap +import android.graphics.Bitmap.CompressFormat.JPEG import android.graphics.Bitmap.Config.ARGB_8888 import android.media.MediaCodec import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -123,8 +124,15 @@ class ReplayCacheTest { val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 1001) + replayCache.addFrame(bitmap, 2001) + + val segment0 = replayCache.createVideoOf(3000L, 0, 0) + assertEquals(3, segment0!!.frameCount) + assertEquals(3000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) - replayCache.createVideoOf(3000L, 0, 0) assertTrue(replayCache.frames.isEmpty()) assertTrue(replayCache.replayCacheDir!!.listFiles()!!.none { it.extension == "jpg" }) } @@ -143,6 +151,7 @@ class ReplayCacheTest { val segment0 = replayCache.createVideoOf(5000L, 0, 0) assertEquals(5, segment0!!.frameCount) assertEquals(5000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) } @@ -161,6 +170,7 @@ class ReplayCacheTest { val segment0 = replayCache.createVideoOf(5000L, 0, 0) assertEquals(5, segment0!!.frameCount) assertEquals(5000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) } @@ -184,6 +194,7 @@ class ReplayCacheTest { val segment1 = replayCache.createVideoOf(5000L, 5000L, 1) assertEquals(5, segment1!!.frameCount) assertEquals(5000, segment1.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } assertEquals(File(replayCache.replayCacheDir, "1.mp4"), segment1.video) } @@ -203,6 +214,34 @@ class ReplayCacheTest { val segment0 = replayCache.createVideoOf(3000L, 0, 0) assertEquals(6, segment0!!.frameCount) assertEquals(3000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) } + + @Test + fun `addFrame with File path works`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val flutterCacheDir = + File(fixture.options.cacheDirPath!!, "flutter_replay").also { it.mkdirs() } + val screenshot = File(flutterCacheDir, "1.jpg").also { it.createNewFile() } + val video = File(flutterCacheDir, "flutter_0.mp4") + + screenshot.outputStream().use { + Bitmap.createBitmap(1, 1, ARGB_8888).compress(JPEG, 80, it) + it.flush() + } + replayCache.addFrame(screenshot, frameTimestamp = 1) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0, videoFile = video) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(flutterCacheDir, "flutter_0.mp4"), segment0.video) + } } From a7ae2b7e5ec44b0a971157dd8acce1041df605a7 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 22:13:59 +0200 Subject: [PATCH 13/19] Improve ReplayCache logic --- .../io/sentry/android/replay/ReplayCache.kt | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt index 7be5e901dc..f5c8fc715d 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -97,22 +97,23 @@ public class ReplayCache internal constructor( var lastFrame: ReplayFrame = frames.first() for (timestamp in from until (from + (duration)) step step) { val iter = frames.iterator() - val frameCountBefore = frameCount while (iter.hasNext()) { val frame = iter.next() if (frame.timestamp in (timestamp..timestamp + step)) { - frameCount++ - encode(frame) lastFrame = frame break // we only support 1 frame per given interval } + + // assuming frames are in order, if out of bounds exit early + if (frame.timestamp > timestamp + step) { + break + } } - // if the frame count hasn't changed we just replicate the last known frame to respect - // the video duration. - if (frameCountBefore == frameCount) { + // we either encode a new frame within the step bounds or replicate the last known frame + // to respect the video duration + if (encode(lastFrame)) { frameCount++ - encode(lastFrame) } } @@ -143,12 +144,18 @@ public class ReplayCache internal constructor( return GeneratedVideo(videoFile, frameCount, videoDuration) } - private fun encode(frame: ReplayFrame) { - val bitmap = BitmapFactory.decodeFile(frame.screenshot.absolutePath) - synchronized(encoderLock) { - encoder?.encode(bitmap) + private fun encode(frame: ReplayFrame): Boolean { + return try { + val bitmap = BitmapFactory.decodeFile(frame.screenshot.absolutePath) + synchronized(encoderLock) { + encoder?.encode(bitmap) + } + bitmap.recycle() + true + } catch (e: Throwable) { + options.logger.log(WARNING, "Unable to decode bitmap and encode it into a video, skipping frame", e) + false } - bitmap.recycle() } private fun deleteFile(file: File) { From bf14d83367dfb6c7e40a3cfe24db59380ed6ae09 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 22:14:11 +0200 Subject: [PATCH 14/19] frameUsec -> frameDurationUsec --- .../io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt index 8a21b0bec0..cf30f9e49f 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt @@ -38,7 +38,7 @@ import java.util.concurrent.TimeUnit.MICROSECONDS import java.util.concurrent.TimeUnit.MILLISECONDS class SimpleMp4FrameMuxer(path: String, fps: Float) : SimpleFrameMuxer { - private val frameUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() + private val frameDurationUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() private val muxer: MediaMuxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) @@ -59,7 +59,7 @@ class SimpleMp4FrameMuxer(path: String, fps: Float) : SimpleFrameMuxer { // This code will break if the encoder supports B frames. // Ideally we would use set the value in the encoder, // don't know how to do that without using OpenGL - finalVideoTime = frameUsec * videoFrames++ + finalVideoTime = frameDurationUsec * videoFrames++ bufferInfo.presentationTimeUs = finalVideoTime // encodedData.position(bufferInfo.offset) @@ -78,6 +78,6 @@ class SimpleMp4FrameMuxer(path: String, fps: Float) : SimpleFrameMuxer { return 0 } // have to add one sec as we calculate it 0-based above - return MILLISECONDS.convert(finalVideoTime + frameUsec, MICROSECONDS) + return MILLISECONDS.convert(finalVideoTime + frameDurationUsec, MICROSECONDS) } } From 82fe21ad444ffda8e6aa788d4d035df6df60182e Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 22:15:34 +0200 Subject: [PATCH 15/19] bottom/right -> height/width --- .../src/main/java/io/sentry/android/replay/ReplayIntegration.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index e1f0d2f7e3..b827751477 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -73,7 +73,7 @@ class ReplayIntegration( } private val aspectRatio by lazy(NONE) { - screenBounds.bottom.toFloat() / screenBounds.right.toFloat() + screenBounds.height().toFloat() / screenBounds.width().toFloat() } private val recorderConfig by lazy(NONE) { From de56e35a53b4e74bfac3f33c5a223a27edb87b5b Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 23:55:54 +0200 Subject: [PATCH 16/19] add todos --- .../main/java/io/sentry/android/replay/ReplayIntegration.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index b827751477..6d0b9ca0dd 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -102,6 +102,7 @@ class ReplayIntegration( fun isRecording() = isRecording.get() fun start() { + // TODO: add lifecycle state instead and manage it in start/pause/resume/stop if (!isEnabled.get()) { options.logger.log( DEBUG, @@ -124,11 +125,13 @@ class ReplayIntegration( cache = ReplayCache(options, currentReplayId.get(), recorderConfig) recorder?.startRecording() + // TODO: replace it with dateProvider.currentTimeMillis to also test it segmentTimestamp.set(DateUtils.getCurrentDateTime()) // TODO: finalize old recording if there's some left on disk and send it using the replayId from persisted scope (e.g. for ANRs) } fun resume() { + // TODO: replace it with dateProvider.currentTimeMillis to also test it segmentTimestamp.set(DateUtils.getCurrentDateTime()) recorder?.resume() } From fa8c5271abd8cfa0320bf28c55ea362a431b4db0 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Tue, 2 Apr 2024 23:58:29 +0200 Subject: [PATCH 17/19] duration -> durationMs --- .../sentry/android/replay/ReplayIntegration.kt | 2 +- sentry/api/sentry.api | 4 ++-- .../java/io/sentry/rrweb/RRWebVideoEvent.java | 18 +++++++++--------- .../rrweb/RRWebVideoEventSerializationTest.kt | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index 6d0b9ca0dd..81f0b605da 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -265,7 +265,7 @@ class ReplayIntegration( RRWebVideoEvent().apply { this.timestamp = segmentTimestamp.time this.segmentId = segmentId - this.duration = duration + this.durationMs = duration this.frameCount = frameCount size = video.length() frameRate = recorderConfig.frameRate diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index 9e3a90946f..3ffc0cda8f 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -4773,7 +4773,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun equals (Ljava/lang/Object;)Z public fun getContainer ()Ljava/lang/String; public fun getDataUnknown ()Ljava/util/Map; - public fun getDuration ()J + public fun getDurationMs ()J public fun getEncoding ()Ljava/lang/String; public fun getFrameCount ()I public fun getFrameRate ()I @@ -4791,7 +4791,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setContainer (Ljava/lang/String;)V public fun setDataUnknown (Ljava/util/Map;)V - public fun setDuration (J)V + public fun setDurationMs (J)V public fun setEncoding (Ljava/lang/String;)V public fun setFrameCount (I)V public fun setFrameRate (I)V diff --git a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java index 532177ff9f..1ba9f19c72 100644 --- a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java +++ b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java @@ -26,7 +26,7 @@ public final class RRWebVideoEvent extends RRWebEvent implements JsonUnknown, Js private @NotNull String tag; private int segmentId; private long size; - private long duration; + private long durationMs; private @NotNull String encoding = REPLAY_ENCODING; private @NotNull String container = REPLAY_CONTAINER; private int height; @@ -72,12 +72,12 @@ public void setSize(final long size) { this.size = size; } - public long getDuration() { - return duration; + public long getDurationMs() { + return durationMs; } - public void setDuration(final long duration) { - this.duration = duration; + public void setDurationMs(final long durationMs) { + this.durationMs = durationMs; } @NotNull @@ -189,7 +189,7 @@ public boolean equals(Object o) { RRWebVideoEvent that = (RRWebVideoEvent) o; return segmentId == that.segmentId && size == that.size - && duration == that.duration + && durationMs == that.durationMs && height == that.height && width == that.width && frameCount == that.frameCount @@ -209,7 +209,7 @@ public int hashCode() { tag, segmentId, size, - duration, + durationMs, encoding, container, height, @@ -279,7 +279,7 @@ private void serializePayload(final @NotNull ObjectWriter writer, final @NotNull writer.beginObject(); writer.name(JsonKeys.SEGMENT_ID).value(segmentId); writer.name(JsonKeys.SIZE).value(size); - writer.name(JsonKeys.DURATION).value(duration); + writer.name(JsonKeys.DURATION).value(durationMs); writer.name(JsonKeys.ENCODING).value(encoding); writer.name(JsonKeys.CONTAINER).value(container); writer.name(JsonKeys.HEIGHT).value(height); @@ -380,7 +380,7 @@ private void deserializePayload( event.size = size == null ? 0 : size; break; case JsonKeys.DURATION: - event.duration = reader.nextLong(); + event.durationMs = reader.nextLong(); break; case JsonKeys.CONTAINER: final String container = reader.nextStringOrNull(); diff --git a/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt b/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt index 79bfd02456..17a790b5cd 100644 --- a/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt +++ b/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt @@ -17,7 +17,7 @@ class RRWebVideoEventSerializationTest { tag = "video" segmentId = 0 size = 4_000_000L - duration = 5000 + durationMs = 5000 height = 1920 width = 1080 frameCount = 5 From dfbb9927fd7661cd9f9e405ef998084eb4d75c26 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Wed, 3 Apr 2024 00:13:34 +0200 Subject: [PATCH 18/19] replaId non-nullable --- .../main/java/io/sentry/android/replay/ReplayIntegration.kt | 6 +++--- sentry/src/main/java/io/sentry/IScope.java | 4 ++-- sentry/src/main/java/io/sentry/NoOpScope.java | 4 ++-- sentry/src/main/java/io/sentry/Scope.java | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt index 81f0b605da..e2244d0bf3 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -53,7 +53,7 @@ class ReplayIntegration( // TODO: probably not everything has to be thread-safe here private val isEnabled = AtomicBoolean(false) private val isRecording = AtomicBoolean(false) - private val currentReplayId = AtomicReference() + private val currentReplayId = AtomicReference(SentryId.EMPTY_ID) private val segmentTimestamp = AtomicReference() private val currentSegment = AtomicInteger(0) private val saver = @@ -177,8 +177,8 @@ class ReplayIntegration( cache?.close() currentSegment.set(0) segmentTimestamp.set(null) - currentReplayId.set(null) - hub?.configureScope { it.replayId = null } + currentReplayId.set(SentryId.EMPTY_ID) + hub?.configureScope { it.replayId = SentryId.EMPTY_ID } isRecording.set(false) } diff --git a/sentry/src/main/java/io/sentry/IScope.java b/sentry/src/main/java/io/sentry/IScope.java index 2d38371ead..4b5930bb54 100644 --- a/sentry/src/main/java/io/sentry/IScope.java +++ b/sentry/src/main/java/io/sentry/IScope.java @@ -91,7 +91,7 @@ public interface IScope { * @return the id of the current session replay */ @ApiStatus.Internal - @Nullable + @NotNull SentryId getReplayId(); /** @@ -100,7 +100,7 @@ public interface IScope { * @param replayId the id of the current session replay */ @ApiStatus.Internal - void setReplayId(final @Nullable SentryId replayId); + void setReplayId(final @NotNull SentryId replayId); /** * Returns the Scope's request diff --git a/sentry/src/main/java/io/sentry/NoOpScope.java b/sentry/src/main/java/io/sentry/NoOpScope.java index 660dca0b69..d2be23eba8 100644 --- a/sentry/src/main/java/io/sentry/NoOpScope.java +++ b/sentry/src/main/java/io/sentry/NoOpScope.java @@ -70,8 +70,8 @@ public void setUser(@Nullable User user) {} public void setScreen(@Nullable String screen) {} @Override - public @Nullable SentryId getReplayId() { - return null; + public @NotNull SentryId getReplayId() { + return SentryId.EMPTY_ID; } @Override diff --git a/sentry/src/main/java/io/sentry/Scope.java b/sentry/src/main/java/io/sentry/Scope.java index 164d52dc2b..161502a9d3 100644 --- a/sentry/src/main/java/io/sentry/Scope.java +++ b/sentry/src/main/java/io/sentry/Scope.java @@ -82,7 +82,7 @@ public final class Scope implements IScope { private @NotNull PropagationContext propagationContext; /** Scope's session replay id */ - private @Nullable SentryId replayId; + private @NotNull SentryId replayId = SentryId.EMPTY_ID; /** * Scope's ctor @@ -318,12 +318,12 @@ public void setScreen(final @Nullable String screen) { } @Override - public @Nullable SentryId getReplayId() { + public @NotNull SentryId getReplayId() { return replayId; } @Override - public void setReplayId(final @Nullable SentryId replayId) { + public void setReplayId(final @NotNull SentryId replayId) { this.replayId = replayId; // TODO: set to contexts and notify observers to persist this as well From 27b15d7235d50cef4c25c546acf58a8929287274 Mon Sep 17 00:00:00 2001 From: Roman Zavarnitsyn Date: Wed, 3 Apr 2024 15:38:49 +0200 Subject: [PATCH 19/19] Add kdoc --- .../io/sentry/android/replay/ReplayCache.kt | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt index f5c8fc715d..fd07d74354 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -13,6 +13,19 @@ import io.sentry.protocol.SentryId import java.io.Closeable import java.io.File +/** + * A basic in-memory and disk cache for Session Replay frames. Frames are stored in order under the + * [SentryOptions.cacheDirPath] + [replayId] folder. The class is also capable of creating an mp4 + * video segment out of the stored frames, provided start time and duration using the available + * on-device [android.media.MediaCodec]. + * + * This class is not thread-safe, meaning, [addFrame] cannot be called concurrently with + * [createVideoOf], and they should be invoked from the same thread. + * + * @param options SentryOptions instance, used for logging and cacheDir + * @param replayId the current replay id, used for giving a unique name to the replay folder + * @param recorderConfig ScreenshotRecorderConfig, used for video resolution and frame-rate + */ public class ReplayCache internal constructor( private val options: SentryOptions, private val replayId: SentryId, @@ -54,6 +67,16 @@ public class ReplayCache internal constructor( // TODO: maybe account for multi-threaded access internal val frames = mutableListOf() + /** + * Stores the current frame screenshot to in-memory cache as well as disk with [frameTimestamp] + * as filename. Uses [Bitmap.CompressFormat.JPEG] format with quality 80. The frames are stored + * under [replayCacheDir]. + * + * This method is not thread-safe. + * + * @param bitmap the frame screenshot + * @param frameTimestamp the timestamp when the frame screenshot was taken + */ internal fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { if (replayCacheDir == null) { return @@ -70,11 +93,36 @@ public class ReplayCache internal constructor( addFrame(screenshot, frameTimestamp) } + /** + * Same as [addFrame], but accepts frame screenshot as [File], the file should contain + * a bitmap/image by the time [createVideoOf] is invoked. + * + * This method is not thread-safe. + * + * @param screenshot file containing the frame screenshot + * @param frameTimestamp the timestamp when the frame screenshot was taken + */ public fun addFrame(screenshot: File, frameTimestamp: Long) { val frame = ReplayFrame(screenshot, frameTimestamp) frames += frame } + /** + * Creates a video out of currently stored [frames] given the start time and duration using the + * on-device codecs [android.media.MediaCodec]. The generated video will be stored in + * [videoFile] location, which defaults to "[replayCacheDir]/[segmentId].mp4". + * + * This method is not thread-safe. + * + * @param duration desired video duration in milliseconds + * @param from desired start of the video represented as unix timestamp in milliseconds + * @param segmentId current segment id, used for inferring the filename to store the + * result video under [replayCacheDir], e.g. "replay_/0.mp4", where segmentId=0 + * @param videoFile optional, location of the file to store the result video. If this is + * provided, [segmentId] from above is disregarded and not used. + * @return a generated video of type [GeneratedVideo], which contains the resulting video file + * location, frame count and duration in milliseconds. + */ public fun createVideoOf( duration: Long, from: Long,