From 772d778ab1742d0e712e48beff534f730586551c Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 7 Jan 2025 00:52:54 -0800 Subject: [PATCH] Fix/android/tts reinit for onstart (#411) * fix: corrects abandoned null android tts observer * fix: corrects null android TTS after destroy * Update guide/src/android-getting-started.md --------- Co-authored-by: Ian Wagner --- android/core/build.gradle | 3 + .../com/stadiamaps/ferrostar/core/Speech.kt | 52 ++++- .../ferrostar/core/AndroidTextToSpeechTest.kt | 182 ++++++++++++++++++ .../com/stadiamaps/ferrostar/MainActivity.kt | 14 +- android/gradle/libs.versions.toml | 11 +- guide/src/android-getting-started.md | 19 +- 6 files changed, 269 insertions(+), 12 deletions(-) create mode 100644 android/core/src/test/java/com/stadiamaps/ferrostar/core/AndroidTextToSpeechTest.kt diff --git a/android/core/build.gradle b/android/core/build.gradle index 44416701..dcbfd67a 100644 --- a/android/core/build.gradle +++ b/android/core/build.gradle @@ -55,6 +55,9 @@ dependencies { implementation 'net.java.dev.jna:jna:5.15.0@aar' testImplementation libs.junit + testImplementation libs.kotlinx.coroutines.test + testImplementation libs.mockk + testImplementation libs.turbine // These probably shouldn't have to be androidTestImplementation... see rant in // ValhallaCoreTest.kt diff --git a/android/core/src/main/java/com/stadiamaps/ferrostar/core/Speech.kt b/android/core/src/main/java/com/stadiamaps/ferrostar/core/Speech.kt index d1d6713b..acff9afa 100644 --- a/android/core/src/main/java/com/stadiamaps/ferrostar/core/Speech.kt +++ b/android/core/src/main/java/com/stadiamaps/ferrostar/core/Speech.kt @@ -3,6 +3,8 @@ package com.stadiamaps.ferrostar.core import android.content.Context import android.speech.tts.TextToSpeech import android.speech.tts.TextToSpeech.OnInitListener +import androidx.annotation.VisibleForTesting +import java.lang.ref.WeakReference import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -10,6 +12,7 @@ import kotlinx.coroutines.flow.update import uniffi.ferrostar.SpokenInstruction interface SpokenInstructionObserver { + /** * Handles spoken instructions as they are triggered. * @@ -39,6 +42,14 @@ interface AndroidTtsStatusListener { */ fun onTtsInitialized(tts: TextToSpeech?, status: Int) + /** + * Invoked when the [TextToSpeech] instance is shut down and released to nil. + * + * After this point you must initialize a new instance to use TTS by calling + * [AndroidTtsObserver.start]. + */ + fun onTtsShutdownAndRelease() + /** * Invoked whenever [TextToSpeech.speak] returns a status code other than [TextToSpeech.SUCCESS]. */ @@ -64,7 +75,8 @@ interface AndroidTtsStatusListener { */ class AndroidTtsObserver( context: Context, - engine: String? = null, + private val weakContext: WeakReference = WeakReference(context), + private val engine: String? = null, var statusObserver: AndroidTtsStatusListener? = null, ) : SpokenInstructionObserver, OnInitListener { companion object { @@ -73,6 +85,9 @@ class AndroidTtsObserver( private var _muteState: MutableStateFlow = MutableStateFlow(false) + private val context: Context? + get() = weakContext.get() + override fun setMuted(isMuted: Boolean) { _muteState.update { _ -> if (isMuted && tts?.isSpeaking == true) { @@ -84,7 +99,7 @@ class AndroidTtsObserver( override val muteState: StateFlow = _muteState.asStateFlow() - var tts: TextToSpeech? + var tts: TextToSpeech? = null private set /** @@ -104,8 +119,23 @@ class AndroidTtsObserver( val isInitializedSuccessfully: Boolean get() = initStatus == TextToSpeech.SUCCESS - init { - tts = TextToSpeech(context, this, engine) + /** + * Starts a new [TextToSpeech] instance. + * + * Except [onInit] to fire when the engine is ready. + */ + fun start(@VisibleForTesting injectedTts: TextToSpeech? = null) { + if (context == null) { + android.util.Log.e(TAG, "Context is null. Unable to start TTS.") + return + } + + if (tts != null) { + android.util.Log.e(TAG, "TTS engine is already initialized.") + return + } + + tts = injectedTts ?: TextToSpeech(context, this, engine) } /** @@ -114,8 +144,13 @@ class AndroidTtsObserver( * Fails silently if TTS is unavailable. */ override fun onSpokenInstructionTrigger(spokenInstruction: SpokenInstruction) { - val tts = tts - if (tts != null && isInitializedSuccessfully && !isMuted) { + if (tts == null) { + android.util.Log.e(TAG, "TTS engine is not initialized.") + return + } + val tts = tts ?: return + + if (isInitializedSuccessfully && !isMuted) { // In the future, someone may wish to parse SSML to get more natural utterances into TtsSpans. // Amazon Polly is generally the intended target for SSML on Android though. val status = @@ -148,11 +183,12 @@ class AndroidTtsObserver( /** * Shuts down the underlying [TextToSpeech] engine. * - * The instance will no longer be usable after this. This method should usually be called from an - * activity's onDestroy method. + * The instance will be shut down and released after this call. You must call [start] to use TTS + * again. If you call this method, ensure that you've handled an start again. */ fun shutdown() { tts?.shutdown() tts = null + statusObserver?.onTtsShutdownAndRelease() } } diff --git a/android/core/src/test/java/com/stadiamaps/ferrostar/core/AndroidTextToSpeechTest.kt b/android/core/src/test/java/com/stadiamaps/ferrostar/core/AndroidTextToSpeechTest.kt new file mode 100644 index 00000000..219e37a3 --- /dev/null +++ b/android/core/src/test/java/com/stadiamaps/ferrostar/core/AndroidTextToSpeechTest.kt @@ -0,0 +1,182 @@ +package com.stadiamaps.ferrostar.core + +import android.content.Context +import android.speech.tts.TextToSpeech +import android.util.Log +import app.cash.turbine.test +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.verify +import java.util.UUID +import junit.framework.TestCase.assertFalse +import junit.framework.TestCase.assertNotNull +import junit.framework.TestCase.assertNull +import junit.framework.TestCase.assertTrue +import kotlin.uuid.ExperimentalUuidApi +import kotlinx.coroutines.test.runTest +import org.junit.Before +import org.junit.Test +import uniffi.ferrostar.SpokenInstruction + +class AndroidTextToSpeechTest { + + private val engine: String = "ferrostar.test" + + private lateinit var context: Context + private lateinit var androidTts: AndroidTtsObserver + + @Before + fun setUp() { + mockkStatic(Log::class) + every { Log.e(any(), any()) } returns 0 + + context = mockk(relaxed = true) + androidTts = AndroidTtsObserver(context = context, engine = engine) + } + + @Test + fun testStart() { + val observer = mockk() + androidTts.statusObserver = observer + androidTts.start() + + assertNotNull(androidTts.tts) + } + + @Test + fun `test shutdown`() { + val tts = mockk(relaxed = true) + androidTts.start(tts) + androidTts.shutdown() + + verify { tts.shutdown() } + assertNull(androidTts.tts) + } + + @Test + fun `test setMuted while speaking`() = runTest { + val tts = mockk(relaxed = true) { every { isSpeaking } returns true } + androidTts.start(tts) + androidTts.setMuted(true) + + verify { tts.stop() } + + androidTts.muteState.test { assertTrue(awaitItem()) } + + androidTts.setMuted(false) + + androidTts.muteState.test { assertFalse(awaitItem()) } + } + + @Test + fun `test setMuted without speaking`() { + val tts = mockk(relaxed = true) { every { isSpeaking } returns false } + androidTts.start(tts) + androidTts.setMuted(true) + + verify(exactly = 0) { tts.stop() } + } + + @Test + fun `test stop and clear`() { + val tts = mockk(relaxed = true) + androidTts.start(tts) + androidTts.stopAndClearQueue() + + verify { tts.stop() } + } + + @OptIn(ExperimentalUuidApi::class) + @Test + fun `test on speak`() { + val tts = mockk(relaxed = true) + androidTts.start(tts) + androidTts.setMuted(false) + androidTts.onInit(TextToSpeech.SUCCESS) + + val uuid = UUID.randomUUID() + val instruction = + mockk { + every { text } returns "Hello, World!" + every { utteranceId } returns uuid + } + + androidTts.onSpokenInstructionTrigger(instruction) + + verify { tts.speak("Hello, World!", TextToSpeech.QUEUE_ADD, any(), uuid.toString()) } + } + + @OptIn(ExperimentalUuidApi::class) + @Test + fun `test on speak error`() { + val tts = mockk(relaxed = true) + every { tts.speak(any(), any(), any(), any()) } returns TextToSpeech.ERROR + + androidTts.start(tts) + androidTts.setMuted(false) + androidTts.onInit(TextToSpeech.SUCCESS) + + val observer = mockk() + every { observer.onTtsSpeakError(any(), any()) } returns Unit + + androidTts.statusObserver = observer + + val uuid = UUID.randomUUID() + val instruction = + mockk { + every { text } returns "Hello, World!" + every { utteranceId } returns uuid + } + + androidTts.onSpokenInstructionTrigger(instruction) + + verify { Log.e(any(), any()) } + verify { observer.onTtsSpeakError(uuid.toString(), TextToSpeech.ERROR) } + } + + @OptIn(ExperimentalUuidApi::class) + @Test + fun `test on speech while muted`() { + val tts = mockk(relaxed = true) + androidTts.start(tts) + androidTts.setMuted(true) + androidTts.onInit(TextToSpeech.SUCCESS) + + val uuid = UUID.randomUUID() + val instruction = + mockk { + every { text } returns "Hello, World!" + every { utteranceId } returns uuid + } + + androidTts.onSpokenInstructionTrigger(instruction) + + verify(exactly = 0) { + tts.speak("Hello, World!", TextToSpeech.QUEUE_ADD, any(), uuid.toString()) + } + } + + @OptIn(ExperimentalUuidApi::class) + @Test + fun `test on speech when not successful`() { + val tts = mockk(relaxed = true) + androidTts.start(tts) + androidTts.setMuted(false) + androidTts.onInit(TextToSpeech.ERROR) + + val uuid = UUID.randomUUID() + val instruction = + mockk { + every { text } returns "Hello, World!" + every { utteranceId } returns uuid + } + + androidTts.onSpokenInstructionTrigger(instruction) + + verify { Log.e(any(), any()) } + verify(exactly = 0) { + tts.speak("Hello, World!", TextToSpeech.QUEUE_ADD, any(), uuid.toString()) + } + } +} diff --git a/android/demo-app/src/main/java/com/stadiamaps/ferrostar/MainActivity.kt b/android/demo-app/src/main/java/com/stadiamaps/ferrostar/MainActivity.kt index d25ef67f..3b47d460 100644 --- a/android/demo-app/src/main/java/com/stadiamaps/ferrostar/MainActivity.kt +++ b/android/demo-app/src/main/java/com/stadiamaps/ferrostar/MainActivity.kt @@ -24,6 +24,13 @@ class MainActivity : ComponentActivity(), AndroidTtsStatusListener { AppModule.ttsObserver.shutdown() } + override fun onStart() { + super.onStart() + + // Start the TTS engine. This ensures that after onDestroy, a new instance is created. + AppModule.ttsObserver.start() + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -70,7 +77,7 @@ class MainActivity : ComponentActivity(), AndroidTtsStatusListener { override fun onTtsInitialized(tts: TextToSpeech?, status: Int) { // Set this up as appropriate for your app if (tts != null) { - tts.setLanguage(Locale.US) + tts.language = Locale.US android.util.Log.i(TAG, "setLanguage status: $status") } else { android.util.Log.e(TAG, "TTS setup failed! $status") @@ -81,4 +88,9 @@ class MainActivity : ComponentActivity(), AndroidTtsStatusListener { android.util.Log.e( TAG, "Something went wrong synthesizing utterance $utteranceId. Status code: $status.") } + + override fun onTtsShutdownAndRelease() { + android.util.Log.i( + TAG, "TTS shutdown and release. After this point you must call start() again.") + } } diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index 7de05e4b..ad08290d 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -21,6 +21,8 @@ junitVersion = "1.2.1" junitCompose = "1.7.5" espressoCore = "3.6.1" okhttp-mock = "2.0.0" +mockk = "1.13.14" +turbine = "1.2.0" mavenPublish = "0.30.0" material = "1.12.0" stadiaAutocompleteSearch = "1.0.0" @@ -30,7 +32,6 @@ desugar_jdk_libs = { module = "com.android.tools:desugar_jdk_libs", version.ref # Kotlin & KotlinX kotlin-bom = { group = "org.jetbrains.kotlin", name = "kotlin-bom", version.ref = "kotlin" } kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "kotlinx-coroutines" } -kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } kotlinx-datetime = { group = "org.jetbrains.kotlinx", name = "kotlinx-datetime", version.ref = "kotlinx-datetime" } # AndroidX androidx-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "ktx" } @@ -49,6 +50,8 @@ androidx-compose-ui-tooling-preview = { group = "androidx.compose.ui", name = "u androidx-compose-material3 = { group = "androidx.compose.material3", name = "material3" } androidx-compose-material-icon-extended = { group = "androidx.compose.material", name = "material-icons-extended" } androidx-compose-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-manifest" } +# Material +material = { group = "com.google.android.material", name = "material", version.ref = "material" } # OkHttp & Moshi okhttp-bom = { group = "com.squareup.okhttp3", name = "okhttp-bom", version.ref = "okhttp" } okhttp-core = { group = "com.squareup.okhttp3", name = "okhttp" } @@ -64,7 +67,11 @@ junit = { group = "junit", name = "junit", version.ref = "junit" } androidx-test-junit = { group = "androidx.test.ext", name = "junit", version.ref = "junitVersion" } androidx-test-espresso = { group = "androidx.test.espresso", name = "espresso-core", version.ref = "espressoCore" } androidx-compose-ui-test-junit4 = { group = "androidx.compose.ui", name = "ui-test-junit4", version.ref = "junitCompose" } -material = { group = "com.google.android.material", name = "material", version.ref = "material" } +kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "kotlinx-coroutines" } +mockk = { group = "io.mockk", name = "mockk", version.ref = "mockk" } +turbine = { group = "app.cash.turbine", name = "turbine", version.ref = "turbine" } + +# Demo App stadiamaps-autocomplete-search = { group = "com.stadiamaps", name = "jetpack-compose-autocomplete", version.ref = "stadiaAutocompleteSearch" } [plugins] diff --git a/guide/src/android-getting-started.md b/guide/src/android-getting-started.md index b9d2bd10..8a6b05d3 100644 --- a/guide/src/android-getting-started.md +++ b/guide/src/android-getting-started.md @@ -119,7 +119,7 @@ If your app uses Google Play Services, you can use the `FusedLocationProvider` This normally offers better device positioning than the default Android location provider on supported devices. -To make use of it, +To make use of it, you will need to include the optional `implementation "com.stadiamaps.ferrostar:google-play-services:${ferrostarVersion}"` in your Gradle dependencies block. @@ -256,6 +256,23 @@ Ferrostar can trigger the speech synthesis at the right time. Ferrostar includes the `AndroidTtsObserver` class, which uses the text-to-speech engine built into Android. +The `AndroidTtsObserver` follows lifecycle recommendations from the Android documentation, +[TextToSpeech shutdown behavior](https://developer.android.com/reference/android/speech/tts/TextToSpeech#shutdown()). +This design means your activity should call `shutdown` on the observer in the `onDestroy` method and start it again +in `onStart` if you want to continue using it. If the instance is shut down and not started again, you will not have spoken instructions. + +```kotlin +override fun onStart() { + super.onStart() + ttsObserver.start() +} + +override fun onDestroy() { + super.onDestroy() + ttsObserver.shutdown() +} +``` + You can also use your own implementation, such as a local AI model or cloud service like Amazon Polly. The `com.stadiamaps.ferrostar.core.SpokenInstructionObserver` interface