NAVAND-2991 - Optimized TextToSpeech usage (#7833) (#7842)

* NAVAND-2991 - Optimized TextToSpeech usage Moved language setting and speak calls to a background thread. * Rename changelog files * NAVAND-2991 - Optimized TextToSpeech usage Unit-test fix --------- Co-authored-by: Tomasz Rybakiewicz <tomaszrybakiewicz@users.noreply.github.com> Co-authored-by: runner <runner@fv-az1428-633>
mapbox · Jul 22, 2024 · dfa79e9 · dfa79e9
1 parent 39a5265
commit dfa79e9
Show file tree

Hide file tree

Showing 6 changed files with 96 additions and 50 deletions.
diff --git a/LICENSE.md b/LICENSE.md
@@ -2469,7 +2469,7 @@ License: [The Apache Software License, Version 2.0](http://www.apache.org/licens
 ===========================================================================
 
 Mapbox Navigation uses portions of the Android Support Library compat (The Support Library is a static library that you can add to your Android application in order to use APIs that are either not available for older platform versions or utility APIs that aren't a part of the framework APIs. Compatible on devices running API 14 or later.).
-URL: [https://developer.android.com/jetpack/androidx/releases/core#1.5.0](https://developer.android.com/jetpack/androidx/releases/core#1.5.0)
+URL: [https://developer.android.com/jetpack/androidx/releases/core#1.6.0](https://developer.android.com/jetpack/androidx/releases/core#1.6.0)
 License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt)
 
 ===========================================================================
@@ -2544,8 +2544,14 @@ License: [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0.txt)
 
 ===========================================================================
 
+Mapbox Navigation uses portions of the Core Kotlin Extensions (Kotlin extensions for 'core' artifact).
+URL: [https://developer.android.com/jetpack/androidx/releases/core#1.6.0](https://developer.android.com/jetpack/androidx/releases/core#1.6.0)
+License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+===========================================================================
+
 Mapbox Navigation uses portions of the Experimental annotation (Java annotation for use on unstable Android API surfaces. When used in conjunction with the Experimental annotation lint checks, this annotation provides functional parity with Kotlin's Experimental annotation.).
-URL: [https://developer.android.com/jetpack/androidx](https://developer.android.com/jetpack/androidx)
+URL: [https://developer.android.com/jetpack/androidx/releases/annotation#1.1.0](https://developer.android.com/jetpack/androidx/releases/annotation#1.1.0)
 License: [The Apache Software License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.txt)
 
 ===========================================================================

diff --git a/changelog/unreleased/bugfixes/7833.md b/changelog/unreleased/bugfixes/7833.md
@@ -0,0 +1 @@
+- Fixed UI jank caused by on-device TextToSpeech player.
diff --git a/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt b/examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt
@@ -6,6 +6,7 @@ import android.location.Location
 import android.os.Bundle
 import android.view.View
 import androidx.appcompat.app.AppCompatActivity
+import androidx.lifecycle.coroutineScope
 import androidx.lifecycle.lifecycleScope
 import com.mapbox.api.directions.v5.models.DirectionsRoute
 import com.mapbox.api.directions.v5.models.RouteOptions
@@ -69,6 +70,7 @@ import com.mapbox.navigation.ui.voice.model.SpeechVolume
 import com.mapbox.navigation.ui.voice.options.VoiceInstructionsPlayerOptions
 import com.mapbox.navigation.utils.internal.ifNonNull
 import com.mapbox.navigation.utils.internal.logD
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import java.util.Locale
 
@@ -112,7 +114,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
      * has to be played. [MapboxVoiceInstructionsPlayer] should be instantiated in
      * `Activity#onCreate`.
      */
-    private lateinit var voiceInstructionsPlayer: MapboxVoiceInstructionsPlayer
+    private var voiceInstructionsPlayer: MapboxVoiceInstructionsPlayer? = null
 
     private val routeLineResources: RouteLineResources by lazy {
         RouteLineResources.Builder().build()
@@ -163,7 +165,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
                     logD("play(fallback): '${error.fallback.announcement}'", TAG)
                     // The data obtained in the form of an error is played using
                     // voiceInstructionsPlayer.
-                    voiceInstructionsPlayer.play(
+                    voiceInstructionsPlayer?.play(
                         error.fallback,
                         voiceInstructionsPlayerCallback
                     )
@@ -172,7 +174,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
                     logD("play: '${value.announcement.announcement}'", TAG)
                     // The data obtained in the form of speech announcement is played using
                     // voiceInstructionsPlayer.
-                    voiceInstructionsPlayer.play(
+                    voiceInstructionsPlayer?.play(
                         value.announcement,
                         voiceInstructionsPlayerCallback
                     )
@@ -220,7 +222,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         RoutesObserver { result -> // Every time a new route is obtained make sure to cancel the [MapboxSpeechApi] and
             // clear the [MapboxVoiceInstructionsPlayer]
             speechApi.cancel()
-            voiceInstructionsPlayer.clear()
+            voiceInstructionsPlayer?.clear()
             if (result.navigationRoutes.isNotEmpty()) {
                 lifecycleScope.launch {
                     routeLineApi.setNavigationRoutes(
@@ -300,7 +302,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         }
 
         binding.addPlay.setOnClickListener {
-            voiceInstructionsPlayer.play(
+            voiceInstructionsPlayer?.play(
                 SpeechAnnouncement.Builder("Test hybrid speech player.").build(),
                 voiceInstructionsPlayerCallback
             )
@@ -323,10 +325,10 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
     private fun handleSoundState(value: Boolean) {
         if (value) {
             // This is used to set the speech volume to mute.
-            voiceInstructionsPlayer.volume(SpeechVolume(0.0f))
+            voiceInstructionsPlayer?.volume(SpeechVolume(0.0f))
         } else {
             // This is used to set the speech volume to max
-            voiceInstructionsPlayer.volume(SpeechVolume(1.0f))
+            voiceInstructionsPlayer?.volume(SpeechVolume(1.0f))
         }
         isMuted = value
     }
@@ -387,13 +389,15 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
             setLocationProvider(navigationLocationProvider)
             enabled = true
         }
-        voiceInstructionsPlayer = MapboxVoiceInstructionsPlayer(
-            this,
-            Locale.US.toLanguageTag(),
-            VoiceInstructionsPlayerOptions.Builder()
-                .abandonFocusDelay(PLAYER_ABANDON_FOCUS_DELAY)
-                .build()
-        )
+        lifecycle.coroutineScope.launch(Dispatchers.Default) {
+            voiceInstructionsPlayer = MapboxVoiceInstructionsPlayer(
+                applicationContext,
+                Locale.US.toLanguageTag(),
+                VoiceInstructionsPlayerOptions.Builder()
+                    .abandonFocusDelay(PLAYER_ABANDON_FOCUS_DELAY)
+                    .build()
+            )
+        }
         init()
         voiceInstructionsPrefetcher.onAttached(mapboxNavigation)
     }
@@ -428,7 +432,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         mapboxNavigation.onDestroy()
         speechApi.cancel()
         voiceInstructionsPrefetcher.onDetached(mapboxNavigation)
-        voiceInstructionsPlayer.shutdown()
+        voiceInstructionsPlayer?.shutdown()
     }
 
     override fun onMapLongClick(point: Point): Boolean {

diff --git a/libnavui-voice/build.gradle b/libnavui-voice/build.gradle
@@ -50,6 +50,7 @@ dependencies {
     // androidX
     implementation dependenciesList.androidXConstraintLayout
     implementation dependenciesList.androidXAppCompat
+    implementation dependenciesList.androidXCoreKtx
 
     apply from: "../gradle/unit-testing-dependencies.gradle"
     testImplementation(project(':libtesting-utils'))

diff --git a/...vui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt b/...vui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayer.kt
@@ -4,9 +4,14 @@ import android.content.Context
 import android.speech.tts.TextToSpeech
 import android.speech.tts.UtteranceProgressListener
 import androidx.annotation.VisibleForTesting
+import androidx.core.os.trace
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechVolume
+import com.mapbox.navigation.utils.internal.InternalJobControlFactory.createDefaultScopeJobControl
+import com.mapbox.navigation.utils.internal.logD
 import com.mapbox.navigation.utils.internal.logE
+import kotlinx.coroutines.cancelChildren
+import kotlinx.coroutines.launch
 import java.util.Locale
 
 /**
@@ -25,18 +30,18 @@ internal class VoiceInstructionsTextPlayer(
     internal var isLanguageSupported: Boolean = false
 
     private var textToSpeechInitStatus: Int? = null
+    private val jobControl = createDefaultScopeJobControl()
 
     @VisibleForTesting
-    internal val textToSpeech =
+    internal val textToSpeech = trace(TRACE_GET_TTS) {
         TextToSpeechProvider.getTextToSpeech(context.applicationContext) { status ->
             textToSpeechInitStatus = status
             if (status == TextToSpeech.SUCCESS) {
                 initializeWithLanguage(Locale(language))
-                if (isLanguageSupported) {
-                    setUpUtteranceProgressListener()
-                }
+                setUpUtteranceProgressListener()
             }
         }
+    }
 
     @VisibleForTesting
     internal var volumeLevel: Float = DEFAULT_VOLUME_LEVEL
@@ -69,14 +74,11 @@ internal class VoiceInstructionsTextPlayer(
             "Only one announcement can be played at a time."
         }
         currentPlay = announcement
-        val announcement = announcement.announcement
-        if (isLanguageSupported && announcement.isNotBlank()) {
-            play(announcement)
+        val text = announcement.announcement
+        if (isLanguageSupported && text.isNotBlank()) {
+            play(text)
         } else {
-            logE(
-                "$LANGUAGE_NOT_SUPPORTED or announcement from state is blank",
-                LOG_CATEGORY
-            )
+            logE { "$LANGUAGE_NOT_SUPPORTED or announcement from state is blank" }
             donePlaying()
         }
     }
@@ -108,6 +110,7 @@ internal class VoiceInstructionsTextPlayer(
      * the announcement should end immediately and any announcements queued should be cleared.
      */
     override fun shutdown() {
+        jobControl.job.cancelChildren()
         textToSpeech.setOnUtteranceProgressListener(null)
         textToSpeech.shutdown()
         currentPlay = null
@@ -116,16 +119,20 @@ internal class VoiceInstructionsTextPlayer(
 
     @VisibleForTesting
     internal fun initializeWithLanguage(language: Locale) {
-        isLanguageSupported = if (playerAttributes.options.checkIsLanguageAvailable) {
-            textToSpeech.isLanguageAvailable(language) == TextToSpeech.LANG_AVAILABLE
-        } else {
-            true
-        }
-        if (!isLanguageSupported) {
-            logE(LANGUAGE_NOT_SUPPORTED, LOG_CATEGORY)
-            return
+        jobControl.scope.launch {
+            trace(TRACE_INIT_LANG) {
+                isLanguageSupported = if (playerAttributes.options.checkIsLanguageAvailable) {
+                    textToSpeech.isLanguageAvailable(language) == TextToSpeech.LANG_AVAILABLE
+                } else {
+                    true
+                }
+                if (!isLanguageSupported) {
+                    logE { LANGUAGE_NOT_SUPPORTED }
+                    return@trace
+                }
+                textToSpeech.language = language
+            }
         }
-        textToSpeech.language = language
     }
 
     private fun setUpUtteranceProgressListener() {
@@ -136,12 +143,12 @@ internal class VoiceInstructionsTextPlayer(
 
             override fun onError(utteranceId: String?) {
                 // Deprecated, may be called due to https://issuetracker.google.com/issues/138321382
-                logE("Unexpected TextToSpeech error", LOG_CATEGORY)
+                logE { "Unexpected TextToSpeech error" }
                 donePlaying()
             }
 
             override fun onError(utteranceId: String?, errorCode: Int) {
-                logE("TextToSpeech error: $errorCode", LOG_CATEGORY)
+                logE { "TextToSpeech error: $errorCode" }
                 donePlaying()
             }
 
@@ -163,18 +170,23 @@ internal class VoiceInstructionsTextPlayer(
     }
 
     private fun play(announcement: String) {
-        val currentBundle = BundleProvider.retrieveBundle()
-        val bundle = currentBundle.apply {
-            putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, volumeLevel)
+        logD { "play: $announcement" }
+        jobControl.scope.launch {
+            trace(TRACE_PLAY) {
+                val currentBundle = BundleProvider.retrieveBundle()
+                val bundle = currentBundle.apply {
+                    putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, volumeLevel)
+                }
+                playerAttributes.applyOn(textToSpeech, bundle)
+
+                textToSpeech.speak(
+                    announcement,
+                    TextToSpeech.QUEUE_FLUSH,
+                    bundle,
+                    DEFAULT_UTTERANCE_ID
+                )
+            }
         }
-        playerAttributes.applyOn(textToSpeech, bundle)
-
-        textToSpeech.speak(
-            announcement,
-            TextToSpeech.QUEUE_FLUSH,
-            bundle,
-            DEFAULT_UTTERANCE_ID
-        )
     }
 
     private companion object {
@@ -184,5 +196,12 @@ internal class VoiceInstructionsTextPlayer(
         private const val DEFAULT_UTTERANCE_ID = "default_id"
         private const val DEFAULT_VOLUME_LEVEL = 1.0f
         private const val MUTE_VOLUME_LEVEL = 0.0f
+
+        private const val TRACE_GET_TTS = "VoiceInstructionsTextPlayer.getTextToSpeech"
+        private const val TRACE_INIT_LANG = "VoiceInstructionsTextPlayer.initializeWithLanguage"
+        private const val TRACE_PLAY = "VoiceInstructionsTextPlayer.play"
+
+        private inline fun logD(msg: () -> String) = logD(LOG_CATEGORY, msg)
+        private inline fun logE(msg: () -> String) = logE(LOG_CATEGORY, msg)
     }
 }
diff --git a/...voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt b/...voice/src/test/java/com/mapbox/navigation/ui/voice/api/VoiceInstructionsTextPlayerTest.kt
@@ -7,15 +7,19 @@ import android.speech.tts.TextToSpeech.LANG_AVAILABLE
 import android.speech.tts.TextToSpeech.LANG_NOT_SUPPORTED
 import android.speech.tts.TextToSpeech.OnInitListener
 import com.mapbox.navigation.testing.LoggingFrontendTestRule
+import com.mapbox.navigation.testing.MainCoroutineRule
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechVolume
 import com.mapbox.navigation.ui.voice.options.VoiceInstructionsPlayerOptions
+import com.mapbox.navigation.utils.internal.InternalJobControlFactory
+import com.mapbox.navigation.utils.internal.JobControl
 import io.mockk.clearMocks
 import io.mockk.every
 import io.mockk.mockk
 import io.mockk.mockkObject
 import io.mockk.unmockkObject
 import io.mockk.verify
+import kotlinx.coroutines.job
 import org.junit.After
 import org.junit.Assert.assertEquals
 import org.junit.Before
@@ -25,6 +29,9 @@ import java.util.Locale
 
 class VoiceInstructionsTextPlayerTest {
 
+    @get:Rule
+    var coroutineRule = MainCoroutineRule()
+
     @get:Rule
     val loggerRule = LoggingFrontendTestRule()
 
@@ -33,6 +40,13 @@ class VoiceInstructionsTextPlayerTest {
 
     @Before
     fun setUp() {
+        mockkObject(InternalJobControlFactory)
+        every {
+            InternalJobControlFactory.createDefaultScopeJobControl()
+        } answers {
+            val defaultScope = coroutineRule.createTestScope()
+            JobControl(defaultScope.coroutineContext.job, defaultScope)
+        }
         mockkObject(BundleProvider)
         mockkObject(TextToSpeechProvider)
         every { BundleProvider.retrieveBundle() } returns mockedBundle
@@ -41,6 +55,7 @@ class VoiceInstructionsTextPlayerTest {
 
     @After
     fun tearDown() {
+        unmockkObject(InternalJobControlFactory)
         unmockkObject(BundleProvider)
         unmockkObject(TextToSpeechProvider)
     }