diff --git a/java/src/org/futo/inputmethod/latin/uix/actions/VoiceInputAction.kt b/java/src/org/futo/inputmethod/latin/uix/actions/VoiceInputAction.kt index 126b37155..40ce20a71 100644 --- a/java/src/org/futo/inputmethod/latin/uix/actions/VoiceInputAction.kt +++ b/java/src/org/futo/inputmethod/latin/uix/actions/VoiceInputAction.kt @@ -38,6 +38,7 @@ import org.futo.inputmethod.latin.uix.PersistentActionState import org.futo.inputmethod.latin.uix.ResourceHelper import org.futo.inputmethod.latin.uix.VERBOSE_PROGRESS import org.futo.inputmethod.latin.uix.getSetting +import org.futo.inputmethod.latin.uix.setSetting import org.futo.inputmethod.latin.uix.voiceinput.downloader.DownloadActivity import org.futo.inputmethod.latin.xlm.UserDictionaryObserver import org.futo.inputmethod.updates.openURI @@ -50,6 +51,7 @@ import org.futo.voiceinput.shared.SoundPlayer import org.futo.voiceinput.shared.types.Language import org.futo.voiceinput.shared.types.ModelLoader import org.futo.voiceinput.shared.types.getLanguageFromWhisperString +import org.futo.voiceinput.shared.ui.MicrophoneDeviceState import org.futo.voiceinput.shared.whisper.DecodingConfiguration import org.futo.voiceinput.shared.whisper.ModelManager import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration @@ -212,9 +214,17 @@ private class VoiceInputActionWindow( } } - override fun recordingStarted(device: String) { + override fun recordingStarted(device: MicrophoneDeviceState) { if (shouldPlaySounds) { state.soundPlayer.playStartSound() + + // Only set the setting if bluetooth is available, else it would reset the setting + // every time it's used without a bluetooth device connected. + if(device.bluetoothAvailable) { + manager.getLifecycleScope().launch { + context.setSetting(PREFER_BLUETOOTH, device.bluetoothActive) + } + } } } diff --git a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/AudioRecognizer.kt b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/AudioRecognizer.kt index 4b2fb5b6d..70576d4b0 100644 --- a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/AudioRecognizer.kt +++ b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/AudioRecognizer.kt @@ -24,6 +24,7 @@ import com.konovalov.vad.config.SampleRate import com.konovalov.vad.models.VadModel import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.coroutines.yield @@ -34,6 +35,7 @@ import org.futo.voiceinput.shared.types.Language import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.types.ModelInferenceCallback import org.futo.voiceinput.shared.types.ModelLoader +import org.futo.voiceinput.shared.ui.MicrophoneDeviceState import org.futo.voiceinput.shared.whisper.DecodingConfiguration import org.futo.voiceinput.shared.whisper.ModelManager import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration @@ -139,24 +141,39 @@ class AudioRecognizer( } } - private fun setCommunicationDevice() { - communicationDevice = "Unset" - if(!settings.recordingConfiguration.preferBluetoothMic) return - + private fun isBluetoothAvailable(): Boolean { try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager val devices = audioManager.availableCommunicationDevices - val tgtDevice = devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO } ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first() + + return devices.firstOrNull { + it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } != null + } + } catch(_: Exception) {} + + return false + } + + private fun setCommunicationDevice(preferBluetoothMic: Boolean): Pair { + try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + val devices = audioManager.availableCommunicationDevices + val tgtDevice = devices.firstOrNull { + preferBluetoothMic && it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first() if (!audioManager.setCommunicationDevice(tgtDevice)) { audioManager.clearCommunicationDevice() + return Pair(false, "") } else { - communicationDevice = - tgtDevice.productName.toString() + " (${getRecordingDeviceKind(tgtDevice.type)})" + return Pair(tgtDevice.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO, tgtDevice.productName.toString()) } } } catch(_: Exception) {} + return Pair(false, "") } private fun clearCommunicationDevice() { @@ -251,14 +268,8 @@ class AudioRecognizer( @Throws(SecurityException::class) private fun createAudioRecorder(): AudioRecord { - val purpose = if(settings.recordingConfiguration.preferBluetoothMic) { - MediaRecorder.AudioSource.VOICE_COMMUNICATION - } else { - MediaRecorder.AudioSource.VOICE_RECOGNITION - } - val recorder = AudioRecord( - purpose, + MediaRecorder.AudioSource.VOICE_RECOGNITION, 16000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, @@ -300,7 +311,6 @@ class AudioRecognizer( while (isRecording) { yield() val nRead = recorder.read(samples, 0, 1600, AudioRecord.READ_BLOCKING) - if (nRead <= 0) break yield() @@ -414,15 +424,56 @@ class AudioRecognizer( .setSpeechDurationMs(150).setSilenceDurationMs(300).build() } - private fun startRecording() { - if (isRecording) { - throw IllegalStateException("Start recording when already recording") + @Throws(SecurityException::class) + private fun createRecorderAndJob(preferBluetoothMic: Boolean): MicrophoneDeviceState { + isRecording = false + recorder?.stop() + + val bluetoothInfo = setCommunicationDevice(preferBluetoothMic) + + val task = { + recorder?.release() + + val recorder = createAudioRecorder() + + recorder.startRecording() + this.recorder = recorder + + isRecording = true + + recorderJob = lifecycleScope.launch { + withContext(Dispatchers.Default) { + createVad().use { vad -> + recordingJob(recorder, vad) + } + } + } } - setCommunicationDevice() + if(recorderJob != null) { + lifecycleScope.launch { + recorderJob?.cancelAndJoin() + task() + } + } else { + task() + } - val recorder = try { - createAudioRecorder() + + return MicrophoneDeviceState( + bluetoothAvailable = bluetoothInfo.first || isBluetoothAvailable(), + bluetoothActive = bluetoothInfo.first, + deviceName = bluetoothInfo.second, + bluetoothPreferredByUser = settings.recordingConfiguration.preferBluetoothMic, + setBluetooth = { + listener.recordingStarted(createRecorderAndJob(it)) + } + ) + } + + private fun startRecording() { + val device = try { + createRecorderAndJob(settings.recordingConfiguration.preferBluetoothMic) } catch (e: SecurityException) { // It's possible we may have lost permission, so let's just ask for permission again clearCommunicationDevice() @@ -432,37 +483,13 @@ class AudioRecognizer( focusAudio() - if(communicationDevice == "Unset") { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - communicationDevice = recorder.activeMicrophones.joinToString { - getRecordingDeviceKind(it.type) - } + "@" - } - } - - listener.recordingStarted(communicationDevice) - - - recorder.startRecording() - - this.recorder = recorder - - isRecording = true - - recorderJob = lifecycleScope.launch { - withContext(Dispatchers.Default) { - createVad().use { vad -> - recordingJob(recorder, vad) - } - } - } + listener.recordingStarted(device) loadModelJob = lifecycleScope.launch { withContext(Dispatchers.Default) { preloadModels() } } - } private val runnerCallback: ModelInferenceCallback = object : ModelInferenceCallback { diff --git a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/RecognizerView.kt b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/RecognizerView.kt index d1c973299..456a875c1 100644 --- a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/RecognizerView.kt +++ b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/RecognizerView.kt @@ -11,6 +11,7 @@ import org.futo.voiceinput.shared.types.InferenceState import org.futo.voiceinput.shared.types.Language import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.ui.InnerRecognize +import org.futo.voiceinput.shared.ui.MicrophoneDeviceState import org.futo.voiceinput.shared.ui.PartialDecodingResult import org.futo.voiceinput.shared.ui.RecognizeLoadingCircle import org.futo.voiceinput.shared.ui.RecognizeMicError @@ -48,7 +49,7 @@ private val DefaultAnnotations = hashMapOf( interface RecognizerViewListener { fun cancelled() - fun recordingStarted(device: String) + fun recordingStarted(device: MicrophoneDeviceState) fun finished(result: String) @@ -76,7 +77,13 @@ class RecognizerView( private val partialDecodingText = mutableStateOf("") private val currentViewState = mutableStateOf(CurrentView.LoadingCircle) - private val currentDeviceState = mutableStateOf("Recording not started") + private val currentDeviceState = mutableStateOf(MicrophoneDeviceState( + bluetoothAvailable = false, + bluetoothActive = false, + setBluetooth = { }, + deviceName = "", + bluetoothPreferredByUser = false + )) @Composable fun Content() { @@ -97,7 +104,7 @@ class RecognizerView( InnerRecognize( magnitude = magnitudeState, state = statusState, - device = if(settings.shouldShowVerboseFeedback) { currentDeviceState } else { null } + device = currentDeviceState ) } @@ -172,7 +179,7 @@ class RecognizerView( } } - override fun recordingStarted(device: String) { + override fun recordingStarted(device: MicrophoneDeviceState) { updateMagnitude(0.0f, MagnitudeState.NOT_TALKED_YET) currentDeviceState.value = device listener.recordingStarted(device) diff --git a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/types/AudioRecognizerListener.kt b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/types/AudioRecognizerListener.kt index 6c59ff167..ee391f164 100644 --- a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/types/AudioRecognizerListener.kt +++ b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/types/AudioRecognizerListener.kt @@ -1,5 +1,7 @@ package org.futo.voiceinput.shared.types +import org.futo.voiceinput.shared.ui.MicrophoneDeviceState + enum class MagnitudeState { NOT_TALKED_YET, MIC_MAY_BE_BLOCKED, TALKING } @@ -14,7 +16,7 @@ interface AudioRecognizerListener { fun loading() fun needPermission(onResult: (Boolean) -> Unit) - fun recordingStarted(device: String) + fun recordingStarted(device: MicrophoneDeviceState) fun updateMagnitude(magnitude: Float, state: MagnitudeState) fun processing() diff --git a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/ui/RecognizeViews.kt b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/ui/RecognizeViews.kt index ec8787c00..d473ba0bb 100644 --- a/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/ui/RecognizeViews.kt +++ b/voiceinput-shared/src/main/java/org/futo/voiceinput/shared/ui/RecognizeViews.kt @@ -1,7 +1,12 @@ package org.futo.voiceinput.shared.ui +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut import androidx.compose.foundation.Canvas +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxScope import androidx.compose.foundation.layout.ColumnScope import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.defaultMinSize @@ -21,21 +26,46 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.MutableFloatState import androidx.compose.runtime.MutableState import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.draw.drawBehind +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.drawscope.Fill +import androidx.compose.ui.graphics.drawscope.Stroke import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.clearAndSetSemantics +import androidx.compose.ui.semantics.role +import androidx.compose.ui.semantics.text +import androidx.compose.ui.semantics.toggleableState +import androidx.compose.ui.state.ToggleableState +import androidx.compose.ui.text.AnnotatedString import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp +import kotlinx.coroutines.delay import org.futo.voiceinput.shared.R import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.ui.theme.Typography +data class MicrophoneDeviceState( + val bluetoothAvailable: Boolean, + val bluetoothActive: Boolean, + val bluetoothPreferredByUser: Boolean, + val setBluetooth: (Boolean) -> Unit, + val deviceName: String +) + @Composable fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0.5f)) { val radius = animateValueChanges(magnitude.floatValue, 100) @@ -51,11 +81,92 @@ fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0 } } +@Composable +private fun FakeToast(modifier: Modifier, message: String?) { + val visible = remember { mutableStateOf(false) } + + LaunchedEffect(message) { + if(message != null) { + visible.value = true + delay(2500L) + visible.value = false + } else { + visible.value = false + } + } + + if(message != null) { + AnimatedVisibility( + visible = visible.value, + modifier = modifier.alpha(0.9f).background(MaterialTheme.colorScheme.surfaceDim, RoundedCornerShape(100)).padding(16.dp), + enter = fadeIn(), + exit = fadeOut() + ) { + Box { + Text(message, modifier = Modifier.align(Alignment.Center), style = Typography.labelSmall) + } + } + } +} + +@Composable +private fun BoxScope.BluetoothToggleIcon(device: MutableState? = null) { + FakeToast(modifier = Modifier.align(Alignment.BottomCenter).offset(y = (-16).dp), message = if(device?.value?.bluetoothActive == true) { + "Using Bluetooth mic (${device.value.deviceName})" + } else if(device?.value?.bluetoothAvailable == true || device?.value?.bluetoothPreferredByUser == true) { + "Using Built-in mic (${device.value.deviceName})" + } else { + null + }) + + if(device?.value?.bluetoothAvailable == true) { + val bluetoothColor = MaterialTheme.colorScheme.primary + val iconColor = if(device.value.bluetoothActive) { + MaterialTheme.colorScheme.onPrimary + } else { + MaterialTheme.colorScheme.onSurface + } + + IconButton(modifier = Modifier + .align(Alignment.BottomEnd) + .offset(x = (-16).dp, y = (-16).dp) + .drawBehind { + val radius = size.height / 4.0f + drawRoundRect( + bluetoothColor, + topLeft = Offset(size.width * 0.1f, size.height * 0.05f), + size = Size(size.width * 0.8f, size.height * 0.9f), + cornerRadius = CornerRadius(radius, radius), + style = if (device.value.bluetoothActive) { + Fill + } else { + Stroke(width = 4.0f) + } + ) + } + .clearAndSetSemantics { + this.text = AnnotatedString("Use bluetooth mic") + this.role = Role.Switch + this.toggleableState = ToggleableState(device.value.bluetoothActive) + }, + onClick = { + device.value.setBluetooth(!device.value.bluetoothActive) + }, + ) { + Icon( + painter = painterResource(id = R.drawable.bluetooth), + contentDescription = null, + tint = iconColor + ) + } + } +} + @Composable fun InnerRecognize( magnitude: MutableFloatState = mutableFloatStateOf(0.5f), state: MutableState = mutableStateOf(MagnitudeState.MIC_MAY_BE_BLOCKED), - device: MutableState? = mutableStateOf("") + device: MutableState? = null ) { Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { AnimatedRecognizeCircle(magnitude = magnitude) @@ -75,20 +186,14 @@ fun InnerRecognize( Text( text, - modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 48.dp), + modifier = Modifier + .fillMaxWidth() + .offset(x = 0.dp, y = 48.dp), textAlign = TextAlign.Center, color = MaterialTheme.colorScheme.onSurface ) - if(device != null) { - Text( - "Device: ${device.value}", - style = Typography.labelSmall, - modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 64.dp), - textAlign = TextAlign.Center, - color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.66f) - ) - } + BluetoothToggleIcon(device) } } diff --git a/voiceinput-shared/src/main/res/drawable/bluetooth.xml b/voiceinput-shared/src/main/res/drawable/bluetooth.xml new file mode 100644 index 000000000..10939e89f --- /dev/null +++ b/voiceinput-shared/src/main/res/drawable/bluetooth.xml @@ -0,0 +1,13 @@ + + +