Add bluetooth toggle button during voice input

This commit is contained in:
Aleksandras Kostarevas 2024-06-06 23:42:28 +03:00
parent 54114c7cf0
commit 242bedd5c1
6 changed files with 227 additions and 63 deletions

View File

@ -38,6 +38,7 @@ import org.futo.inputmethod.latin.uix.PersistentActionState
import org.futo.inputmethod.latin.uix.ResourceHelper import org.futo.inputmethod.latin.uix.ResourceHelper
import org.futo.inputmethod.latin.uix.VERBOSE_PROGRESS import org.futo.inputmethod.latin.uix.VERBOSE_PROGRESS
import org.futo.inputmethod.latin.uix.getSetting import org.futo.inputmethod.latin.uix.getSetting
import org.futo.inputmethod.latin.uix.setSetting
import org.futo.inputmethod.latin.uix.voiceinput.downloader.DownloadActivity import org.futo.inputmethod.latin.uix.voiceinput.downloader.DownloadActivity
import org.futo.inputmethod.latin.xlm.UserDictionaryObserver import org.futo.inputmethod.latin.xlm.UserDictionaryObserver
import org.futo.inputmethod.updates.openURI import org.futo.inputmethod.updates.openURI
@ -50,6 +51,7 @@ import org.futo.voiceinput.shared.SoundPlayer
import org.futo.voiceinput.shared.types.Language import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.ModelLoader import org.futo.voiceinput.shared.types.ModelLoader
import org.futo.voiceinput.shared.types.getLanguageFromWhisperString import org.futo.voiceinput.shared.types.getLanguageFromWhisperString
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.whisper.DecodingConfiguration import org.futo.voiceinput.shared.whisper.DecodingConfiguration
import org.futo.voiceinput.shared.whisper.ModelManager import org.futo.voiceinput.shared.whisper.ModelManager
import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration
@ -212,9 +214,17 @@ private class VoiceInputActionWindow(
} }
} }
override fun recordingStarted(device: String) { override fun recordingStarted(device: MicrophoneDeviceState) {
if (shouldPlaySounds) { if (shouldPlaySounds) {
state.soundPlayer.playStartSound() state.soundPlayer.playStartSound()
// Only set the setting if bluetooth is available, else it would reset the setting
// every time it's used without a bluetooth device connected.
if(device.bluetoothAvailable) {
manager.getLifecycleScope().launch {
context.setSetting(PREFER_BLUETOOTH, device.bluetoothActive)
}
}
} }
} }

View File

@ -24,6 +24,7 @@ import com.konovalov.vad.config.SampleRate
import com.konovalov.vad.models.VadModel import com.konovalov.vad.models.VadModel
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import kotlinx.coroutines.yield import kotlinx.coroutines.yield
@ -34,6 +35,7 @@ import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.types.ModelInferenceCallback import org.futo.voiceinput.shared.types.ModelInferenceCallback
import org.futo.voiceinput.shared.types.ModelLoader import org.futo.voiceinput.shared.types.ModelLoader
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.whisper.DecodingConfiguration import org.futo.voiceinput.shared.whisper.DecodingConfiguration
import org.futo.voiceinput.shared.whisper.ModelManager import org.futo.voiceinput.shared.whisper.ModelManager
import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration
@ -139,24 +141,39 @@ class AudioRecognizer(
} }
} }
private fun setCommunicationDevice() { private fun isBluetoothAvailable(): Boolean {
communicationDevice = "Unset"
if(!settings.recordingConfiguration.preferBluetoothMic) return
try { try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val devices = audioManager.availableCommunicationDevices val devices = audioManager.availableCommunicationDevices
val tgtDevice = devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO } ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first()
return devices.firstOrNull {
it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
} != null
}
} catch(_: Exception) {}
return false
}
private fun setCommunicationDevice(preferBluetoothMic: Boolean): Pair<Boolean, String> {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val devices = audioManager.availableCommunicationDevices
val tgtDevice = devices.firstOrNull {
preferBluetoothMic && it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
} ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first()
if (!audioManager.setCommunicationDevice(tgtDevice)) { if (!audioManager.setCommunicationDevice(tgtDevice)) {
audioManager.clearCommunicationDevice() audioManager.clearCommunicationDevice()
return Pair(false, "")
} else { } else {
communicationDevice = return Pair(tgtDevice.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO, tgtDevice.productName.toString())
tgtDevice.productName.toString() + " (${getRecordingDeviceKind(tgtDevice.type)})"
} }
} }
} catch(_: Exception) {} } catch(_: Exception) {}
return Pair(false, "")
} }
private fun clearCommunicationDevice() { private fun clearCommunicationDevice() {
@ -251,14 +268,8 @@ class AudioRecognizer(
@Throws(SecurityException::class) @Throws(SecurityException::class)
private fun createAudioRecorder(): AudioRecord { private fun createAudioRecorder(): AudioRecord {
val purpose = if(settings.recordingConfiguration.preferBluetoothMic) {
MediaRecorder.AudioSource.VOICE_COMMUNICATION
} else {
MediaRecorder.AudioSource.VOICE_RECOGNITION
}
val recorder = AudioRecord( val recorder = AudioRecord(
purpose, MediaRecorder.AudioSource.VOICE_RECOGNITION,
16000, 16000,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_PCM_16BIT,
@ -300,7 +311,6 @@ class AudioRecognizer(
while (isRecording) { while (isRecording) {
yield() yield()
val nRead = recorder.read(samples, 0, 1600, AudioRecord.READ_BLOCKING) val nRead = recorder.read(samples, 0, 1600, AudioRecord.READ_BLOCKING)
if (nRead <= 0) break if (nRead <= 0) break
yield() yield()
@ -414,15 +424,56 @@ class AudioRecognizer(
.setSpeechDurationMs(150).setSilenceDurationMs(300).build() .setSpeechDurationMs(150).setSilenceDurationMs(300).build()
} }
private fun startRecording() { @Throws(SecurityException::class)
if (isRecording) { private fun createRecorderAndJob(preferBluetoothMic: Boolean): MicrophoneDeviceState {
throw IllegalStateException("Start recording when already recording") isRecording = false
recorder?.stop()
val bluetoothInfo = setCommunicationDevice(preferBluetoothMic)
val task = {
recorder?.release()
val recorder = createAudioRecorder()
recorder.startRecording()
this.recorder = recorder
isRecording = true
recorderJob = lifecycleScope.launch {
withContext(Dispatchers.Default) {
createVad().use { vad ->
recordingJob(recorder, vad)
}
}
}
} }
setCommunicationDevice() if(recorderJob != null) {
lifecycleScope.launch {
recorderJob?.cancelAndJoin()
task()
}
} else {
task()
}
val recorder = try {
createAudioRecorder() return MicrophoneDeviceState(
bluetoothAvailable = bluetoothInfo.first || isBluetoothAvailable(),
bluetoothActive = bluetoothInfo.first,
deviceName = bluetoothInfo.second,
bluetoothPreferredByUser = settings.recordingConfiguration.preferBluetoothMic,
setBluetooth = {
listener.recordingStarted(createRecorderAndJob(it))
}
)
}
private fun startRecording() {
val device = try {
createRecorderAndJob(settings.recordingConfiguration.preferBluetoothMic)
} catch (e: SecurityException) { } catch (e: SecurityException) {
// It's possible we may have lost permission, so let's just ask for permission again // It's possible we may have lost permission, so let's just ask for permission again
clearCommunicationDevice() clearCommunicationDevice()
@ -432,37 +483,13 @@ class AudioRecognizer(
focusAudio() focusAudio()
if(communicationDevice == "Unset") { listener.recordingStarted(device)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
communicationDevice = recorder.activeMicrophones.joinToString {
getRecordingDeviceKind(it.type)
} + "@"
}
}
listener.recordingStarted(communicationDevice)
recorder.startRecording()
this.recorder = recorder
isRecording = true
recorderJob = lifecycleScope.launch {
withContext(Dispatchers.Default) {
createVad().use { vad ->
recordingJob(recorder, vad)
}
}
}
loadModelJob = lifecycleScope.launch { loadModelJob = lifecycleScope.launch {
withContext(Dispatchers.Default) { withContext(Dispatchers.Default) {
preloadModels() preloadModels()
} }
} }
} }
private val runnerCallback: ModelInferenceCallback = object : ModelInferenceCallback { private val runnerCallback: ModelInferenceCallback = object : ModelInferenceCallback {

View File

@ -11,6 +11,7 @@ import org.futo.voiceinput.shared.types.InferenceState
import org.futo.voiceinput.shared.types.Language import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.ui.InnerRecognize import org.futo.voiceinput.shared.ui.InnerRecognize
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.ui.PartialDecodingResult import org.futo.voiceinput.shared.ui.PartialDecodingResult
import org.futo.voiceinput.shared.ui.RecognizeLoadingCircle import org.futo.voiceinput.shared.ui.RecognizeLoadingCircle
import org.futo.voiceinput.shared.ui.RecognizeMicError import org.futo.voiceinput.shared.ui.RecognizeMicError
@ -48,7 +49,7 @@ private val DefaultAnnotations = hashMapOf(
interface RecognizerViewListener { interface RecognizerViewListener {
fun cancelled() fun cancelled()
fun recordingStarted(device: String) fun recordingStarted(device: MicrophoneDeviceState)
fun finished(result: String) fun finished(result: String)
@ -76,7 +77,13 @@ class RecognizerView(
private val partialDecodingText = mutableStateOf("") private val partialDecodingText = mutableStateOf("")
private val currentViewState = mutableStateOf(CurrentView.LoadingCircle) private val currentViewState = mutableStateOf(CurrentView.LoadingCircle)
private val currentDeviceState = mutableStateOf("Recording not started") private val currentDeviceState = mutableStateOf(MicrophoneDeviceState(
bluetoothAvailable = false,
bluetoothActive = false,
setBluetooth = { },
deviceName = "",
bluetoothPreferredByUser = false
))
@Composable @Composable
fun Content() { fun Content() {
@ -97,7 +104,7 @@ class RecognizerView(
InnerRecognize( InnerRecognize(
magnitude = magnitudeState, magnitude = magnitudeState,
state = statusState, state = statusState,
device = if(settings.shouldShowVerboseFeedback) { currentDeviceState } else { null } device = currentDeviceState
) )
} }
@ -172,7 +179,7 @@ class RecognizerView(
} }
} }
override fun recordingStarted(device: String) { override fun recordingStarted(device: MicrophoneDeviceState) {
updateMagnitude(0.0f, MagnitudeState.NOT_TALKED_YET) updateMagnitude(0.0f, MagnitudeState.NOT_TALKED_YET)
currentDeviceState.value = device currentDeviceState.value = device
listener.recordingStarted(device) listener.recordingStarted(device)

View File

@ -1,5 +1,7 @@
package org.futo.voiceinput.shared.types package org.futo.voiceinput.shared.types
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
enum class MagnitudeState { enum class MagnitudeState {
NOT_TALKED_YET, MIC_MAY_BE_BLOCKED, TALKING NOT_TALKED_YET, MIC_MAY_BE_BLOCKED, TALKING
} }
@ -14,7 +16,7 @@ interface AudioRecognizerListener {
fun loading() fun loading()
fun needPermission(onResult: (Boolean) -> Unit) fun needPermission(onResult: (Boolean) -> Unit)
fun recordingStarted(device: String) fun recordingStarted(device: MicrophoneDeviceState)
fun updateMagnitude(magnitude: Float, state: MagnitudeState) fun updateMagnitude(magnitude: Float, state: MagnitudeState)
fun processing() fun processing()

View File

@ -1,7 +1,12 @@
package org.futo.voiceinput.shared.ui package org.futo.voiceinput.shared.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.foundation.Canvas import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxScope
import androidx.compose.foundation.layout.ColumnScope import androidx.compose.foundation.layout.ColumnScope
import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.defaultMinSize import androidx.compose.foundation.layout.defaultMinSize
@ -21,21 +26,46 @@ import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface import androidx.compose.material3.Surface
import androidx.compose.material3.Text import androidx.compose.material3.Text
import androidx.compose.runtime.Composable import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.MutableFloatState import androidx.compose.runtime.MutableFloatState
import androidx.compose.runtime.MutableState import androidx.compose.runtime.MutableState
import androidx.compose.runtime.mutableFloatStateOf import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.draw.drawBehind
import androidx.compose.ui.geometry.CornerRadius
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.drawscope.Fill
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.painterResource
import androidx.compose.ui.res.stringResource import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.semantics.clearAndSetSemantics
import androidx.compose.ui.semantics.role
import androidx.compose.ui.semantics.text
import androidx.compose.ui.semantics.toggleableState
import androidx.compose.ui.state.ToggleableState
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import kotlinx.coroutines.delay
import org.futo.voiceinput.shared.R import org.futo.voiceinput.shared.R
import org.futo.voiceinput.shared.types.MagnitudeState import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.ui.theme.Typography import org.futo.voiceinput.shared.ui.theme.Typography
data class MicrophoneDeviceState(
val bluetoothAvailable: Boolean,
val bluetoothActive: Boolean,
val bluetoothPreferredByUser: Boolean,
val setBluetooth: (Boolean) -> Unit,
val deviceName: String
)
@Composable @Composable
fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0.5f)) { fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0.5f)) {
val radius = animateValueChanges(magnitude.floatValue, 100) val radius = animateValueChanges(magnitude.floatValue, 100)
@ -51,11 +81,92 @@ fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0
} }
} }
@Composable
private fun FakeToast(modifier: Modifier, message: String?) {
val visible = remember { mutableStateOf(false) }
LaunchedEffect(message) {
if(message != null) {
visible.value = true
delay(2500L)
visible.value = false
} else {
visible.value = false
}
}
if(message != null) {
AnimatedVisibility(
visible = visible.value,
modifier = modifier.alpha(0.9f).background(MaterialTheme.colorScheme.surfaceDim, RoundedCornerShape(100)).padding(16.dp),
enter = fadeIn(),
exit = fadeOut()
) {
Box {
Text(message, modifier = Modifier.align(Alignment.Center), style = Typography.labelSmall)
}
}
}
}
@Composable
private fun BoxScope.BluetoothToggleIcon(device: MutableState<MicrophoneDeviceState>? = null) {
FakeToast(modifier = Modifier.align(Alignment.BottomCenter).offset(y = (-16).dp), message = if(device?.value?.bluetoothActive == true) {
"Using Bluetooth mic (${device.value.deviceName})"
} else if(device?.value?.bluetoothAvailable == true || device?.value?.bluetoothPreferredByUser == true) {
"Using Built-in mic (${device.value.deviceName})"
} else {
null
})
if(device?.value?.bluetoothAvailable == true) {
val bluetoothColor = MaterialTheme.colorScheme.primary
val iconColor = if(device.value.bluetoothActive) {
MaterialTheme.colorScheme.onPrimary
} else {
MaterialTheme.colorScheme.onSurface
}
IconButton(modifier = Modifier
.align(Alignment.BottomEnd)
.offset(x = (-16).dp, y = (-16).dp)
.drawBehind {
val radius = size.height / 4.0f
drawRoundRect(
bluetoothColor,
topLeft = Offset(size.width * 0.1f, size.height * 0.05f),
size = Size(size.width * 0.8f, size.height * 0.9f),
cornerRadius = CornerRadius(radius, radius),
style = if (device.value.bluetoothActive) {
Fill
} else {
Stroke(width = 4.0f)
}
)
}
.clearAndSetSemantics {
this.text = AnnotatedString("Use bluetooth mic")
this.role = Role.Switch
this.toggleableState = ToggleableState(device.value.bluetoothActive)
},
onClick = {
device.value.setBluetooth(!device.value.bluetoothActive)
},
) {
Icon(
painter = painterResource(id = R.drawable.bluetooth),
contentDescription = null,
tint = iconColor
)
}
}
}
@Composable @Composable
fun InnerRecognize( fun InnerRecognize(
magnitude: MutableFloatState = mutableFloatStateOf(0.5f), magnitude: MutableFloatState = mutableFloatStateOf(0.5f),
state: MutableState<MagnitudeState> = mutableStateOf(MagnitudeState.MIC_MAY_BE_BLOCKED), state: MutableState<MagnitudeState> = mutableStateOf(MagnitudeState.MIC_MAY_BE_BLOCKED),
device: MutableState<String>? = mutableStateOf("") device: MutableState<MicrophoneDeviceState>? = null
) { ) {
Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
AnimatedRecognizeCircle(magnitude = magnitude) AnimatedRecognizeCircle(magnitude = magnitude)
@ -75,20 +186,14 @@ fun InnerRecognize(
Text( Text(
text, text,
modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 48.dp), modifier = Modifier
.fillMaxWidth()
.offset(x = 0.dp, y = 48.dp),
textAlign = TextAlign.Center, textAlign = TextAlign.Center,
color = MaterialTheme.colorScheme.onSurface color = MaterialTheme.colorScheme.onSurface
) )
if(device != null) { BluetoothToggleIcon(device)
Text(
"Device: ${device.value}",
style = Typography.labelSmall,
modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 64.dp),
textAlign = TextAlign.Center,
color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.66f)
)
}
} }
} }

View File

@ -0,0 +1,13 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path
android:pathData="M6.5,6.5l11,11l-5.5,5.5l0,-22l5.5,5.5l-11,11"
android:strokeLineJoin="round"
android:strokeWidth="2"
android:fillColor="#00000000"
android:strokeColor="#ffffff"
android:strokeLineCap="round"/>
</vector>