n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add bluetooth toggle button during voice input

Browse Source
This commit is contained in:
Aleksandras Kostarevas 2024-06-06 23:42:28 +03:00
parent 54114c7cf0
commit 242bedd5c1
6 changed files with 227 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
java/src/org/futo/inputmethod/latin/uix/actions/VoiceInputAction.kt
View File

@ -38,6 +38,7 @@ import org.futo.inputmethod.latin.uix.PersistentActionState
import org.futo.inputmethod.latin.uix.ResourceHelper
import org.futo.inputmethod.latin.uix.VERBOSE_PROGRESS
import org.futo.inputmethod.latin.uix.getSetting
import org.futo.inputmethod.latin.uix.setSetting
import org.futo.inputmethod.latin.uix.voiceinput.downloader.DownloadActivity
import org.futo.inputmethod.latin.xlm.UserDictionaryObserver
import org.futo.inputmethod.updates.openURI
@ -50,6 +51,7 @@ import org.futo.voiceinput.shared.SoundPlayer
import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.ModelLoader
import org.futo.voiceinput.shared.types.getLanguageFromWhisperString
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.whisper.DecodingConfiguration
import org.futo.voiceinput.shared.whisper.ModelManager
import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration
@ -212,9 +214,17 @@ private class VoiceInputActionWindow(
}
}
override fun recordingStarted(device: String) {
override fun recordingStarted(device: MicrophoneDeviceState) {
if (shouldPlaySounds) {
state.soundPlayer.playStartSound()
// Only set the setting if bluetooth is available, else it would reset the setting
// every time it's used without a bluetooth device connected.
if(device.bluetoothAvailable) {
manager.getLifecycleScope().launch {
context.setSetting(PREFER_BLUETOOTH, device.bluetoothActive)
}
}
}
}

119
voiceinput-shared/src/main/java/org/futo/voiceinput/shared/AudioRecognizer.kt
View File

@ -24,6 +24,7 @@ import com.konovalov.vad.config.SampleRate
import com.konovalov.vad.models.VadModel
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.coroutines.yield
@ -34,6 +35,7 @@ import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.types.ModelInferenceCallback
import org.futo.voiceinput.shared.types.ModelLoader
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.whisper.DecodingConfiguration
import org.futo.voiceinput.shared.whisper.ModelManager
import org.futo.voiceinput.shared.whisper.MultiModelRunConfiguration
@ -139,24 +141,39 @@ class AudioRecognizer(
}
}
private fun setCommunicationDevice() {
communicationDevice = "Unset"
if(!settings.recordingConfiguration.preferBluetoothMic) return
private fun isBluetoothAvailable(): Boolean {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val devices = audioManager.availableCommunicationDevices
val tgtDevice = devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO } ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first()
return devices.firstOrNull {
it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
} != null
}
} catch(_: Exception) {}
return false
}
private fun setCommunicationDevice(preferBluetoothMic: Boolean): Pair<Boolean, String> {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
val devices = audioManager.availableCommunicationDevices
val tgtDevice = devices.firstOrNull {
preferBluetoothMic && it.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
} ?: devices.firstOrNull { it.type == AudioDeviceInfo.TYPE_BUILTIN_MIC } ?: devices.first()
if (!audioManager.setCommunicationDevice(tgtDevice)) {
audioManager.clearCommunicationDevice()
return Pair(false, "")
} else {
communicationDevice =
tgtDevice.productName.toString() + " (${getRecordingDeviceKind(tgtDevice.type)})"
return Pair(tgtDevice.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO, tgtDevice.productName.toString())
}
}
} catch(_: Exception) {}
return Pair(false, "")
}
private fun clearCommunicationDevice() {
@ -251,14 +268,8 @@ class AudioRecognizer(
@Throws(SecurityException::class)
private fun createAudioRecorder(): AudioRecord {
val purpose = if(settings.recordingConfiguration.preferBluetoothMic) {
MediaRecorder.AudioSource.VOICE_COMMUNICATION
} else {
MediaRecorder.AudioSource.VOICE_RECOGNITION
}
val recorder = AudioRecord(
purpose,
MediaRecorder.AudioSource.VOICE_RECOGNITION,
16000,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
@ -300,7 +311,6 @@ class AudioRecognizer(
while (isRecording) {
yield()
val nRead = recorder.read(samples, 0, 1600, AudioRecord.READ_BLOCKING)
if (nRead <= 0) break
yield()
@ -414,15 +424,56 @@ class AudioRecognizer(
.setSpeechDurationMs(150).setSilenceDurationMs(300).build()
}
private fun startRecording() {
if (isRecording) {
throw IllegalStateException("Start recording when already recording")
@Throws(SecurityException::class)
private fun createRecorderAndJob(preferBluetoothMic: Boolean): MicrophoneDeviceState {
isRecording = false
recorder?.stop()
val bluetoothInfo = setCommunicationDevice(preferBluetoothMic)
val task = {
recorder?.release()
val recorder = createAudioRecorder()
recorder.startRecording()
this.recorder = recorder
isRecording = true
recorderJob = lifecycleScope.launch {
withContext(Dispatchers.Default) {
createVad().use { vad ->
recordingJob(recorder, vad)
}
}
}
}
setCommunicationDevice()
if(recorderJob != null) {
lifecycleScope.launch {
recorderJob?.cancelAndJoin()
task()
}
} else {
task()
}
val recorder = try {
createAudioRecorder()
return MicrophoneDeviceState(
bluetoothAvailable = bluetoothInfo.first || isBluetoothAvailable(),
bluetoothActive = bluetoothInfo.first,
deviceName = bluetoothInfo.second,
bluetoothPreferredByUser = settings.recordingConfiguration.preferBluetoothMic,
setBluetooth = {
listener.recordingStarted(createRecorderAndJob(it))
}
)
}
private fun startRecording() {
val device = try {
createRecorderAndJob(settings.recordingConfiguration.preferBluetoothMic)
} catch (e: SecurityException) {
// It's possible we may have lost permission, so let's just ask for permission again
clearCommunicationDevice()
@ -432,37 +483,13 @@ class AudioRecognizer(
focusAudio()
if(communicationDevice == "Unset") {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
communicationDevice = recorder.activeMicrophones.joinToString {
getRecordingDeviceKind(it.type)
} + "@"
}
}
listener.recordingStarted(communicationDevice)
recorder.startRecording()
this.recorder = recorder
isRecording = true
recorderJob = lifecycleScope.launch {
withContext(Dispatchers.Default) {
createVad().use { vad ->
recordingJob(recorder, vad)
}
}
}
listener.recordingStarted(device)
loadModelJob = lifecycleScope.launch {
withContext(Dispatchers.Default) {
preloadModels()
}
}
}
private val runnerCallback: ModelInferenceCallback = object : ModelInferenceCallback {

15
voiceinput-shared/src/main/java/org/futo/voiceinput/shared/RecognizerView.kt
View File

@ -11,6 +11,7 @@ import org.futo.voiceinput.shared.types.InferenceState
import org.futo.voiceinput.shared.types.Language
import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.ui.InnerRecognize
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
import org.futo.voiceinput.shared.ui.PartialDecodingResult
import org.futo.voiceinput.shared.ui.RecognizeLoadingCircle
import org.futo.voiceinput.shared.ui.RecognizeMicError
@ -48,7 +49,7 @@ private val DefaultAnnotations = hashMapOf(
interface RecognizerViewListener {
fun cancelled()
fun recordingStarted(device: String)
fun recordingStarted(device: MicrophoneDeviceState)
fun finished(result: String)
@ -76,7 +77,13 @@ class RecognizerView(
private val partialDecodingText = mutableStateOf("")
private val currentViewState = mutableStateOf(CurrentView.LoadingCircle)
private val currentDeviceState = mutableStateOf("Recording not started")
private val currentDeviceState = mutableStateOf(MicrophoneDeviceState(
bluetoothAvailable = false,
bluetoothActive = false,
setBluetooth = { },
deviceName = "",
bluetoothPreferredByUser = false
))
@Composable
fun Content() {
@ -97,7 +104,7 @@ class RecognizerView(
InnerRecognize(
magnitude = magnitudeState,
state = statusState,
device = if(settings.shouldShowVerboseFeedback) { currentDeviceState } else { null }
device = currentDeviceState
)
}
@ -172,7 +179,7 @@ class RecognizerView(
}
}
override fun recordingStarted(device: String) {
override fun recordingStarted(device: MicrophoneDeviceState) {
updateMagnitude(0.0f, MagnitudeState.NOT_TALKED_YET)
currentDeviceState.value = device
listener.recordingStarted(device)

4
voiceinput-shared/src/main/java/org/futo/voiceinput/shared/types/AudioRecognizerListener.kt
View File

@ -1,5 +1,7 @@
package org.futo.voiceinput.shared.types
import org.futo.voiceinput.shared.ui.MicrophoneDeviceState
enum class MagnitudeState {
NOT_TALKED_YET, MIC_MAY_BE_BLOCKED, TALKING
}
@ -14,7 +16,7 @@ interface AudioRecognizerListener {
fun loading()
fun needPermission(onResult: (Boolean) -> Unit)
fun recordingStarted(device: String)
fun recordingStarted(device: MicrophoneDeviceState)
fun updateMagnitude(magnitude: Float, state: MagnitudeState)
fun processing()

127
voiceinput-shared/src/main/java/org/futo/voiceinput/shared/ui/RecognizeViews.kt
View File

@ -1,7 +1,12 @@
package org.futo.voiceinput.shared.ui
import androidx.compose.animation.AnimatedVisibility
import androidx.compose.animation.fadeIn
import androidx.compose.animation.fadeOut
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxScope
import androidx.compose.foundation.layout.ColumnScope
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.defaultMinSize
@ -21,21 +26,46 @@ import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.MutableFloatState
import androidx.compose.runtime.MutableState
import androidx.compose.runtime.mutableFloatStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.alpha
import androidx.compose.ui.draw.drawBehind
import androidx.compose.ui.geometry.CornerRadius
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.drawscope.Fill
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.semantics.clearAndSetSemantics
import androidx.compose.ui.semantics.role
import androidx.compose.ui.semantics.text
import androidx.compose.ui.semantics.toggleableState
import androidx.compose.ui.state.ToggleableState
import androidx.compose.ui.text.AnnotatedString
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import kotlinx.coroutines.delay
import org.futo.voiceinput.shared.R
import org.futo.voiceinput.shared.types.MagnitudeState
import org.futo.voiceinput.shared.ui.theme.Typography
data class MicrophoneDeviceState(
val bluetoothAvailable: Boolean,
val bluetoothActive: Boolean,
val bluetoothPreferredByUser: Boolean,
val setBluetooth: (Boolean) -> Unit,
val deviceName: String
)
@Composable
fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0.5f)) {
val radius = animateValueChanges(magnitude.floatValue, 100)
@ -51,11 +81,92 @@ fun AnimatedRecognizeCircle(magnitude: MutableFloatState = mutableFloatStateOf(0
}
}
@Composable
private fun FakeToast(modifier: Modifier, message: String?) {
val visible = remember { mutableStateOf(false) }
LaunchedEffect(message) {
if(message != null) {
visible.value = true
delay(2500L)
visible.value = false
} else {
visible.value = false
}
}
if(message != null) {
AnimatedVisibility(
visible = visible.value,
modifier = modifier.alpha(0.9f).background(MaterialTheme.colorScheme.surfaceDim, RoundedCornerShape(100)).padding(16.dp),
enter = fadeIn(),
exit = fadeOut()
) {
Box {
Text(message, modifier = Modifier.align(Alignment.Center), style = Typography.labelSmall)
}
}
}
}
@Composable
private fun BoxScope.BluetoothToggleIcon(device: MutableState<MicrophoneDeviceState>? = null) {
FakeToast(modifier = Modifier.align(Alignment.BottomCenter).offset(y = (-16).dp), message = if(device?.value?.bluetoothActive == true) {
"Using Bluetooth mic (${device.value.deviceName})"
} else if(device?.value?.bluetoothAvailable == true || device?.value?.bluetoothPreferredByUser == true) {
"Using Built-in mic (${device.value.deviceName})"
} else {
null
})
if(device?.value?.bluetoothAvailable == true) {
val bluetoothColor = MaterialTheme.colorScheme.primary
val iconColor = if(device.value.bluetoothActive) {
MaterialTheme.colorScheme.onPrimary
} else {
MaterialTheme.colorScheme.onSurface
}
IconButton(modifier = Modifier
.align(Alignment.BottomEnd)
.offset(x = (-16).dp, y = (-16).dp)
.drawBehind {
val radius = size.height / 4.0f
drawRoundRect(
bluetoothColor,
topLeft = Offset(size.width * 0.1f, size.height * 0.05f),
size = Size(size.width * 0.8f, size.height * 0.9f),
cornerRadius = CornerRadius(radius, radius),
style = if (device.value.bluetoothActive) {
Fill
} else {
Stroke(width = 4.0f)
}
)
}
.clearAndSetSemantics {
this.text = AnnotatedString("Use bluetooth mic")
this.role = Role.Switch
this.toggleableState = ToggleableState(device.value.bluetoothActive)
},
onClick = {
device.value.setBluetooth(!device.value.bluetoothActive)
},
) {
Icon(
painter = painterResource(id = R.drawable.bluetooth),
contentDescription = null,
tint = iconColor
)
}
}
}
@Composable
fun InnerRecognize(
magnitude: MutableFloatState = mutableFloatStateOf(0.5f),
state: MutableState<MagnitudeState> = mutableStateOf(MagnitudeState.MIC_MAY_BE_BLOCKED),
device: MutableState<String>? = mutableStateOf("")
device: MutableState<MicrophoneDeviceState>? = null
) {
Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) {
AnimatedRecognizeCircle(magnitude = magnitude)
@ -75,20 +186,14 @@ fun InnerRecognize(
Text(
text,
modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 48.dp),
modifier = Modifier
.fillMaxWidth()
.offset(x = 0.dp, y = 48.dp),
textAlign = TextAlign.Center,
color = MaterialTheme.colorScheme.onSurface
)
if(device != null) {
Text(
"Device: ${device.value}",
style = Typography.labelSmall,
modifier = Modifier.fillMaxWidth().offset(x = 0.dp, y = 64.dp),
textAlign = TextAlign.Center,
color = MaterialTheme.colorScheme.onSurface.copy(alpha = 0.66f)
)
}
BluetoothToggleIcon(device)
}
}

13
voiceinput-shared/src/main/res/drawable/bluetooth.xml Normal file
View File

@ -0,0 +1,13 @@
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path
android:pathData="M6.5,6.5l11,11l-5.5,5.5l0,-22l5.5,5.5l-11,11"
android:strokeLineJoin="round"
android:strokeWidth="2"
android:fillColor="#00000000"
android:strokeColor="#ffffff"
android:strokeLineCap="round"/>
</vector>