From 4f15ff4a73d025c0901eccce49d1b829a24789e1 Mon Sep 17 00:00:00 2001 From: Aleksandras Kostarevas Date: Tue, 28 Nov 2023 17:01:58 +0000 Subject: [PATCH] Add experimental swipe typing --- java/res/values/gesture-input.xml | 2 +- .../keyboard/MainKeyboardView.java | 2 +- .../inputmethod/latin/LatinIMELegacy.java | 2 + .../org/futo/inputmethod/latin/Suggest.java | 6 +- .../latin/inputlogic/InputLogic.java | 7 + .../latin/xlm/BatchInputConverter.kt | 63 +++++++++ .../inputmethod/latin/xlm/LanguageModel.java | 39 +++--- .../latin/xlm/LanguageModelFacilitator.kt | 125 +++--------------- ...to_inputmethod_latin_xlm_LanguageModel.cpp | 2 +- 9 files changed, 113 insertions(+), 135 deletions(-) create mode 100644 java/src/org/futo/inputmethod/latin/xlm/BatchInputConverter.kt diff --git a/java/res/values/gesture-input.xml b/java/res/values/gesture-input.xml index 235616fbe..445a389b8 100644 --- a/java/res/values/gesture-input.xml +++ b/java/res/values/gesture-input.xml @@ -18,5 +18,5 @@ */ --> - false + true diff --git a/java/src/org/futo/inputmethod/keyboard/MainKeyboardView.java b/java/src/org/futo/inputmethod/keyboard/MainKeyboardView.java index a7808cc04..c62adf8ce 100644 --- a/java/src/org/futo/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/org/futo/inputmethod/keyboard/MainKeyboardView.java @@ -162,7 +162,7 @@ public final class MainKeyboardView extends KeyboardView implements DrawingProxy // TODO: Make this parameter customizable by user via settings. private int mGestureFloatingPreviewTextLingerTimeout; - private final KeyDetector mKeyDetector; + public final KeyDetector mKeyDetector; private final NonDistinctMultitouchHelper mNonDistinctMultitouchHelper; private final TimerHandler mTimerHandler; diff --git a/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java b/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java index a4ba15308..74f123d34 100644 --- a/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java +++ b/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java @@ -784,6 +784,8 @@ public class LatinIMELegacy implements KeyboardActionListener, } public boolean isImeSuppressedByHardwareKeyboard() { + if(true) return false; // TODO: This function returning true causes some initialization issues + final KeyboardSwitcher switcher = KeyboardSwitcher.getInstance(); return !onEvaluateInputViewShown() && switcher.isImeSuppressedByHardwareKeyboard( mSettings.getCurrent(), switcher.getKeyboardSwitchState()); diff --git a/java/src/org/futo/inputmethod/latin/Suggest.java b/java/src/org/futo/inputmethod/latin/Suggest.java index eb6a2f3e0..30ddacfcc 100644 --- a/java/src/org/futo/inputmethod/latin/Suggest.java +++ b/java/src/org/futo/inputmethod/latin/Suggest.java @@ -124,7 +124,7 @@ public final class Suggest { || 0 != trailingSingleQuotesCount) { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - final Locale wordLocale = wordInfo.mSourceDict.mLocale; + final Locale wordLocale = (wordInfo.mSourceDict != null) ? wordInfo.mSourceDict.mLocale : null; final SuggestedWordInfo transformedWordInfo = getTransformedSuggestedWordInfo( wordInfo, null == wordLocale ? defaultLocale : wordLocale, shouldMakeSuggestionsAllUpperCase, isOnlyFirstCharCapitalized, @@ -318,7 +318,7 @@ public final class Suggest { if (isFirstCharCapitalized || isAllUpperCase) { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - final Locale wordlocale = wordInfo.mSourceDict.mLocale; + final Locale wordlocale = (wordInfo.mSourceDict != null) ? wordInfo.mSourceDict.mLocale : null; final SuggestedWordInfo transformedWordInfo = getTransformedSuggestedWordInfo( wordInfo, null == wordlocale ? locale : wordlocale, isAllUpperCase, isFirstCharCapitalized, 0 /* trailingSingleQuotesCount */); @@ -407,7 +407,7 @@ public final class Suggest { * @return whether it's fine to auto-correct to this. */ private static boolean isAllowedByAutoCorrectionWithSpaceFilter(final SuggestedWordInfo info) { - final Locale locale = info.mSourceDict.mLocale; + final Locale locale = (info.mSourceDict != null) ? info.mSourceDict.mLocale : null; if (null == locale) { return true; } diff --git a/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java b/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java index c24b82098..41ff7f60b 100644 --- a/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java @@ -491,6 +491,11 @@ public final class InputLogic { return inputTransaction; } + public void showBatchSuggestions(final SuggestedWords suggestedWordsForBatchInput, + final boolean isTailBatchInput) { + mInputLogicHandler.showGestureSuggestionsWithPreviewVisuals(suggestedWordsForBatchInput, isTailBatchInput); + } + public void onStartBatchInput(final SettingsValues settingsValues, final KeyboardSwitcher keyboardSwitcher, final LatinIMELegacy.UIHandler handler) { mWordBeingCorrectedByCursor = null; @@ -560,7 +565,9 @@ public final class InputLogic { */ private int mAutoCommitSequenceNumber = 1; public void onUpdateBatchInput(final InputPointers batchPointers) { + Log.d(TAG, "InputLogic has received batch input update, now we call for " + mInputLogicHandler.toString()); mInputLogicHandler.onUpdateBatchInput(batchPointers, mAutoCommitSequenceNumber); + Log.d(TAG, "Finished calling onUpddateBatchInput"); } public void onEndBatchInput(final InputPointers batchPointers) { diff --git a/java/src/org/futo/inputmethod/latin/xlm/BatchInputConverter.kt b/java/src/org/futo/inputmethod/latin/xlm/BatchInputConverter.kt new file mode 100644 index 000000000..f3668cdab --- /dev/null +++ b/java/src/org/futo/inputmethod/latin/xlm/BatchInputConverter.kt @@ -0,0 +1,63 @@ +package org.futo.inputmethod.latin.xlm + +import org.futo.inputmethod.keyboard.KeyDetector +import kotlin.math.sqrt + +private fun normalize(pair: Pair): Pair { + val magnitude = sqrt((pair.first * pair.first + pair.second * pair.second).toDouble()) + + if(magnitude == 0.0) { + return Pair(Float.NaN, Float.NaN) + } + + return Pair((pair.first.toFloat() / magnitude).toFloat(), (pair.second.toFloat() / magnitude).toFloat()) +} + +private fun dot(pair1: Pair, pair2: Pair): Float { + return pair1.first * pair2.first + pair1.second * pair2.second +} + +object BatchInputConverter { + fun convertToString(x: IntArray, y: IntArray, size: Int, keyDetector: KeyDetector): String { + var coords = x.zip(y).toMutableList() + + var s = "" + for(i in 0 until size){ + if((i == 0) || (i == (size - 1))) { + val key = + keyDetector.detectHitKey(coords[i].first, coords[i].second)?.label ?: continue + if(s.isNotEmpty() && s.last() == key.first()) continue + s += key + continue + } + + val currCoord = coords[i] + val lastCoord = coords[i - 1] + val nextCoord = coords[i + 1] + + val directionFromLastCoord = normalize(Pair(currCoord.first - lastCoord.first, currCoord.second - lastCoord.second)) + val directionFromNextCoord = normalize(Pair(nextCoord.first - currCoord.first, nextCoord.second - currCoord.second)) + + if(directionFromLastCoord.first.isNaN() || directionFromLastCoord.second.isNaN()) continue + if(directionFromNextCoord.first.isNaN() || directionFromNextCoord.second.isNaN()) continue + + val dot = dot(directionFromLastCoord, directionFromNextCoord) + + // TODO: Figure out a good threshold + if(dot < 0.95) { + val key = + keyDetector.detectHitKey(coords[i].first, coords[i].second)?.label ?: continue + if(s.isNotEmpty() && s.last() == key.first()) continue + s += key + //println("Adding $key, dot $dot, dirs $directionFromLastCoord $directionFromNextCoord, coords $lastCoord $currCoord $nextCoord") + } else { + // Simplify + coords[i] = lastCoord + } + } + + println("Transformed string: [$s]") + + return s.lowercase() + } +} \ No newline at end of file diff --git a/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java b/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java index 6300e41ed..28cf85897 100644 --- a/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java +++ b/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java @@ -3,34 +3,24 @@ package org.futo.inputmethod.latin.xlm; import android.content.Context; import android.util.Log; -import org.futo.inputmethod.latin.Dictionary; +import org.futo.inputmethod.keyboard.KeyDetector; import org.futo.inputmethod.latin.NgramContext; -import org.futo.inputmethod.latin.R; import org.futo.inputmethod.latin.SuggestedWords; import org.futo.inputmethod.latin.common.ComposedData; import org.futo.inputmethod.latin.common.InputPointers; import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; -import java.util.function.IntPredicate; -// TODO: Avoid loading the LanguageModel if the setting is disabled -public class LanguageModel extends Dictionary { +public class LanguageModel { static long mNativeState = 0; Context context = null; Thread initThread = null; Locale locale = null; public LanguageModel(Context context, String dictType, Locale locale) { - super(dictType, locale); - this.context = context; this.locale = locale; } @@ -64,11 +54,10 @@ public class LanguageModel extends Dictionary { initThread.start(); } - @Override public ArrayList getSuggestions( ComposedData composedData, NgramContext ngramContext, - long proximityInfoHandle, + KeyDetector keyDetector, SettingsValuesForSuggestion settingsValuesForSuggestion, int sessionId, float weightForLocale, @@ -108,6 +97,16 @@ public class LanguageModel extends Dictionary { context = context.substring(0, context.length() - partialWord.length()).trim(); } + if(isGesture) { + // Partial word is gonna be derived from batch data + partialWord = BatchInputConverter.INSTANCE.convertToString( + composedData.mInputPointers.getXCoordinates(), + composedData.mInputPointers.getYCoordinates(), + inputSize, + keyDetector + ); + } + if(!partialWord.isEmpty()) { partialWord = partialWord.trim(); } @@ -160,7 +159,7 @@ public class LanguageModel extends Dictionary { String[] outStrings = new String[maxResults]; // TOOD: Pass multiple previous words information for n-gram. - getSuggestionsNative(mNativeState, proximityInfoHandle, context, partialWord, xCoords, yCoords, outStrings, outProbabilities); + getSuggestionsNative(mNativeState, 0L, context, partialWord, xCoords, yCoords, outStrings, outProbabilities); final ArrayList suggestions = new ArrayList<>(); @@ -197,7 +196,7 @@ public class LanguageModel extends Dictionary { currKind |= SuggestedWords.SuggestedWordInfo.KIND_FLAG_EXACT_MATCH; } - suggestions.add(new SuggestedWords.SuggestedWordInfo( word, context, (int)(outProbabilities[i] * 100.0f), currKind, this, 0, 0 )); + suggestions.add(new SuggestedWords.SuggestedWordInfo( word, context, (int)(outProbabilities[i] * 100.0f), currKind, null, 0, 0 )); } /* @@ -235,7 +234,6 @@ public class LanguageModel extends Dictionary { } } - @Override protected void finalize() throws Throwable { try { @@ -245,13 +243,6 @@ public class LanguageModel extends Dictionary { } } - @Override - public boolean isInDictionary(String word) { - // TODO: Provide the word spelling to the model and see if the probability of correcting it to that is beyond a certain limit - return false; - } - - private static native long openNative(String sourceDir); private static native void closeNative(long state); private static native void getSuggestionsNative( diff --git a/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt index 67e9c38cb..eb714921c 100644 --- a/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt +++ b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt @@ -1,119 +1,30 @@ package org.futo.inputmethod.latin.xlm; -import android.content.ComponentCallbacks2 import android.content.Context -import android.content.res.Configuration -import android.inputmethodservice.InputMethodService -import android.os.Build -import android.os.Bundle -import android.view.KeyEvent -import android.view.View -import android.view.ViewGroup -import android.view.inputmethod.CompletionInfo -import android.view.inputmethod.EditorInfo -import android.view.inputmethod.InlineSuggestion -import android.view.inputmethod.InlineSuggestionsRequest -import android.view.inputmethod.InlineSuggestionsResponse -import android.view.inputmethod.InputMethodSubtype -import androidx.annotation.RequiresApi -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.Spacer -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.size -import androidx.compose.material3.ColorScheme -import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Surface -import androidx.compose.material3.Text -import androidx.compose.runtime.Composable -import androidx.compose.runtime.MutableState -import androidx.compose.runtime.key -import androidx.compose.ui.Alignment.Companion.CenterVertically -import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.clipToBounds -import androidx.compose.ui.layout.onSizeChanged -import androidx.compose.ui.platform.ComposeView -import androidx.compose.ui.platform.LocalDensity -import androidx.compose.ui.platform.ViewCompositionStrategy -import androidx.compose.ui.res.painterResource -import androidx.compose.ui.unit.dp -import androidx.compose.ui.viewinterop.AndroidView -import androidx.lifecycle.Lifecycle import androidx.lifecycle.LifecycleCoroutineScope -import androidx.lifecycle.LifecycleOwner -import androidx.lifecycle.LifecycleRegistry -import androidx.lifecycle.ViewModelStore -import androidx.lifecycle.ViewModelStoreOwner -import androidx.lifecycle.findViewTreeLifecycleOwner -import androidx.lifecycle.findViewTreeViewModelStoreOwner -import androidx.lifecycle.lifecycleScope -import androidx.lifecycle.setViewTreeLifecycleOwner -import androidx.lifecycle.setViewTreeViewModelStoreOwner -import androidx.savedstate.SavedStateRegistry -import androidx.savedstate.SavedStateRegistryController -import androidx.savedstate.SavedStateRegistryOwner -import androidx.savedstate.findViewTreeSavedStateRegistryOwner -import androidx.savedstate.setViewTreeSavedStateRegistryOwner -import androidx.work.WorkManager -import kotlinx.coroutines.Job -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.MutableSharedFlow -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.withContext -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.conflate -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout -import org.futo.inputmethod.latin.common.Constants -import org.futo.inputmethod.latin.common.ComposedData -import org.futo.inputmethod.latin.uix.Action -import org.futo.inputmethod.latin.uix.ActionBar -import org.futo.inputmethod.latin.uix.ActionInputTransaction -import org.futo.inputmethod.latin.uix.ActionWindow -import org.futo.inputmethod.latin.uix.BasicThemeProvider -import org.futo.inputmethod.latin.uix.DynamicThemeProvider -import org.futo.inputmethod.latin.uix.DynamicThemeProviderOwner -import org.futo.inputmethod.latin.uix.KeyboardManagerForAction -import org.futo.inputmethod.latin.uix.PersistentActionState -import org.futo.inputmethod.latin.uix.THEME_KEY -import org.futo.inputmethod.latin.uix.actions.VoiceInputAction -import org.futo.inputmethod.latin.uix.createInlineSuggestionsRequest -import org.futo.inputmethod.latin.uix.deferGetSetting -import org.futo.inputmethod.latin.uix.deferSetSetting -import org.futo.inputmethod.latin.uix.differsFrom -import org.futo.inputmethod.latin.uix.inflateInlineSuggestion -import org.futo.inputmethod.latin.uix.theme.DarkColorScheme -import org.futo.inputmethod.latin.uix.theme.ThemeOption -import org.futo.inputmethod.latin.uix.theme.ThemeOptions -import org.futo.inputmethod.latin.uix.theme.Typography -import org.futo.inputmethod.latin.uix.theme.UixThemeWrapper -import org.futo.inputmethod.latin.uix.theme.presets.ClassicMaterialDark -import org.futo.inputmethod.latin.uix.theme.presets.DynamicSystemTheme -import org.futo.inputmethod.latin.uix.theme.presets.VoiceInputTheme -import org.futo.inputmethod.latin.settings.SettingsValues; -import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion -import org.futo.inputmethod.latin.settings.Settings -import org.futo.inputmethod.latin.xlm.LanguageModel; -import org.futo.inputmethod.latin.utils.SuggestionResults -import org.futo.inputmethod.latin.NgramContext -import org.futo.inputmethod.latin.LatinIMELegacy -import org.futo.inputmethod.latin.inputlogic.InputLogic +import org.futo.inputmethod.keyboard.KeyboardSwitcher import org.futo.inputmethod.latin.DictionaryFacilitator +import org.futo.inputmethod.latin.NgramContext import org.futo.inputmethod.latin.Suggest import org.futo.inputmethod.latin.SuggestedWords -import org.futo.inputmethod.keyboard.KeyboardSwitcher +import org.futo.inputmethod.latin.common.ComposedData +import org.futo.inputmethod.latin.inputlogic.InputLogic +import org.futo.inputmethod.latin.settings.Settings +import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion +import org.futo.inputmethod.latin.utils.SuggestionResults public class LanguageModelFacilitator( val context: Context, @@ -186,7 +97,7 @@ public class LanguageModelFacilitator( val lmSuggestions = languageModel!!.getSuggestions( values.composedData, values.ngramContext, - proximityInfoHandle, + keyboardSwitcher.mainKeyboardView.mKeyDetector, settingsForPrediction, -1, 0.0f, @@ -209,6 +120,10 @@ public class LanguageModelFacilitator( job.cancel() inputLogic.mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords) + + if(values.composedData.mIsBatchMode) { + inputLogic.showBatchSuggestions(suggestedWords, values.inputStyle == SuggestedWords.INPUT_STYLE_TAIL_BATCH); + } sequenceIdFinishedFlow.emit(values.sequenceId) } finally { computationSemaphore.release() diff --git a/native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp b/native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp index 0fc835d14..23eb340da 100644 --- a/native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp +++ b/native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp @@ -436,7 +436,7 @@ namespace latinime { static void xlm_LanguageModel_getSuggestions(JNIEnv *env, jclass clazz, // inputs jlong dict, - jlong proximityInfo, + jlong _unused, jstring context, jstring partialWord, jfloatArray inComposeX,