From 33be6fb3ed46618dc8e58bb6f9c0cdef02d8e269 Mon Sep 17 00:00:00 2001 From: Aleksandras Kostarevas Date: Mon, 13 Nov 2023 16:44:52 +0200 Subject: [PATCH] Refactor LanguageModel invocation code --- java/AndroidManifest.xml | 1 + java/res/values/config-common.xml | 3 +- .../futo/inputmethod/latin/Dictionary.java | 1 - .../latin/DictionaryFacilitator.java | 1 - .../latin/DictionaryFacilitatorImpl.java | 20 +- .../org/futo/inputmethod/latin/LatinIME.kt | 30 +++ .../inputmethod/latin/LatinIMELegacy.java | 32 ++- .../org/futo/inputmethod/latin/Suggest.java | 44 ++-- .../latin/inputlogic/InputLogic.java | 144 +++++----- .../latin/settings/SettingsValues.java | 5 - .../inputmethod/latin/xlm/LanguageModel.java | 6 +- .../latin/xlm/LanguageModelFacilitator.kt | 246 ++++++++++++++++++ 12 files changed, 422 insertions(+), 111 deletions(-) create mode 100644 java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt diff --git a/java/AndroidManifest.xml b/java/AndroidManifest.xml index e25812552..eb764e8ff 100644 --- a/java/AndroidManifest.xml +++ b/java/AndroidManifest.xml @@ -33,6 +33,7 @@ + 3 36% - 100 - 300 + 0 12dp diff --git a/java/src/org/futo/inputmethod/latin/Dictionary.java b/java/src/org/futo/inputmethod/latin/Dictionary.java index b2cfde048..a826087e0 100644 --- a/java/src/org/futo/inputmethod/latin/Dictionary.java +++ b/java/src/org/futo/inputmethod/latin/Dictionary.java @@ -64,7 +64,6 @@ public abstract class Dictionary { // User history dictionary internal to LatinIME. public static final String TYPE_USER_HISTORY = "history"; - public static final String TYPE_GGML = "ggml"; public final String mDictType; // The locale for this dictionary. May be null if unknown (phony dictionary for example). public final Locale mLocale; diff --git a/java/src/org/futo/inputmethod/latin/DictionaryFacilitator.java b/java/src/org/futo/inputmethod/latin/DictionaryFacilitator.java index 09f1ed4e3..70f13c1ca 100644 --- a/java/src/org/futo/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/org/futo/inputmethod/latin/DictionaryFacilitator.java @@ -45,7 +45,6 @@ import javax.annotation.Nullable; public interface DictionaryFacilitator { public static final String[] ALL_DICTIONARY_TYPES = new String[] { - Dictionary.TYPE_GGML, Dictionary.TYPE_MAIN, Dictionary.TYPE_CONTACTS, Dictionary.TYPE_USER_HISTORY, diff --git a/java/src/org/futo/inputmethod/latin/DictionaryFacilitatorImpl.java b/java/src/org/futo/inputmethod/latin/DictionaryFacilitatorImpl.java index c4891aed8..5b107f5d4 100644 --- a/java/src/org/futo/inputmethod/latin/DictionaryFacilitatorImpl.java +++ b/java/src/org/futo/inputmethod/latin/DictionaryFacilitatorImpl.java @@ -136,8 +136,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { @Nullable public final String mAccount; @Nullable private Dictionary mMainDict; - - @Nullable private LanguageModel mGGMLDict = null; // Confidence that the most probable language is actually the language the user is // typing in. For now, this is simply the number of times a word from this language // has been committed in a row. @@ -185,9 +183,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { if (Dictionary.TYPE_MAIN.equals(dictType)) { return mMainDict; } - if (Dictionary.TYPE_GGML.equals(dictType)) { - return mGGMLDict; - } return getSubDict(dictType); } @@ -199,9 +194,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { if (Dictionary.TYPE_MAIN.equals(dictType)) { return mMainDict != null; } - if (Dictionary.TYPE_GGML.equals(dictType)) { - return mGGMLDict != null; - } if (Dictionary.TYPE_USER_HISTORY.equals(dictType) && !TextUtils.equals(account, mAccount)) { // If the dictionary type is user history, & if the account doesn't match, @@ -358,7 +350,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { DictionaryGroup newDictionaryGroup = new DictionaryGroup(newLocale, mainDict, account, subDicts); - newDictionaryGroup.mGGMLDict = new LanguageModel(context, Dictionary.TYPE_GGML, newLocale); // Replace Dictionaries. final DictionaryGroup oldDictionaryGroup; synchronized (mLock) { @@ -416,7 +407,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { synchronized (mLock) { if (locale.equals(dictionaryGroup.mLocale)) { dictionaryGroup.setMainDict(mainDict); - dictionaryGroup.mGGMLDict = new LanguageModel(context, Dictionary.TYPE_GGML, locale); } else { // Dictionary facilitator has been reset for another locale. mainDict.close(); @@ -628,6 +618,13 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { NgramContext ngramContext, @Nonnull final Keyboard keyboard, SettingsValuesForSuggestion settingsValuesForSuggestion, int sessionId, int inputStyle) { + + if(settingsValuesForSuggestion.mUseTransformerLM) { + throw new IllegalStateException("Invalid code path TransformerLM"); + } + + + long proximityInfoHandle = keyboard.getProximityInfo().getNativeProximityInfo(); final SuggestionResults suggestionResults = new SuggestionResults( SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext(), @@ -635,9 +632,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { final float[] weightOfLangModelVsSpatialModel = new float[] { Dictionary.NOT_A_WEIGHT_OF_LANG_MODEL_VS_SPATIAL_MODEL }; for (final String dictType : ALL_DICTIONARY_TYPES) { - if(settingsValuesForSuggestion.mUseTransformerLM && dictType != Dictionary.TYPE_GGML) continue; - else if(!settingsValuesForSuggestion.mUseTransformerLM && dictType == Dictionary.TYPE_GGML) continue; - final Dictionary dictionary = mDictionaryGroup.getDict(dictType); if (null == dictionary) continue; final float weightForLocale = composedData.mIsBatchMode diff --git a/java/src/org/futo/inputmethod/latin/LatinIME.kt b/java/src/org/futo/inputmethod/latin/LatinIME.kt index eddafb015..71f8bea2c 100644 --- a/java/src/org/futo/inputmethod/latin/LatinIME.kt +++ b/java/src/org/futo/inputmethod/latin/LatinIME.kt @@ -60,9 +60,19 @@ import androidx.savedstate.findViewTreeSavedStateRegistryOwner import androidx.savedstate.setViewTreeSavedStateRegistryOwner import kotlinx.coroutines.Job import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.delay import org.futo.inputmethod.latin.common.Constants +import org.futo.inputmethod.latin.common.ComposedData import org.futo.inputmethod.latin.uix.Action import org.futo.inputmethod.latin.uix.ActionBar import org.futo.inputmethod.latin.uix.ActionInputTransaction @@ -87,6 +97,11 @@ import org.futo.inputmethod.latin.uix.theme.UixThemeWrapper import org.futo.inputmethod.latin.uix.theme.presets.ClassicMaterialDark import org.futo.inputmethod.latin.uix.theme.presets.DynamicSystemTheme import org.futo.inputmethod.latin.uix.theme.presets.VoiceInputTheme +import org.futo.inputmethod.latin.settings.SettingsValues; +import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion +import org.futo.inputmethod.latin.xlm.LanguageModel; +import org.futo.inputmethod.latin.xlm.LanguageModelFacilitator; +import org.futo.inputmethod.latin.utils.SuggestionResults class LatinIME : InputMethodService(), LifecycleOwner, ViewModelStoreOwner, SavedStateRegistryOwner, LatinIMELegacy.SuggestionStripController, DynamicThemeProviderOwner, KeyboardManagerForAction { @@ -127,6 +142,15 @@ class LatinIME : InputMethodService(), LifecycleOwner, ViewModelStoreOwner, Save this as LatinIMELegacy.SuggestionStripController ) + public val languageModelFacilitator = LanguageModelFacilitator( + this, + latinIMELegacy.mInputLogic, + latinIMELegacy.mDictionaryFacilitator, + latinIMELegacy.mSettings, + latinIMELegacy.mKeyboardSwitcher, + lifecycleScope + ) + private var activeThemeOption: ThemeOption? = null private var activeColorScheme = DarkColorScheme private var colorSchemeLoaderJob: Job? = null @@ -209,6 +233,8 @@ class LatinIME : InputMethodService(), LifecycleOwner, ViewModelStoreOwner, Save handleLifecycleEvent(Lifecycle.Event.ON_RESUME) latinIMELegacy.onCreate() + + languageModelFacilitator.launchProcessor() } override fun onDestroy() { @@ -698,4 +724,8 @@ class LatinIME : InputMethodService(), LifecycleOwner, ViewModelStoreOwner, Save return true } + + fun postUpdateSuggestionStrip(inputStyle: Int) { + languageModelFacilitator.updateSuggestionStripAsync(inputStyle); + } } \ No newline at end of file diff --git a/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java b/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java index bdcfbb884..668566ed7 100644 --- a/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java +++ b/java/src/org/futo/inputmethod/latin/LatinIMELegacy.java @@ -101,6 +101,7 @@ import org.futo.inputmethod.latin.utils.StatsUtils; import org.futo.inputmethod.latin.utils.StatsUtilsManager; import org.futo.inputmethod.latin.utils.SubtypeLocaleUtils; import org.futo.inputmethod.latin.utils.ViewLayoutUtils; +import org.futo.inputmethod.latin.xlm.LanguageModelFacilitator; import java.io.FileDescriptor; import java.io.PrintWriter; @@ -157,7 +158,7 @@ public class LatinIMELegacy implements KeyboardActionListener, private static final String SCHEME_PACKAGE = "package"; final Settings mSettings; - private final DictionaryFacilitator mDictionaryFacilitator = + final DictionaryFacilitator mDictionaryFacilitator = DictionaryFacilitatorProvider.getDictionaryFacilitator( false /* isNeededForSpellChecking */); final InputLogic mInputLogic; @@ -220,7 +221,7 @@ public class LatinIMELegacy implements KeyboardActionListener, public static final class UIHandler extends LeakGuardHandlerWrapper { private static final int MSG_UPDATE_SHIFT_STATE = 0; private static final int MSG_PENDING_IMS_CALLBACK = 1; - private static final int MSG_UPDATE_SUGGESTION_STRIP = 2; + private static final int MSG_UPDATE_SUGGESTION_STRIP_LEGACY = 2; private static final int MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP = 3; private static final int MSG_RESUME_SUGGESTIONS = 4; private static final int MSG_REOPEN_DICTIONARIES = 5; @@ -266,7 +267,7 @@ public class LatinIMELegacy implements KeyboardActionListener, } final KeyboardSwitcher switcher = latinImeLegacy.mKeyboardSwitcher; switch (msg.what) { - case MSG_UPDATE_SUGGESTION_STRIP: + case MSG_UPDATE_SUGGESTION_STRIP_LEGACY: cancelUpdateSuggestionStrip(); latinImeLegacy.mInputLogic.performUpdateSuggestionStripSync( latinImeLegacy.mSettings.getCurrent(), msg.arg1 /* inputStyle */); @@ -332,8 +333,13 @@ public class LatinIMELegacy implements KeyboardActionListener, } public void postUpdateSuggestionStrip(final int inputStyle) { - sendMessageDelayed(obtainMessage(MSG_UPDATE_SUGGESTION_STRIP, inputStyle, - 0 /* ignored */), mDelayInMillisecondsToUpdateSuggestions); + final LatinIMELegacy latinImeLegacy = getOwnerInstance(); + if(latinImeLegacy.mSettings.getCurrent().mTransformerPredictionEnabled) { + ((LatinIME)latinImeLegacy.getInputMethodService()).postUpdateSuggestionStrip(inputStyle); + } else { + sendMessageDelayed(obtainMessage(MSG_UPDATE_SUGGESTION_STRIP_LEGACY, inputStyle, + 0 /* ignored */), mDelayInMillisecondsToUpdateSuggestions); + } } public void postReopenDictionaries() { @@ -389,11 +395,16 @@ public class LatinIMELegacy implements KeyboardActionListener, } public void cancelUpdateSuggestionStrip() { - removeMessages(MSG_UPDATE_SUGGESTION_STRIP); + removeMessages(MSG_UPDATE_SUGGESTION_STRIP_LEGACY); } public boolean hasPendingUpdateSuggestions() { - return hasMessages(MSG_UPDATE_SUGGESTION_STRIP); + return hasMessages(MSG_UPDATE_SUGGESTION_STRIP_LEGACY); + } + + public LanguageModelFacilitator getLanguageModelFacilitator() { + final LatinIMELegacy latinImeLegacy = getOwnerInstance(); + return ((LatinIME)(latinImeLegacy.mInputMethodService)).getLanguageModelFacilitator(); } public boolean hasPendingReopenDictionaries() { @@ -1562,12 +1573,17 @@ public class LatinIMELegacy implements KeyboardActionListener, // TODO[IL]: Move this out of LatinIME. public void getSuggestedWords(final int inputStyle, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { + SettingsValues settings = mSettings.getCurrent(); + if(settings.mTransformerPredictionEnabled) { + ((LatinIME)getInputMethodService()).postUpdateSuggestionStrip(inputStyle); + return; + } final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); if (keyboard == null) { callback.onGetSuggestedWords(SuggestedWords.getEmptyInstance()); return; } - mInputLogic.getSuggestedWords(mSettings.getCurrent(), keyboard, + mInputLogic.getSuggestedWords(settings, keyboard, mKeyboardSwitcher.getKeyboardShiftMode(), inputStyle, sequenceNumber, callback); } diff --git a/java/src/org/futo/inputmethod/latin/Suggest.java b/java/src/org/futo/inputmethod/latin/Suggest.java index 06f47fa4c..eb6a2f3e0 100644 --- a/java/src/org/futo/inputmethod/latin/Suggest.java +++ b/java/src/org/futo/inputmethod/latin/Suggest.java @@ -147,13 +147,12 @@ public final class Suggest { return firstSuggestedWordInfo; } - // Retrieves suggestions for non-batch input (typing, recorrection, predictions...) - // and calls the callback function with the suggestions. - private void getSuggestedWordsForNonBatchInput(final WordComposer wordComposer, - final NgramContext ngramContext, final Keyboard keyboard, - final SettingsValuesForSuggestion settingsValuesForSuggestion, - final int inputStyleIfNotPrediction, final boolean isCorrectionEnabled, - final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { + public static SuggestedWords obtainNonBatchedInputSuggestedWords( + final WordComposer wordComposer, final int inputStyleIfNotPrediction, + final boolean isCorrectionEnabled, final int sequenceNumber, + final Locale locale, final SuggestionResults suggestionResults, + final float autoCorrectionThreshold + ) { final String typedWordString = wordComposer.getTypedWord(); final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(typedWordString); @@ -161,10 +160,6 @@ public final class Suggest { ? typedWordString.substring(0, typedWordString.length() - trailingSingleQuotesCount) : typedWordString; - final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( - wordComposer.getComposedDataSnapshot(), ngramContext, keyboard, - settingsValuesForSuggestion, SESSION_ID_TYPING, inputStyleIfNotPrediction); - final Locale locale = mDictionaryFacilitator.getLocale(); final ArrayList suggestionsContainer = getTransformedSuggestedWordInfoList(wordComposer, suggestionResults, trailingSingleQuotesCount, locale); @@ -223,7 +218,7 @@ public final class Suggest { // list, "will" would always auto-correct to "Will" which is unwanted. Hence, no // main dict => no auto-correct. Also, it would probably get obnoxious quickly. // TODO: now that we have personalization, we may want to re-evaluate this decision - || !mDictionaryFacilitator.hasAtLeastOneInitializedMainDictionary() + //|| !mDictionaryFacilitator.hasAtLeastOneInitializedMainDictionary() // If the first suggestion is a shortcut we never auto-correct to it, regardless // of how strong it is (allowlist entries are not KIND_SHORTCUT but KIND_WHITELIST). // TODO: we may want to have shortcut-only entries auto-correct in the future. @@ -235,7 +230,7 @@ public final class Suggest { && firstOcurrenceOfTypedWordInSuggestions != 0) { hasAutoCorrection = true; } else if (!AutoCorrectionUtils.suggestionExceedsThreshold( - firstSuggestion, consideredWord, mAutoCorrectionThreshold)) { + firstSuggestion, consideredWord, autoCorrectionThreshold)) { // Score is too low for autocorrect hasAutoCorrection = false; } else { @@ -277,11 +272,30 @@ public final class Suggest { final boolean isTypedWordValid = firstOcurrenceOfTypedWordInSuggestions > -1 || (!resultsArePredictions && !allowsToBeAutoCorrected); - callback.onGetSuggestedWords(new SuggestedWords(suggestionsList, + return new SuggestedWords(suggestionsList, suggestionResults.mRawSuggestions, typedWordInfo, isTypedWordValid, hasAutoCorrection /* willAutoCorrect */, - false /* isObsoleteSuggestions */, inputStyle, sequenceNumber)); + false /* isObsoleteSuggestions */, inputStyle, sequenceNumber); + } + + + // Retrieves suggestions for non-batch input (typing, recorrection, predictions...) + // and calls the callback function with the suggestions. + private void getSuggestedWordsForNonBatchInput(final WordComposer wordComposer, + final NgramContext ngramContext, final Keyboard keyboard, + final SettingsValuesForSuggestion settingsValuesForSuggestion, + final int inputStyleIfNotPrediction, final boolean isCorrectionEnabled, + final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { + final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults( + wordComposer.getComposedDataSnapshot(), ngramContext, keyboard, + settingsValuesForSuggestion, SESSION_ID_TYPING, inputStyleIfNotPrediction); + final Locale locale = mDictionaryFacilitator.getLocale(); + + callback.onGetSuggestedWords( + obtainNonBatchedInputSuggestedWords(wordComposer, inputStyleIfNotPrediction, + isCorrectionEnabled, sequenceNumber, locale, suggestionResults, mAutoCorrectionThreshold) + ); } // Retrieves suggestions for the batch input diff --git a/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java b/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java index a6ada04bb..e21b11cb6 100644 --- a/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java @@ -58,6 +58,7 @@ import org.futo.inputmethod.latin.utils.InputTypeUtils; import org.futo.inputmethod.latin.utils.RecapitalizeStatus; import org.futo.inputmethod.latin.utils.StatsUtils; import org.futo.inputmethod.latin.utils.TextRange; +import org.futo.inputmethod.latin.xlm.LanguageModelFacilitator; import java.util.ArrayList; import java.util.Locale; @@ -74,7 +75,7 @@ public final class InputLogic { // TODO : Remove this member when we can. final LatinIMELegacy mLatinIMELegacy; - private final SuggestionStripViewAccessor mSuggestionStripViewAccessor; + public final SuggestionStripViewAccessor mSuggestionStripViewAccessor; // Never null. private InputLogicHandler mInputLogicHandler = InputLogicHandler.NULL_HANDLER; @@ -88,8 +89,8 @@ public final class InputLogic { private final DictionaryFacilitator mDictionaryFacilitator; public LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; - // This has package visibility so it can be accessed from InputLogicHandler. - /* package */ final WordComposer mWordComposer; + + public final WordComposer mWordComposer; public final RichInputConnection mConnection; private final RecapitalizeStatus mRecapitalizeStatus = new RecapitalizeStatus(); @@ -1443,62 +1444,90 @@ public final class InputLogic { ngramContext, timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive); } + private void ensureSuggestionStripCompleted(final SettingsValues settingsValues, + final String separator, final LatinIMELegacy.UIHandler handler) { + if(settingsValues.mTransformerPredictionEnabled) { + LanguageModelFacilitator facilitator = handler.getLanguageModelFacilitator(); + if(facilitator.hasPendingUpdate()) { + facilitator.blockUntilComplete(); + } + } else { + if (handler.hasPendingUpdateSuggestions()) { + handler.cancelUpdateSuggestionStrip(); + // To know the input style here, we should retrieve the in-flight "update suggestions" + // message and read its arg1 member here. However, the Handler class does not let + // us retrieve this message, so we can't do that. But in fact, we notice that + // we only ever come here when the input style was typing. In the case of batch + // input, we update the suggestions synchronously when the tail batch comes. Likewise + // for application-specified completions. As for recorrections, we never auto-correct, + // so we don't come here either. Hence, the input style is necessarily + // INPUT_STYLE_TYPING. + performUpdateSuggestionStripSync(settingsValues, SuggestedWords.INPUT_STYLE_TYPING); + } + } + } + public void performUpdateSuggestionStripSync(final SettingsValues settingsValues, final int inputStyle) { - long startTimeMillis = 0; - if (DebugFlags.DEBUG_ENABLED) { - startTimeMillis = System.currentTimeMillis(); - Log.d(TAG, "performUpdateSuggestionStripSync()"); - } - // Check if we have a suggestion engine attached. - if (!settingsValues.needsToLookupSuggestions()) { - if (mWordComposer.isComposingWord()) { - Log.w(TAG, "Called updateSuggestionsOrPredictions but suggestions were not " - + "requested!"); + if(settingsValues.mTransformerPredictionEnabled) { + throw new IllegalStateException("called performUpdateSuggestionStripSync during TransformerLM"); + } else { + long startTimeMillis = 0; + if (DebugFlags.DEBUG_ENABLED) { + startTimeMillis = System.currentTimeMillis(); + Log.d(TAG, "performUpdateSuggestionStripSync()"); + } + // Check if we have a suggestion engine attached. + if (!settingsValues.needsToLookupSuggestions()) { + if (mWordComposer.isComposingWord()) { + Log.w(TAG, "Called updateSuggestionsOrPredictions but suggestions were not " + + "requested!"); + } + // Clear the suggestions strip. + mSuggestionStripViewAccessor.showSuggestionStrip(SuggestedWords.getEmptyInstance()); + return; + } + + if (!mWordComposer.isComposingWord() && !settingsValues.mBigramPredictionEnabled) { + mSuggestionStripViewAccessor.setNeutralSuggestionStrip(); + return; } - // Clear the suggestions strip. - mSuggestionStripViewAccessor.showSuggestionStrip(SuggestedWords.getEmptyInstance()); - return; - } - if (!mWordComposer.isComposingWord() && !settingsValues.mBigramPredictionEnabled) { mSuggestionStripViewAccessor.setNeutralSuggestionStrip(); - return; - } - - final AsyncResultHolder holder = new AsyncResultHolder<>("Suggest"); - mInputLogicHandler.getSuggestedWords(inputStyle, SuggestedWords.NOT_A_SEQUENCE_NUMBER, - new OnGetSuggestedWordsCallback() { - @Override - public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - final String typedWordString = mWordComposer.getTypedWord(); - final SuggestedWordInfo typedWordInfo = new SuggestedWordInfo( - typedWordString, "" /* prevWordsContext */, - SuggestedWordInfo.MAX_SCORE, - SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, - SuggestedWordInfo.NOT_A_CONFIDENCE); - // Show new suggestions if we have at least one. Otherwise keep the old - // suggestions with the new typed word. Exception: if the length of the - // typed word is <= 1 (after a deletion typically) we clear old suggestions. - if (suggestedWords.size() > 1 || typedWordString.length() <= 1) { - holder.set(suggestedWords); - } else { - holder.set(retrieveOlderSuggestions(typedWordInfo, mSuggestedWords)); + final AsyncResultHolder holder = new AsyncResultHolder<>("Suggest"); + mInputLogicHandler.getSuggestedWords(inputStyle, SuggestedWords.NOT_A_SEQUENCE_NUMBER, + new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + final String typedWordString = mWordComposer.getTypedWord(); + final SuggestedWordInfo typedWordInfo = new SuggestedWordInfo( + typedWordString, "" /* prevWordsContext */, + SuggestedWordInfo.MAX_SCORE, + SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE); + // Show new suggestions if we have at least one. Otherwise keep the old + // suggestions with the new typed word. Exception: if the length of the + // typed word is <= 1 (after a deletion typically) we clear old suggestions. + if (suggestedWords.size() > 1 || typedWordString.length() <= 1) { + holder.set(suggestedWords); + } else { + holder.set(retrieveOlderSuggestions(typedWordInfo, mSuggestedWords)); + } } } - } - ); + ); - // This line may cause the current thread to wait. - final SuggestedWords suggestedWords = holder.get(null, - Constants.GET_SUGGESTED_WORDS_TIMEOUT); - if (suggestedWords != null) { - mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords); - } - if (DebugFlags.DEBUG_ENABLED) { - long runTimeMillis = System.currentTimeMillis() - startTimeMillis; - Log.d(TAG, "performUpdateSuggestionStripSync() : " + runTimeMillis + " ms to finish"); + // This line may cause the current thread to wait. + final SuggestedWords suggestedWords = holder.get(null, + Constants.GET_SUGGESTED_WORDS_TIMEOUT); + if (suggestedWords != null) { + mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords); + } + if (DebugFlags.DEBUG_ENABLED) { + long runTimeMillis = System.currentTimeMillis() - startTimeMillis; + Log.d(TAG, "performUpdateSuggestionStripSync() : " + runTimeMillis + " ms to finish"); + } } } @@ -2094,18 +2123,7 @@ public final class InputLogic { private void commitCurrentAutoCorrection(final SettingsValues settingsValues, final String separator, final LatinIMELegacy.UIHandler handler) { // Complete any pending suggestions query first - if (handler.hasPendingUpdateSuggestions()) { - handler.cancelUpdateSuggestionStrip(); - // To know the input style here, we should retrieve the in-flight "update suggestions" - // message and read its arg1 member here. However, the Handler class does not let - // us retrieve this message, so we can't do that. But in fact, we notice that - // we only ever come here when the input style was typing. In the case of batch - // input, we update the suggestions synchronously when the tail batch comes. Likewise - // for application-specified completions. As for recorrections, we never auto-correct, - // so we don't come here either. Hence, the input style is necessarily - // INPUT_STYLE_TYPING. - performUpdateSuggestionStripSync(settingsValues, SuggestedWords.INPUT_STYLE_TYPING); - } + ensureSuggestionStripCompleted(settingsValues, separator, handler); final SuggestedWordInfo autoCorrectionOrNull = mWordComposer.getAutoCorrectionOrNull(); final String typedWord = mWordComposer.getTypedWord(); final String stringToCommit = (autoCorrectionOrNull != null) diff --git a/java/src/org/futo/inputmethod/latin/settings/SettingsValues.java b/java/src/org/futo/inputmethod/latin/settings/SettingsValues.java index 354651b99..dd28b39c8 100644 --- a/java/src/org/futo/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/org/futo/inputmethod/latin/settings/SettingsValues.java @@ -55,7 +55,6 @@ public class SettingsValues { // From resources: public final SpacingAndPunctuations mSpacingAndPunctuations; - public final int mDelayInMillisecondsToUpdateOldSuggestions; public final long mDoubleSpacePeriodTimeout; // From configuration: public final Locale mLocale; @@ -124,8 +123,6 @@ public class SettingsValues { @Nonnull final InputAttributes inputAttributes) { mLocale = res.getConfiguration().locale; // Get the resources - mDelayInMillisecondsToUpdateOldSuggestions = - res.getInteger(R.integer.config_delay_in_milliseconds_to_update_old_suggestions); mSpacingAndPunctuations = new SpacingAndPunctuations(res); // Store the input attributes @@ -381,8 +378,6 @@ public class SettingsValues { final StringBuilder sb = new StringBuilder("Current settings :"); sb.append("\n mSpacingAndPunctuations = "); sb.append("" + mSpacingAndPunctuations.dump()); - sb.append("\n mDelayInMillisecondsToUpdateOldSuggestions = "); - sb.append("" + mDelayInMillisecondsToUpdateOldSuggestions); sb.append("\n mAutoCap = "); sb.append("" + mAutoCap); sb.append("\n mVibrateOn = "); diff --git a/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java b/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java index ebf67eafb..8e144d0de 100644 --- a/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java +++ b/java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java @@ -95,16 +95,16 @@ public class LanguageModel extends Dictionary { @Override public void run() { if(mNativeState != 0) return; - String modelPath = getPathToModelResource(context, R.raw.ml4_f16, R.raw.ml3_tokenizer, true); + String modelPath = getPathToModelResource(context, R.raw.ml4_1_f16, R.raw.ml3_tokenizer, true); mNativeState = openNative(modelPath); if(mNativeState == 0){ - modelPath = getPathToModelResource(context, R.raw.ml4_f16, R.raw.ml3_tokenizer, true); + modelPath = getPathToModelResource(context, R.raw.ml4_1_f16, R.raw.ml3_tokenizer, true); mNativeState = openNative(modelPath); } if(mNativeState == 0){ - throw new RuntimeException("Failed to load R.raw.ml4_f16, R.raw.ml3_tokenizer model"); + throw new RuntimeException("Failed to load R.raw.ml4_1_f16, R.raw.ml3_tokenizer model"); } } }; diff --git a/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt new file mode 100644 index 000000000..72c3896fc --- /dev/null +++ b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt @@ -0,0 +1,246 @@ +package org.futo.inputmethod.latin.xlm; + +import android.content.ComponentCallbacks2 +import android.content.Context +import android.content.res.Configuration +import android.inputmethodservice.InputMethodService +import android.os.Build +import android.os.Bundle +import android.view.KeyEvent +import android.view.View +import android.view.ViewGroup +import android.view.inputmethod.CompletionInfo +import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InlineSuggestion +import android.view.inputmethod.InlineSuggestionsRequest +import android.view.inputmethod.InlineSuggestionsResponse +import android.view.inputmethod.InputMethodSubtype +import androidx.annotation.RequiresApi +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.size +import androidx.compose.material3.ColorScheme +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.key +import androidx.compose.ui.Alignment.Companion.CenterVertically +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clipToBounds +import androidx.compose.ui.layout.onSizeChanged +import androidx.compose.ui.platform.ComposeView +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.platform.ViewCompositionStrategy +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleCoroutineScope +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry +import androidx.lifecycle.ViewModelStore +import androidx.lifecycle.ViewModelStoreOwner +import androidx.lifecycle.findViewTreeLifecycleOwner +import androidx.lifecycle.findViewTreeViewModelStoreOwner +import androidx.lifecycle.lifecycleScope +import androidx.lifecycle.setViewTreeLifecycleOwner +import androidx.lifecycle.setViewTreeViewModelStoreOwner +import androidx.savedstate.SavedStateRegistry +import androidx.savedstate.SavedStateRegistryController +import androidx.savedstate.SavedStateRegistryOwner +import androidx.savedstate.findViewTreeSavedStateRegistryOwner +import androidx.savedstate.setViewTreeSavedStateRegistryOwner +import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.conflate +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.Semaphore +import org.futo.inputmethod.latin.common.Constants +import org.futo.inputmethod.latin.common.ComposedData +import org.futo.inputmethod.latin.uix.Action +import org.futo.inputmethod.latin.uix.ActionBar +import org.futo.inputmethod.latin.uix.ActionInputTransaction +import org.futo.inputmethod.latin.uix.ActionWindow +import org.futo.inputmethod.latin.uix.BasicThemeProvider +import org.futo.inputmethod.latin.uix.DynamicThemeProvider +import org.futo.inputmethod.latin.uix.DynamicThemeProviderOwner +import org.futo.inputmethod.latin.uix.KeyboardManagerForAction +import org.futo.inputmethod.latin.uix.PersistentActionState +import org.futo.inputmethod.latin.uix.THEME_KEY +import org.futo.inputmethod.latin.uix.actions.VoiceInputAction +import org.futo.inputmethod.latin.uix.createInlineSuggestionsRequest +import org.futo.inputmethod.latin.uix.deferGetSetting +import org.futo.inputmethod.latin.uix.deferSetSetting +import org.futo.inputmethod.latin.uix.differsFrom +import org.futo.inputmethod.latin.uix.inflateInlineSuggestion +import org.futo.inputmethod.latin.uix.theme.DarkColorScheme +import org.futo.inputmethod.latin.uix.theme.ThemeOption +import org.futo.inputmethod.latin.uix.theme.ThemeOptions +import org.futo.inputmethod.latin.uix.theme.Typography +import org.futo.inputmethod.latin.uix.theme.UixThemeWrapper +import org.futo.inputmethod.latin.uix.theme.presets.ClassicMaterialDark +import org.futo.inputmethod.latin.uix.theme.presets.DynamicSystemTheme +import org.futo.inputmethod.latin.uix.theme.presets.VoiceInputTheme +import org.futo.inputmethod.latin.settings.SettingsValues; +import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion +import org.futo.inputmethod.latin.settings.Settings +import org.futo.inputmethod.latin.xlm.LanguageModel; +import org.futo.inputmethod.latin.utils.SuggestionResults +import org.futo.inputmethod.latin.NgramContext +import org.futo.inputmethod.latin.LatinIMELegacy +import org.futo.inputmethod.latin.inputlogic.InputLogic +import org.futo.inputmethod.latin.DictionaryFacilitator +import org.futo.inputmethod.latin.Suggest +import org.futo.inputmethod.latin.SuggestedWords +import org.futo.inputmethod.keyboard.KeyboardSwitcher + +public class LanguageModelFacilitator( + val context: Context, + val inputLogic: InputLogic, + val dictionaryFacilitator: DictionaryFacilitator, + val settings: Settings, + val keyboardSwitcher: KeyboardSwitcher, + val lifecycleScope: LifecycleCoroutineScope +) { + private var languageModel: LanguageModel? = null + data class PredictionInputValues( + val composedData: ComposedData, + val ngramContext: NgramContext, + val inputStyle: Int, + val sequenceId: Int + ) + private val sharedFlow = MutableSharedFlow(replay = 0, extraBufferCapacity = 1) + + private var currentSequenceId = 0 + private val sequenceIdFinishedFlow = MutableSharedFlow(replay = 4, extraBufferCapacity = 4) + + private val computationSemaphore = Semaphore(1) + public fun hasPendingUpdate(): Boolean = + computationSemaphore.availablePermits == 0 + + public fun blockUntilComplete() { + runBlocking { + computationSemaphore.acquire() + computationSemaphore.release() + try { + sequenceIdFinishedFlow.first { it >= currentSequenceId } + } catch(ignored: Exception) { + + } + } + } + + private suspend fun processUpdateSuggestionStrip(values: PredictionInputValues) { + computationSemaphore.acquire() + try { + val job = Job() + CoroutineScope(Dispatchers.Default + job).launch { + delay(200) + inputLogic.mSuggestionStripViewAccessor.setNeutralSuggestionStrip() + } + + val locale = dictionaryFacilitator.locale + if(languageModel == null) { + languageModel = LanguageModel(context, "lm", locale) + } + + val settingsValues = settings.current + + val keyboard = keyboardSwitcher.getKeyboard() + val settingsForPrediction = SettingsValuesForSuggestion( + settingsValues.mBlockPotentiallyOffensive, + settingsValues.mTransformerPredictionEnabled + ) + val proximityInfoHandle = keyboard.getProximityInfo().getNativeProximityInfo() + + val suggestionResults = SuggestionResults( + 3, values.ngramContext.isBeginningOfSentenceContext(), false) + + val lmSuggestions = languageModel!!.getSuggestions( + values.composedData, + values.ngramContext, + proximityInfoHandle, + settingsForPrediction, + -1, + 0.0f, + floatArrayOf()) + + if(lmSuggestions == null) { + job.cancel() + inputLogic.mSuggestionStripViewAccessor.setNeutralSuggestionStrip() + return + } + + suggestionResults.addAll(lmSuggestions) + if(suggestionResults.mRawSuggestions != null) { + suggestionResults.mRawSuggestions.addAll(lmSuggestions) + } + + val wordComposer = inputLogic.mWordComposer + val suggestedWords = Suggest.obtainNonBatchedInputSuggestedWords( + wordComposer, values.inputStyle, true, -1, locale, suggestionResults, settingsValues.mAutoCorrectionThreshold) + + job.cancel() + inputLogic.mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords) + sequenceIdFinishedFlow.emit(values.sequenceId) + } finally { + computationSemaphore.release() + } + } + + public fun launchProcessor() = lifecycleScope.launch { + println("LatinIME: Starting processor") + withContext(Dispatchers.Default) { + sharedFlow.conflate().collect { value -> + println("LatinIME: Collecting") + processUpdateSuggestionStrip(value) + } + } + } + + public fun updateSuggestionStripAsync(inputStyle: Int) { + val settingsValues = settings.current + if (!settingsValues.needsToLookupSuggestions()) { + inputLogic.mSuggestionStripViewAccessor.showSuggestionStrip(SuggestedWords.getEmptyInstance()) + return + } + + if(!settingsValues.mTransformerPredictionEnabled) { + // TODO: Call old path + return + } + + val wordComposer = inputLogic.mWordComposer + val ngramContext = inputLogic.getNgramContextFromNthPreviousWordForSuggestion(settingsValues.mSpacingAndPunctuations, 2) + + val values = PredictionInputValues( + wordComposer.getComposedDataSnapshot(), + ngramContext, + inputStyle, + ++currentSequenceId + ) + + lifecycleScope.launch { + println("LatinIME: Emitting values") + sharedFlow.emit(values) + } + } +} \ No newline at end of file