n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix non-English dictionary prediction

Browse Source
This commit is contained in:
Aleksandras Kostarevas 2024-01-16 21:02:55 +02:00
parent ff903bd4a4
commit dbad61d2e6
7 changed files with 83 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
java/src/org/futo/inputmethod/latin/DictionaryFacilitatorImpl.java
View File

@ -618,13 +618,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator {
NgramContext ngramContext, @Nonnull final Keyboard keyboard,
SettingsValuesForSuggestion settingsValuesForSuggestion, int sessionId,
int inputStyle) {
if(settingsValuesForSuggestion.mUseTransformerLM) {
throw new IllegalStateException("Invalid code path TransformerLM");
}
long proximityInfoHandle = keyboard.getProximityInfo().getNativeProximityInfo();
final SuggestionResults suggestionResults = new SuggestionResults(
SuggestedWords.MAX_SUGGESTIONS, ngramContext.isBeginningOfSentenceContext(),

5
java/src/org/futo/inputmethod/latin/LatinIME.kt
View File

@ -494,7 +494,10 @@ class LatinIME : InputMethodService(), LifecycleOwner, ViewModelStoreOwner, Save
return uixManager.onInlineSuggestionsResponse(response)
}
fun postUpdateSuggestionStrip(inputStyle: Int) {
fun postUpdateSuggestionStrip(inputStyle: Int): Boolean {
if(languageModelFacilitator.shouldPassThroughToLegacy()) return false
languageModelFacilitator.updateSuggestionStripAsync(inputStyle);
return true
}
}

20
java/src/org/futo/inputmethod/latin/LatinIMELegacy.java
View File

@ -336,14 +336,20 @@ public class LatinIMELegacy implements KeyboardActionListener,
public void postUpdateSuggestionStrip(final int inputStyle) {
final LatinIMELegacy latinImeLegacy = getOwnerInstance();
if(latinImeLegacy.mSettings.getCurrent().mTransformerPredictionEnabled) {
((LatinIME)latinImeLegacy.getInputMethodService()).postUpdateSuggestionStrip(inputStyle);
} else {
sendMessageDelayed(obtainMessage(MSG_UPDATE_SUGGESTION_STRIP_LEGACY, inputStyle,
0 /* ignored */), mDelayInMillisecondsToUpdateSuggestions);
assert latinImeLegacy != null;
final LatinIME latinIme = (LatinIME)latinImeLegacy.getInputMethodService();
if(!latinIme.postUpdateSuggestionStrip(inputStyle)) {
updateSuggestionStripLegacy(inputStyle);
}
}
public void updateSuggestionStripLegacy(final int inputStyle) {
sendMessageDelayed(obtainMessage(MSG_UPDATE_SUGGESTION_STRIP_LEGACY, inputStyle,
0 /* ignored */), mDelayInMillisecondsToUpdateSuggestions);
}
public void postReopenDictionaries() {
sendMessage(obtainMessage(MSG_REOPEN_DICTIONARIES));
}
@ -1621,10 +1627,10 @@ public class LatinIMELegacy implements KeyboardActionListener,
public void getSuggestedWords(final int inputStyle, final int sequenceNumber,
final OnGetSuggestedWordsCallback callback) {
SettingsValues settings = mSettings.getCurrent();
if(settings.mTransformerPredictionEnabled) {
((LatinIME)getInputMethodService()).postUpdateSuggestionStrip(inputStyle);
if(((LatinIME)getInputMethodService()).postUpdateSuggestionStrip(inputStyle)) {
return;
}
final Keyboard keyboard = mKeyboardSwitcher.getKeyboard();
if (keyboard == null) {
callback.onGetSuggestedWords(SuggestedWords.getEmptyInstance());

106
java/src/org/futo/inputmethod/latin/inputlogic/InputLogic.java
View File

@ -1469,8 +1469,8 @@ public final class InputLogic {
private void ensureSuggestionStripCompleted(final SettingsValues settingsValues,
final String separator, final LatinIMELegacy.UIHandler handler) {
if(settingsValues.mTransformerPredictionEnabled) {
LanguageModelFacilitator facilitator = handler.getLanguageModelFacilitator();
LanguageModelFacilitator facilitator = handler.getLanguageModelFacilitator();
if(!facilitator.shouldPassThroughToLegacy()) {
if(facilitator.hasPendingUpdate()) {
facilitator.blockUntilComplete();
}
@ -1492,65 +1492,61 @@ public final class InputLogic {
public void performUpdateSuggestionStripSync(final SettingsValues settingsValues,
final int inputStyle) {
if(settingsValues.mTransformerPredictionEnabled) {
throw new IllegalStateException("called performUpdateSuggestionStripSync during TransformerLM");
} else {
long startTimeMillis = 0;
if (DebugFlags.DEBUG_ENABLED) {
startTimeMillis = System.currentTimeMillis();
Log.d(TAG, "performUpdateSuggestionStripSync()");
}
// Check if we have a suggestion engine attached.
if (!settingsValues.needsToLookupSuggestions()) {
if (mWordComposer.isComposingWord()) {
Log.w(TAG, "Called updateSuggestionsOrPredictions but suggestions were not "
+ "requested!");
}
// Clear the suggestions strip.
mSuggestionStripViewAccessor.showSuggestionStrip(SuggestedWords.getEmptyInstance());
return;
}
if (!mWordComposer.isComposingWord() && !settingsValues.mBigramPredictionEnabled) {
mSuggestionStripViewAccessor.setNeutralSuggestionStrip();
return;
long startTimeMillis = 0;
if (DebugFlags.DEBUG_ENABLED) {
startTimeMillis = System.currentTimeMillis();
Log.d(TAG, "performUpdateSuggestionStripSync()");
}
// Check if we have a suggestion engine attached.
if (!settingsValues.needsToLookupSuggestions()) {
if (mWordComposer.isComposingWord()) {
Log.w(TAG, "Called updateSuggestionsOrPredictions but suggestions were not "
+ "requested!");
}
// Clear the suggestions strip.
mSuggestionStripViewAccessor.showSuggestionStrip(SuggestedWords.getEmptyInstance());
return;
}
if (!mWordComposer.isComposingWord() && !settingsValues.mBigramPredictionEnabled) {
mSuggestionStripViewAccessor.setNeutralSuggestionStrip();
final AsyncResultHolder<SuggestedWords> holder = new AsyncResultHolder<>("Suggest");
mInputLogicHandler.getSuggestedWords(inputStyle, SuggestedWords.NOT_A_SEQUENCE_NUMBER,
new OnGetSuggestedWordsCallback() {
@Override
public void onGetSuggestedWords(final SuggestedWords suggestedWords) {
final String typedWordString = mWordComposer.getTypedWord();
final SuggestedWordInfo typedWordInfo = new SuggestedWordInfo(
typedWordString, "" /* prevWordsContext */,
SuggestedWordInfo.MAX_SCORE,
SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED,
SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */,
SuggestedWordInfo.NOT_A_CONFIDENCE);
// Show new suggestions if we have at least one. Otherwise keep the old
// suggestions with the new typed word. Exception: if the length of the
// typed word is <= 1 (after a deletion typically) we clear old suggestions.
if (suggestedWords.size() > 1 || typedWordString.length() <= 1) {
holder.set(suggestedWords);
} else {
holder.set(retrieveOlderSuggestions(typedWordInfo, mSuggestedWords));
}
return;
}
mSuggestionStripViewAccessor.setNeutralSuggestionStrip();
final AsyncResultHolder<SuggestedWords> holder = new AsyncResultHolder<>("Suggest");
mInputLogicHandler.getSuggestedWords(inputStyle, SuggestedWords.NOT_A_SEQUENCE_NUMBER,
new OnGetSuggestedWordsCallback() {
@Override
public void onGetSuggestedWords(final SuggestedWords suggestedWords) {
final String typedWordString = mWordComposer.getTypedWord();
final SuggestedWordInfo typedWordInfo = new SuggestedWordInfo(
typedWordString, "" /* prevWordsContext */,
SuggestedWordInfo.MAX_SCORE,
SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED,
SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */,
SuggestedWordInfo.NOT_A_CONFIDENCE);
// Show new suggestions if we have at least one. Otherwise keep the old
// suggestions with the new typed word. Exception: if the length of the
// typed word is <= 1 (after a deletion typically) we clear old suggestions.
if (suggestedWords.size() > 1 || typedWordString.length() <= 1) {
holder.set(suggestedWords);
} else {
holder.set(retrieveOlderSuggestions(typedWordInfo, mSuggestedWords));
}
}
);
}
);
// This line may cause the current thread to wait.
final SuggestedWords suggestedWords = holder.get(null,
Constants.GET_SUGGESTED_WORDS_TIMEOUT);
if (suggestedWords != null) {
mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords);
}
if (DebugFlags.DEBUG_ENABLED) {
long runTimeMillis = System.currentTimeMillis() - startTimeMillis;
Log.d(TAG, "performUpdateSuggestionStripSync() : " + runTimeMillis + " ms to finish");
}
// This line may cause the current thread to wait.
final SuggestedWords suggestedWords = holder.get(null,
Constants.GET_SUGGESTED_WORDS_TIMEOUT);
if (suggestedWords != null) {
mSuggestionStripViewAccessor.showSuggestionStrip(suggestedWords);
}
if (DebugFlags.DEBUG_ENABLED) {
long runTimeMillis = System.currentTimeMillis() - startTimeMillis;
Log.d(TAG, "performUpdateSuggestionStripSync() : " + runTimeMillis + " ms to finish");
}
}

10
java/src/org/futo/inputmethod/latin/xlm/LanguageModel.java
View File

@ -26,6 +26,10 @@ public class LanguageModel {
this.locale = locale;
}
public Locale getLocale() {
return Locale.ENGLISH;
}
private void loadModel() {
if (initThread != null && initThread.isAlive()){
Log.d("LanguageModel", "Cannot load model again, as initThread is still active");
@ -67,12 +71,6 @@ public class LanguageModel {
) {
Log.d("LanguageModel", "getSuggestions called");
// Language Model currently only supports English
if(locale.getLanguage() != Locale.ENGLISH.getLanguage()) {
Log.d("LanguageModel", "Exiting because locale is not English");
return null;
}
if (mNativeState == 0) {
loadModel();
Log.d("LanguageModel", "Exiting because mNativeState == 0");

15
java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt
View File

@ -166,6 +166,12 @@ public class LanguageModelFacilitator(
scheduleTrainingWorkerBackground(context)
}
public fun shouldPassThroughToLegacy(): Boolean =
(!settings.current.mTransformerPredictionEnabled) ||
(languageModel?.let {
it.getLocale().language != dictionaryFacilitator.locale.language
} ?: false)
public fun updateSuggestionStripAsync(inputStyle: Int) {
val settingsValues = settings.current
if (!settingsValues.needsToLookupSuggestions()) {
@ -173,11 +179,6 @@ public class LanguageModelFacilitator(
return
}
if(!settingsValues.mTransformerPredictionEnabled) {
// TODO: Call old path
return
}
if(!inputLogic.mConnection.isConnected) return
try {
@ -214,6 +215,8 @@ public class LanguageModelFacilitator(
blockPotentiallyOffensive: Boolean,
importance: Int
) {
if(shouldPassThroughToLegacy()) return
val wordCtx = ngramContext.fullContext.trim().lines().last()
var committedNgramCtx = ngramContext.extractPrevWordsContext().replace(NgramContext.BEGINNING_OF_SENTENCE_TAG, " ").trim();
if(committedNgramCtx.isEmpty()) {
@ -271,6 +274,8 @@ public class LanguageModelFacilitator(
timeStampInSeconds: Long,
eventType: Int
) {
if(shouldPassThroughToLegacy()) return
val wordCtx = ngramContext.fullContext.trim().lines().last()
var committedNgramCtx = ngramContext.extractPrevWordsContext().replace(NgramContext.BEGINNING_OF_SENTENCE_TAG, " ").trim();
if(committedNgramCtx.isEmpty()) {

2
native/jni/src/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp
View File

@ -192,7 +192,7 @@ template<class DictConstants, class DictBuffers, class DictBuffersPtr, class Str
AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path);
break;
}
ASSERT(false);
//ASSERT(false);
return nullptr;
}