From e209eefd48fbd98d71864071c0ccf33b3ca4332b Mon Sep 17 00:00:00 2001 From: Aleksandras Kostarevas Date: Wed, 13 Mar 2024 16:47:17 -0500 Subject: [PATCH] Add emoji suggestions --- .../inputmethod/latin/SuggestedWords.java | 1 + .../futo/inputmethod/latin/uix/ActionBar.kt | 33 +++++++++++----- .../latin/uix/actions/EmojiAction.kt | 17 ++++++++- .../latin/uix/actions/emoji/EmojiItem.kt | 4 +- .../latin/xlm/LanguageModelFacilitator.kt | 38 ++++++++++++++++++- 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/java/src/org/futo/inputmethod/latin/SuggestedWords.java b/java/src/org/futo/inputmethod/latin/SuggestedWords.java index 4a2eff1d0..94a8af6f8 100644 --- a/java/src/org/futo/inputmethod/latin/SuggestedWords.java +++ b/java/src/org/futo/inputmethod/latin/SuggestedWords.java @@ -255,6 +255,7 @@ public class SuggestedWords { // in java for re-correction) public static final int KIND_RESUMED = 9; public static final int KIND_OOV_CORRECTION = 10; // Most probable string correction + public static final int KIND_EMOJI_SUGGESTION = 11; public static final int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000; public static final int KIND_FLAG_EXACT_MATCH = 0x40000000; diff --git a/java/src/org/futo/inputmethod/latin/uix/ActionBar.kt b/java/src/org/futo/inputmethod/latin/uix/ActionBar.kt index c60c2e439..a400a9b20 100644 --- a/java/src/org/futo/inputmethod/latin/uix/ActionBar.kt +++ b/java/src/org/futo/inputmethod/latin/uix/ActionBar.kt @@ -295,19 +295,32 @@ fun RowScope.SuggestionItems(words: SuggestedWords, onClick: (i: Int) -> Unit, o } - for (i in 0 until maxSuggestions) { - val remapped = if(offset == 1 && i == 2) { - 0 - offset - } else { - ORDER_OF_SUGGESTIONS[i] - } + val suggestionOrder = mutableListOf( + ORDER_OF_SUGGESTIONS[0] + offset, + ORDER_OF_SUGGESTIONS[1] + offset, + if(offset == 1) { 0 - offset } else { ORDER_OF_SUGGESTIONS[2] } + offset, + ) + // Find emoji + try { + for(i in 0 until words.size()) { + val info = words.getInfo(i) + if(info.mKindAndFlags == SuggestedWordInfo.KIND_EMOJI_SUGGESTION) { + suggestionOrder[0] = i + } + } + } catch(_: IndexOutOfBoundsException) { + + } + + + for (i in 0 until maxSuggestions) { SuggestionItem( words, - remapped + offset, - isPrimary = remapped == 0, - onClick = { onClick(remapped + offset) }, - onLongClick = { onLongClick(remapped + offset) } + suggestionOrder[i], + isPrimary = i == (maxSuggestions / 2), + onClick = { onClick(suggestionOrder[i]) }, + onLongClick = { onLongClick(suggestionOrder[i]) } ) if (i < maxSuggestions - 1) SuggestionSeparator() diff --git a/java/src/org/futo/inputmethod/latin/uix/actions/EmojiAction.kt b/java/src/org/futo/inputmethod/latin/uix/actions/EmojiAction.kt index aef5eb2d1..04d3a2621 100644 --- a/java/src/org/futo/inputmethod/latin/uix/actions/EmojiAction.kt +++ b/java/src/org/futo/inputmethod/latin/uix/actions/EmojiAction.kt @@ -239,7 +239,9 @@ fun Emojis( emoji = emoji, description = popupInfo.emoji.description, category = popupInfo.emoji.category, - skinTones = false + skinTones = false, + aliases = listOf(), + tags = listOf() ) ) activePopup = null @@ -342,6 +344,7 @@ fun EmojiGrid( class PersistentEmojiState : PersistentActionState { var emojis: MutableState?> = mutableStateOf(null) var emojiMap: HashMap = HashMap() + var emojiAliases: HashMap = HashMap() suspend fun loadEmojis(context: Context) = withContext(Dispatchers.IO) { val stream = context.resources.openRawResource(R.raw.gemoji) @@ -354,7 +357,9 @@ class PersistentEmojiState : PersistentActionState { emoji = it.jsonObject["emoji"]!!.jsonPrimitive.content, description = it.jsonObject["description"]!!.jsonPrimitive.content, category = it.jsonObject["category"]!!.jsonPrimitive.content, - skinTones = it.jsonObject["skin_tones"]?.jsonPrimitive?.booleanOrNull ?: false + skinTones = it.jsonObject["skin_tones"]?.jsonPrimitive?.booleanOrNull ?: false, + tags = it.jsonObject["tags"]?.jsonArray?.map { it.jsonPrimitive.content }?.toList() ?: listOf(), + aliases = it.jsonObject["aliases"]?.jsonArray?.map { it.jsonPrimitive.content }?.toList() ?: listOf(), ) } @@ -363,6 +368,14 @@ class PersistentEmojiState : PersistentActionState { put(it.emoji, it) } } + + emojiAliases = HashMap().apply { + emojis.value!!.forEach { emoji -> + emoji.tags.forEach { put(it, emoji) } + emoji.aliases.forEach { put(it, emoji) } + } + } + } } diff --git a/java/src/org/futo/inputmethod/latin/uix/actions/emoji/EmojiItem.kt b/java/src/org/futo/inputmethod/latin/uix/actions/emoji/EmojiItem.kt index 4d197dced..a359421cd 100644 --- a/java/src/org/futo/inputmethod/latin/uix/actions/emoji/EmojiItem.kt +++ b/java/src/org/futo/inputmethod/latin/uix/actions/emoji/EmojiItem.kt @@ -4,5 +4,7 @@ data class EmojiItem( val emoji: String, val description: String, val category: String, - val skinTones: Boolean + val skinTones: Boolean, + val tags: List, + val aliases: List ) \ No newline at end of file diff --git a/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt index b025d1dab..2ca222196 100644 --- a/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt +++ b/java/src/org/futo/inputmethod/latin/xlm/LanguageModelFacilitator.kt @@ -1,6 +1,7 @@ package org.futo.inputmethod.latin.xlm; import android.content.Context +import android.util.Log import androidx.datastore.preferences.core.floatPreferencesKey import androidx.lifecycle.LifecycleCoroutineScope import kotlinx.coroutines.CoroutineScope @@ -30,6 +31,7 @@ import org.futo.inputmethod.latin.settings.Settings import org.futo.inputmethod.latin.settings.SettingsValuesForSuggestion import org.futo.inputmethod.latin.uix.SettingsKey import org.futo.inputmethod.latin.uix.USE_TRANSFORMER_FINETUNING +import org.futo.inputmethod.latin.uix.actions.PersistentEmojiState import org.futo.inputmethod.latin.uix.getSetting import org.futo.inputmethod.latin.uix.getSettingFlow import org.futo.inputmethod.latin.utils.AsyncResultHolder @@ -77,6 +79,7 @@ public class LanguageModelFacilitator( val suggestionBlacklist: SuggestionBlacklist ) { private val userDictionary = UserDictionaryObserver(context) + private val emojiData = PersistentEmojiState() private var languageModel: LanguageModel? = null data class PredictionInputValues( @@ -112,6 +115,25 @@ public class LanguageModelFacilitator( } } + private fun getEmojiCandidate(word: String): SuggestedWordInfo? { + val emoji = emojiData.emojiAliases[word.lowercase()] + + if(emoji != null) { + Log.i("LanguageModelFacilitator", "Found emoji ${emoji.emoji} for $word") + return SuggestedWordInfo( + emoji.emoji, + "", + 100, + SuggestedWordInfo.KIND_EMOJI_SUGGESTION, + null, + SuggestedWordInfo.NOT_AN_INDEX, + SuggestedWordInfo.NOT_A_CONFIDENCE + ) + } else { + return null + } + } + private suspend fun processUpdateSuggestionStrip(values: PredictionInputValues) { computationSemaphore.acquire() @@ -164,7 +186,7 @@ public class LanguageModelFacilitator( val proximityInfoHandle = keyboard.proximityInfo.nativeProximityInfo val suggestionResults = SuggestionResults( - 3, values.ngramContext.isBeginningOfSentenceContext, false) + 14, values.ngramContext.isBeginningOfSentenceContext, false) val lmSuggestions = languageModel!!.getSuggestions( values.composedData, @@ -241,7 +263,15 @@ public class LanguageModelFacilitator( it != words.typedWordInfo && !filtered.contains( it ) - }) + }.take(10)) + } + } + + if(values.composedData.mTypedWord.isNotEmpty()) { + (getEmojiCandidate(values.composedData.mTypedWord) + ?: maxWord?.let { getEmojiCandidate(it.mWord) } + ?: maxWordDict?.let { getEmojiCandidate(it.mWord) })?.let { + suggestionResults.add(it) } } @@ -314,6 +344,10 @@ public class LanguageModelFacilitator( } } + launch { + emojiData.loadEmojis(context) + } + scheduleTrainingWorkerBackground(context) }