n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix issue with apostrophe token being banned

Browse Source
This commit is contained in:
Aleksandras Kostarevas 2024-01-22 08:20:55 +02:00
parent c53c279155
commit 5e0722c984
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
native/jni/org_futo_inputmethod_latin_xlm_LanguageModel.cpp
View File

@ -127,6 +127,10 @@ struct LanguageModelState {
// TODO: Since ". " is still blocked, we get "U.S" instead of the expected "U.S. "
if(i == model->tokenToId(".")) continue;
// Specifically allow ' for words like Wasn't, which may be tokenized as
// [Wasn] ['] [t ]
if(i == model->tokenToId("'")) continue;
specialTokens.SAMPLING_BAD_TOKENS.emplace_back(i);
}
for(int i = model->tokenToId(":"); i <= model->tokenToId("~"); i++) {