mirror of
https://gitlab.futo.org/keyboard/latinime.git
synced 2024-09-28 14:54:30 +01:00
AI 143472: Reduce dictionary size.
Changed the tree structure to have variable length nodes to save an average of 21% on the dictionary size. Created a shortened English dictionary for Dream - 50K words. Added a shortened Spanish dictionary for Dream - 32K words. BUG=1743626 Automated import of CL 143472
This commit is contained in:
parent
772b83b036
commit
8503b063d4
@ -85,10 +85,14 @@ int
|
||||
Dictionary::getAddress(int *pos)
|
||||
{
|
||||
int address = 0;
|
||||
address += (mDict[*pos] & 0x7F) << 16;
|
||||
address += (mDict[*pos + 1] & 0xFF) << 8;
|
||||
address += (mDict[*pos + 2] & 0xFF);
|
||||
*pos += 3;
|
||||
if ((mDict[*pos] & FLAG_ADDRESS_MASK) == 0) {
|
||||
*pos += 1;
|
||||
} else {
|
||||
address += (mDict[*pos] & (ADDRESS_MASK >> 16)) << 16;
|
||||
address += (mDict[*pos + 1] & 0xFF) << 8;
|
||||
address += (mDict[*pos + 2] & 0xFF);
|
||||
*pos += 3;
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
@ -193,7 +197,8 @@ Dictionary::getWordsRec(int pos, int depth, int maxDepth, bool completion, int s
|
||||
unsigned short lowerC = toLowerCase(c, depth);
|
||||
bool terminal = getTerminal(&pos);
|
||||
int childrenAddress = getAddress(&pos);
|
||||
int freq = getFreq(&pos);
|
||||
int freq = 1;
|
||||
if (terminal) freq = getFreq(&pos);
|
||||
// If we are only doing completions, no need to look at the typed characters.
|
||||
if (completion) {
|
||||
mWord[depth] = c;
|
||||
@ -266,7 +271,9 @@ Dictionary::isValidWordRec(int pos, unsigned short *word, int offset, int length
|
||||
}
|
||||
}
|
||||
}
|
||||
getFreq(&pos);
|
||||
if (terminal) {
|
||||
getFreq(&pos);
|
||||
}
|
||||
// There could be two instances of each alphabet - upper and lower case. So continue
|
||||
// looking ...
|
||||
}
|
||||
|
@ -19,35 +19,44 @@
|
||||
|
||||
namespace latinime {
|
||||
|
||||
// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words
|
||||
#define ADDRESS_MASK 0x3FFFFF
|
||||
|
||||
// The bit that decides if an address follows in the next 22 bits
|
||||
#define FLAG_ADDRESS_MASK 0x40
|
||||
// The bit that decides if this is a terminal node for a word. The node could still have children,
|
||||
// if the word has other endings.
|
||||
#define FLAG_TERMINAL_MASK 0x80
|
||||
|
||||
class Dictionary {
|
||||
public:
|
||||
Dictionary(void *dict, int typedLetterMultipler, int fullWordMultiplier);
|
||||
int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
|
||||
int getSuggestions(int *codes, int codesSize, unsigned short *outWords, int *frequencies,
|
||||
int maxWordLength, int maxWords, int maxAlternatives);
|
||||
bool isValidWord(unsigned short *word, int length);
|
||||
void setAsset(void *asset) { mAsset = asset; }
|
||||
void *getAsset() { return mAsset; }
|
||||
~Dictionary();
|
||||
|
||||
|
||||
private:
|
||||
|
||||
int getAddress(int *pos);
|
||||
bool getTerminal(int *pos) { return (mDict[*pos] & 0x80) > 0; }
|
||||
bool getTerminal(int *pos) { return (mDict[*pos] & FLAG_TERMINAL_MASK) > 0; }
|
||||
int getFreq(int *pos) { return mDict[(*pos)++] & 0xFF; }
|
||||
int getCount(int *pos) { return mDict[(*pos)++] & 0xFF; }
|
||||
unsigned short getChar(int *pos);
|
||||
int wideStrLen(unsigned short *str);
|
||||
|
||||
|
||||
bool sameAsTyped(unsigned short *word, int length);
|
||||
bool addWord(unsigned short *word, int length, int frequency);
|
||||
unsigned short toLowerCase(unsigned short c, int depth);
|
||||
void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency,
|
||||
void getWordsRec(int pos, int depth, int maxDepth, bool completion, int frequency,
|
||||
int inputIndex);
|
||||
bool isValidWordRec(int pos, unsigned short *word, int offset, int length);
|
||||
|
||||
unsigned char *mDict;
|
||||
void *mAsset;
|
||||
|
||||
|
||||
int *mFrequencies;
|
||||
int mMaxWords;
|
||||
int mMaxWordLength;
|
||||
@ -57,7 +66,7 @@ private:
|
||||
int mInputLength;
|
||||
int mMaxAlternatives;
|
||||
unsigned short mWord[128];
|
||||
|
||||
|
||||
int mFullWordMultiplier;
|
||||
int mTypedLetterMultiplier;
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user