n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add getDictDecoder.

Browse Source 
Bug: 9618601
Change-Id: I173100ac704c03f7d5d0d53477e83cab5d1110d4
This commit is contained in:
Yuichiro Hanada 2013-09-12 17:46:09 +09:00
parent be470f06e4
commit 0e40cd0c40
13 changed files with 80 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java
View File

@ -21,9 +21,10 @@ import android.content.SharedPreferences;
import android.content.res.AssetFileDescriptor;
import android.util.Log;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.DictionaryInfoUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
@ -228,7 +229,7 @@ final public class BinaryDictionaryGetter {
private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) {
try {
// Read the version of the file
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f);
final FileHeader header = dictDecoder.readHeader();
final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY);

6
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -148,7 +148,7 @@ public final class BinaryDictIOUtils {
* @throws IOException if the file can't be read.
* @throws UnsupportedFormatException if the format of the file is not recognized.
*/
/* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder,
/* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
@ -169,7 +169,7 @@ public final class BinaryDictIOUtils {
* @throws UnsupportedFormatException if the format of the file is not recognized.
*/
@UsedForTesting
/* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder,
/* package */ static int getTerminalPosition(final DictDecoder dictDecoder,
final String word) throws IOException, UnsupportedFormatException {
if (word == null) return FormatSpec.NOT_VALID_WORD;
dictDecoder.setPosition(0);
@ -519,7 +519,7 @@ public final class BinaryDictIOUtils {
final File file, final long offset, final long length)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE];
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
new DictDecoder.DictionaryBufferFactory() {
@Override
public DictBuffer getDictionaryBuffer(File file)

8
java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
View File

@ -118,6 +118,14 @@ public interface DictDecoder {
public boolean readForwardLinkAndAdvancePosition();
public boolean hasNextPtNodeArray();
/**
* Opens the dictionary file and makes DictBuffer.
*/
@UsedForTesting
public void openDictBuffer() throws FileNotFoundException, IOException;
@UsedForTesting
public boolean isOpenedDictBuffer();
// Flags for DictionaryBufferFactory.
public static final int USE_READONLY_BYTEBUFFER = 0x01000000;
public static final int USE_BYTEARRAY = 0x02000000;

25
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -18,8 +18,11 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import java.io.File;
/**
* Dictionary File Format Specification.
*/
@ -341,6 +344,28 @@ public final class FormatSpec {
}
}
/**
* Returns new dictionary decoder.
*
* @param dictFile the dictionary file.
* @param bufferType the flag indicating buffer type which is used by the dictionary decoder.
* @return new dictionary decoder if the dictionary file exists, otherwise null.
*/
public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) {
if (!dictFile.isFile()) return null;
return new Ver3DictDecoder(dictFile, bufferType);
}
public static DictDecoder getDictDecoder(final File dictFile,
final DictionaryBufferFactory factory) {
if (!dictFile.isFile()) return null;
return new Ver3DictDecoder(dictFile, factory);
}
public static DictDecoder getDictDecoder(final File dictFile) {
return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER);
}
private FormatSpec() {
// This utility class is not publicly instantiable.
}

14
java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java
View File

@ -173,11 +173,7 @@ public class Ver3DictDecoder implements DictDecoder {
private final DictionaryBufferFactory mBufferFactory;
private DictBuffer mDictBuffer;
public Ver3DictDecoder(final File file) {
this(file, USE_READONLY_BYTEBUFFER);
}
public Ver3DictDecoder(final File file, final int factoryFlag) {
/* package */ Ver3DictDecoder(final File file, final int factoryFlag) {
mDictionaryBinaryFile = file;
mDictBuffer = null;
@ -192,15 +188,21 @@ public class Ver3DictDecoder implements DictDecoder {
}
}
public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
/* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) {
mDictionaryBinaryFile = file;
mBufferFactory = factory;
}
@Override
public void openDictBuffer() throws FileNotFoundException, IOException {
mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile);
}
@Override
public boolean isOpenedDictBuffer() {
return mDictBuffer != null;
}
/* package */ DictBuffer getDictBuffer() {
return mDictBuffer;
}

16
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -25,14 +25,13 @@ import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.ExpandableBinaryDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@ -173,16 +172,19 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
// Load the dictionary from binary file
final File dictFile = new File(mContext.getFilesDir(), mFileName);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
DictDecoder.USE_BYTEARRAY);
if (dictDecoder == null) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
return;
}
try {
dictDecoder.openDictBuffer();
UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
} catch (IOException e) {
Log.e(TAG, "IOException on opening a bytebuffer", e);
Log.d(TAG, "IOException on opening a bytebuffer", e);
} finally {
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;

4
java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
View File

@ -20,13 +20,13 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.PendingAttribute;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import java.io.IOException;
@ -125,7 +125,7 @@ public final class UserHistoryDictIOUtils {
/**
* Reads dictionary from file.
*/
public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder,
public static void readDictionaryBinary(final DictDecoder dictDecoder,
final OnAddWordListener dict) {
final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap();
final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap();

33
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -131,18 +131,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// Utilities for test
/**
* Makes new DictDecoder according to BUFFER_TYPE.
*/
private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) {
if (bufferType == USE_BYTE_BUFFER) {
return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER);
} else if (bufferType == USE_BYTE_ARRAY) {
return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
}
return null;
}
/**
* Generates a random word.
*/
@ -285,9 +273,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FusionDictionary dict = null;
try {
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull(dictDecoder.getDictBuffer());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
@ -443,9 +429,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
long now = -1, diff = -1;
try {
final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType);
dictDecoder.openDictBuffer();
assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer());
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
@ -531,9 +515,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
// Tests for getTerminalPosition
private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) {
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
if (dictBuffer.position() != 0) dictBuffer.position(0);
private String getWordFromBinary(final DictDecoder dictDecoder, final int address) {
if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0);
FileHeader fileHeader = null;
try {
@ -548,7 +531,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
address, fileHeader.mFormatOptions).mWord;
}
private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word,
private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word,
int index, boolean contained) {
final int expectedFrequency = (UNIGRAM_FREQ + index) % 255;
long diff = -1;
@ -584,14 +567,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
try {
// too long word
@ -648,7 +631,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer());
assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer());
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,

16
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java
View File

@ -140,7 +140,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
int position = FormatSpec.NOT_VALID_WORD;
try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file,
DictDecoder.USE_READONLY_BYTEBUFFER);
position = dictDecoder.getTerminalPosition(word);
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
@ -149,7 +150,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
}
/**
* Find a word using the Ver3DictDecoder.
* Find a word using the DictDecoder.
*
* @param dictDecoder the dict decoder
* @param word the word searched
@ -157,21 +158,20 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
* @throws IOException
* @throws UnsupportedFormatException
*/
private static PtNodeInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder,
private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder,
final String word) throws IOException, UnsupportedFormatException {
int position = dictDecoder.getTerminalPosition(word);
final DictBuffer dictBuffer = dictDecoder.getDictBuffer();
if (position != FormatSpec.NOT_VALID_WORD) {
dictBuffer.position(0);
dictDecoder.setPosition(0);
final FileHeader header = dictDecoder.readHeader();
dictBuffer.position(position);
dictDecoder.setPosition(position);
return dictDecoder.readPtNode(position, header.mFormatOptions);
}
return null;
}
private PtNodeInfo findWordFromFile(final File file, final String word) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
PtNodeInfo info = null;
try {
dictDecoder.openDictBuffer();
@ -234,7 +234,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase {
private void checkReverseLookup(final File file, final String word, final int position) {
try {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
final FileHeader fileHeader = dictDecoder.readHeader();
assertEquals(word,
BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize,

2
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -143,7 +143,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (FileNotFoundException e) {

4
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -18,9 +18,9 @@ package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import org.xml.sax.SAXException;
@ -185,7 +185,7 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile,
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file,
DictDecoder.USE_BYTEARRAY);
if (report) {
System.out.println("Format : Binary dictionary format");

2
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -267,7 +267,7 @@ public class DictionaryMaker {
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
final File file = new File(binaryFilename);
final DictDecoder dictDecoder = new Ver3DictDecoder(file);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file);
return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */);
}

4
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -18,12 +18,12 @@ package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictEncoder;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.makedict.Ver3DictDecoder;
import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
import junit.framework.TestCase;
@ -69,7 +69,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile);
final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile);
final FusionDictionary resultDict = dictDecoder.readDictionaryBinary(
null /* dict : an optional dictionary to add words to, or null */,
false /* deleteDictIfBroken */);