n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge "Add BinaryDictReader."

Browse Source
This commit is contained in:
Ken Wakasa 2013-08-13 11:03:44 +00:00 committed by Android (Google) Code Review
commit f27297d2ec
11 changed files with 327 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -145,21 +145,21 @@ public final class BinaryDictIOUtils {
* Reads unigrams and bigrams from the binary file.
* Doesn't make the memory representation of the dictionary.
*
* @param buffer the buffer to read.
* @param reader the reader.
* @param words the map to store the address as a key and the word as a value.
* @param frequencies the map to store the address as a key and the frequency as a value.
* @param bigrams the map to store the address as a key and the list of address as a value.
* @throws IOException
* @throws UnsupportedFormatException
*/
public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader,
final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
UnsupportedFormatException {
// Read header
final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
header.mFormatOptions);
final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
frequencies, bigrams, header.mFormatOptions);
}
/**

20
java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java
View File

@ -66,6 +66,7 @@ public final class BinaryDictInputOutput {
public void position(int newPosition);
public void put(final byte b);
public int limit();
@UsedForTesting
public int capacity();
}
@ -1722,23 +1723,30 @@ public final class BinaryDictInputOutput {
* FusionDictionary structure. The optional dict argument is an existing dictionary to
* which words from the buffer should be added. If it is null, a new dictionary is created.
*
* @param buffer the buffer to read.
* @param reader the reader.
* @param dict an optional dictionary to add words to, or null.
* @return the created (or merged) dictionary.
*/
@UsedForTesting
public static FusionDictionary readDictionaryBinary(
final FusionDictionaryBufferInterface buffer, final FusionDictionary dict)
throws IOException, UnsupportedFormatException {
public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
final FusionDictionary dict) throws FileNotFoundException, IOException,
UnsupportedFormatException {
// clear cache
wordCache.clear();
// if the buffer has not been opened, open the buffer with bytebuffer.
if (reader.getBuffer() == null) reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
if (reader.getBuffer() == null) {
MakedictLog.e("Cannot open the buffer");
}
// Read header
final FileHeader header = readHeader(buffer);
final FileHeader header = readHeader(reader.getBuffer());
Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping,
final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
reverseGroupMapping, header.mFormatOptions);
FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);

109
java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
public class BinaryDictReader {
public interface FusionDictionaryBufferFactory {
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException;
}
/**
* Creates FusionDictionaryBuffer from a ByteBuffer
*/
public static final class FusionDictionaryBufferFromByteBufferFactory
implements FusionDictionaryBufferFactory {
@Override
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException {
FileInputStream inStream = null;
ByteBuffer buffer = null;
try {
inStream = new FileInputStream(file);
buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY,
0, file.length());
} finally {
if (inStream != null) {
inStream.close();
}
}
if (buffer != null) {
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
}
return null;
}
}
/**
* Creates FusionDictionaryBuffer from a byte array
*/
public static final class FusionDictionaryBufferFromByteArrayFactory
implements FusionDictionaryBufferFactory {
@Override
public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
throws FileNotFoundException, IOException {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
final byte[] array = new byte[(int) file.length()];
inStream.read(array);
return new ByteArrayWrapper(array);
} finally {
if (inStream != null) {
inStream.close();
}
}
}
}
private final File mDictionaryBinaryFile;
private FusionDictionaryBufferInterface mFusionDictionaryBuffer;
public BinaryDictReader(final File file) {
mDictionaryBinaryFile = file;
mFusionDictionaryBuffer = null;
}
public void openBuffer(final FusionDictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
mFusionDictionaryBuffer = factory.getFusionDictionaryBuffer(mDictionaryBinaryFile);
}
public FusionDictionaryBufferInterface getBuffer() {
return mFusionDictionaryBuffer;
}
@UsedForTesting
public FusionDictionaryBufferInterface openAndGetBuffer(
final FusionDictionaryBufferFactory factory)
throws FileNotFoundException, IOException {
openBuffer(factory);
return getBuffer();
}
}

21
java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
View File

@ -28,9 +28,9 @@ import com.android.inputmethod.latin.ExpandableDictionary;
import com.android.inputmethod.latin.LatinImeLogger;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.settings.Settings;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.CollectionUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
@ -39,7 +39,6 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@ -234,27 +233,17 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
};
// Load the dictionary from binary file
FileInputStream inStream = null;
final BinaryDictReader reader = new BinaryDictReader(
new File(getContext().getFilesDir(), fileName));
try {
final File file = new File(getContext().getFilesDir(), fileName);
final byte[] buffer = new byte[(int)file.length()];
inStream = new FileInputStream(file);
inStream.read(buffer);
UserHistoryDictIOUtils.readDictionaryBinary(
new ByteArrayWrapper(buffer), listener);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
} catch (FileNotFoundException e) {
// This is an expected condition: we don't have a user history dictionary for this
// language yet. It will be created sometime later.
} catch (IOException e) {
Log.e(TAG, "IOException on opening a bytebuffer", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
if (PROFILE_SAVE_RESTORE) {
final long diff = System.currentTimeMillis() - now;
Log.d(TAG, "PROF: Load UserHistoryDictionary: "

6
java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java
View File

@ -21,7 +21,7 @@ import android.util.Log;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
@ -118,13 +118,13 @@ public final class UserHistoryDictIOUtils {
/**
* Reads dictionary from file.
*/
public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer,
public static void readDictionaryBinary(final BinaryDictReader reader,
final OnAddWordListener dict) {
final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
try {
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, unigrams, frequencies,
bigrams);
} catch (IOException e) {
Log.e(TAG, "IO exception while reading file", e);

70
tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java
View File

@ -27,15 +27,13 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.CollectionUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -120,31 +118,13 @@ public class BinaryDictIOTests extends AndroidTestCase {
/**
* Makes new buffer according to BUFFER_TYPE.
*/
private FusionDictionaryBufferInterface getBuffer(final File file, final int bufferType) {
FileInputStream inStream = null;
try {
inStream = new FileInputStream(file);
if (bufferType == USE_BYTE_ARRAY) {
final byte[] array = new byte[(int)file.length()];
inStream.read(array);
return new ByteArrayWrapper(array);
} else if (bufferType == USE_BYTE_BUFFER){
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
}
} catch (IOException e) {
Log.e(TAG, "IOException while making buffer", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
Log.e(TAG, "IOException while closing stream", e);
}
}
private void getBuffer(final BinaryDictReader reader, final int bufferType)
throws FileNotFoundException, IOException {
if (bufferType == USE_BYTE_BUFFER) {
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
} else if (bufferType == USE_BYTE_ARRAY) {
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
}
return null;
}
/**
@ -285,13 +265,14 @@ public class BinaryDictIOTests extends AndroidTestCase {
final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap,
final int bufferType) {
long now, diff = -1;
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
assertNotNull(buffer);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionary dict = null;
try {
getBuffer(reader, bufferType);
assertNotNull(reader.getBuffer());
now = System.currentTimeMillis();
dict = BinaryDictInputOutput.readDictionaryBinary(buffer, null);
dict = BinaryDictInputOutput.readDictionaryBinary(reader, null);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
@ -421,11 +402,12 @@ public class BinaryDictIOTests extends AndroidTestCase {
final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap();
long now = -1, diff = -1;
final FusionDictionaryBufferInterface buffer = getBuffer(file, bufferType);
assertNotNull("Can't get buffer.", buffer);
final BinaryDictReader reader = new BinaryDictReader(file);
try {
getBuffer(reader, bufferType);
assertNotNull("Can't get buffer.", reader.getBuffer());
now = System.currentTimeMillis();
BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, resultWords, resultFreqs,
BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, resultWords, resultFreqs,
resultBigrams);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
@ -562,7 +544,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", buffer);
try {
// too long word
@ -614,7 +605,16 @@ public class BinaryDictIOTests extends AndroidTestCase {
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE);
final FusionDictionaryBufferInterface buffer = getBuffer(file, USE_BYTE_ARRAY);
final BinaryDictReader reader = new BinaryDictReader(file);
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (IOException e) {
// ignore
Log.e(TAG, "IOException while opening the buffer", e);
}
assertNotNull("Can't get the buffer", buffer);
try {
MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD,

138
tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory;
import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory;
import android.test.AndroidTestCase;
import android.util.Log;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
/**
* Unit tests for BinaryDictReader
*/
public class BinaryDictReaderTests extends AndroidTestCase {
private static final String TAG = BinaryDictReaderTests.class.getSimpleName();
private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
// Utilities for testing
public void writeDataToFile(final File file) {
FileOutputStream outStream = null;
try {
outStream = new FileOutputStream(file);
outStream.write(data);
} catch (IOException e) {
fail ("Can't write data to the test file");
} finally {
if (outStream != null) {
try {
outStream.close();
} catch (IOException e) {
Log.e(TAG, "Failed to close the output stream", e);
}
}
}
}
@SuppressWarnings("null")
public void runTestOpenBuffer(final String testName,
final FusionDictionaryBufferFactory factory) {
File testFile = null;
try {
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException while the creating temporary file", e);
}
assertNotNull(testFile);
final BinaryDictReader reader = new BinaryDictReader(testFile);
try {
reader.openBuffer(factory);
} catch (Exception e) {
Log.e(TAG, "Failed to open the buffer", e);
}
writeDataToFile(testFile);
try {
reader.openBuffer(factory);
} catch (Exception e) {
Log.e(TAG, "Raised the exception while opening buffer", e);
}
assertEquals(testFile.length(), reader.getBuffer().capacity());
}
public void testOpenBufferWithByteBuffer() {
runTestOpenBuffer("testOpenBufferWithByteBuffer",
new FusionDictionaryBufferFromByteBufferFactory());
}
public void testOpenBufferWithByteArray() {
runTestOpenBuffer("testOpenBufferWithByteArray",
new FusionDictionaryBufferFromByteArrayFactory());
}
@SuppressWarnings("null")
public void runTestGetBuffer(final String testName,
final FusionDictionaryBufferFactory factory) {
File testFile = null;
try {
testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir());
} catch (IOException e) {
Log.e(TAG, "IOException while the creating temporary file", e);
}
final BinaryDictReader reader = new BinaryDictReader(testFile);
// the default return value of getBuffer() must be null.
assertNull("the default return value of getBuffer() is not null", reader.getBuffer());
writeDataToFile(testFile);
assertTrue(testFile.exists());
Log.d(TAG, "file length = " + testFile.length());
FusionDictionaryBufferInterface buffer = null;
try {
buffer = reader.openAndGetBuffer(factory);
} catch (IOException e) {
Log.e(TAG, "Failed to open and get the buffer", e);
}
assertNotNull("the buffer must not be null", buffer);
for (int i = 0; i < data.length; ++i) {
assertEquals(data[i], buffer.readUnsignedByte());
}
}
public void testGetBufferWithByteBuffer() {
runTestGetBuffer("testGetBufferWithByteBuffer",
new FusionDictionaryBufferFromByteBufferFactory());
}
public void testGetBufferWithByteArray() {
runTestGetBuffer("testGetBufferWithByteArray",
new FusionDictionaryBufferFromByteArrayFactory());
}
}

21
tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java
View File

@ -21,16 +21,15 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Log;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup;
import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList;
import com.android.inputmethod.latin.utils.ByteArrayWrapper;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@ -147,27 +146,15 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase
}
private void readDictFromFile(final File file, final OnAddWordListener listener) {
FileInputStream inStream = null;
final BinaryDictReader reader = new BinaryDictReader(file);
try {
inStream = new FileInputStream(file);
final byte[] buffer = new byte[(int)file.length()];
inStream.read(buffer);
UserHistoryDictIOUtils.readDictionaryBinary(new ByteArrayWrapper(buffer), listener);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
} catch (FileNotFoundException e) {
Log.e(TAG, "file not found", e);
} catch (IOException e) {
Log.e(TAG, "IOException", e);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
}
public void testGenerateFusionDictionary() {

10
tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
@ -185,16 +186,15 @@ public final class BinaryDictOffdeviceUtils {
crash(filename, new RuntimeException(
filename + " does not seem to be a dictionary file"));
} else {
final FileInputStream inStream = new FileInputStream(decodedSpec.mFile);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length());
final BinaryDictReader reader = new BinaryDictReader(decodedSpec.mFile);
reader.openBuffer(
new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
if (report) {
System.out.println("Format : Binary dictionary format");
System.out.println("Packaging : " + decodedSpec.describeChain());
System.out.println("Uncompressed size : " + decodedSpec.mFile.length());
}
return BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
}
}
} catch (IOException e) {

23
tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.MakedictLog;
@ -265,24 +266,10 @@ public class DictionaryMaker {
*/
private static FusionDictionary readBinaryFile(final String binaryFilename)
throws FileNotFoundException, IOException, UnsupportedFormatException {
FileInputStream inStream = null;
try {
final File file = new File(binaryFilename);
inStream = new FileInputStream(file);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, file.length());
return BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer), null);
} finally {
if (inStream != null) {
try {
inStream.close();
} catch (IOException e) {
// do nothing
}
}
}
final File file = new File(binaryFilename);
final BinaryDictReader reader = new BinaryDictReader(file);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
return BinaryDictInputOutput.readDictionaryBinary(reader, null);
}
/**

9
tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java
View File

@ -17,6 +17,7 @@
package com.android.inputmethod.latin.dicttool;
import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
import com.android.inputmethod.latin.makedict.BinaryDictReader;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary;
import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions;
@ -69,11 +70,9 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase {
assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step);
}
assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size());
final FileInputStream inStream = new FileInputStream(decodeSpec.mFile);
final ByteBuffer buffer = inStream.getChannel().map(
FileChannel.MapMode.READ_ONLY, 0, decodeSpec.mFile.length());
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(
new BinaryDictInputOutput.ByteBufferWrapper(buffer),
final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile);
reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader,
null /* dict : an optional dictionary to add words to, or null */);
assertEquals("Dictionary can't be read back correctly",
resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ);