n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove unused code.

Browse Source 
Bug: 12810574
Change-Id: I9c7fff60ae0e94d52f3bd19c3e88de5a53b917d7
This commit is contained in:
Keisuke Kuroyanagi 2014-02-15 17:38:11 +09:00
parent 0fc93fe445
commit 95d16561e0
12 changed files with 99 additions and 593 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java
View File

@ -17,14 +17,10 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.TreeMap;
/**
@ -35,34 +31,6 @@ public abstract class AbstractDictDecoder implements DictDecoder {
private static final int ERROR_CANNOT_READ = 1;
private static final int ERROR_WRONG_FORMAT = 2;
protected DictionaryHeader readHeader(final DictBuffer headerBuffer)
throws IOException, UnsupportedFormatException {
if (headerBuffer == null) {
openDictBuffer();
}
final int version = HeaderReader.readVersion(headerBuffer);
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
throw new UnsupportedFormatException("Unsupported version : " + version);
}
// TODO: Remove this field.
HeaderReader.readOptionFlags(headerBuffer);
final int headerSize = HeaderReader.readHeaderSize(headerBuffer);
if (headerSize < 0) {
throw new UnsupportedFormatException("header size can't be negative.");
}
final HashMap<String, String> attributes = HeaderReader.readAttributes(headerBuffer,
headerSize);
final DictionaryHeader header = new DictionaryHeader(headerSize,
new FusionDictionary.DictionaryOptions(attributes),
new FormatOptions(version, DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals(
attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY))));
return header;
}
@Override @UsedForTesting
public int getTerminalPosition(final String word)
throws IOException, UnsupportedFormatException {
@ -83,38 +51,6 @@ public abstract class AbstractDictDecoder implements DictDecoder {
BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams);
}
/**
* A utility class for reading a file header.
*/
protected static class HeaderReader {
protected static int readVersion(final DictBuffer dictBuffer)
throws IOException, UnsupportedFormatException {
return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer);
}
protected static int readOptionFlags(final DictBuffer dictBuffer) {
return dictBuffer.readUnsignedShort();
}
protected static int readHeaderSize(final DictBuffer dictBuffer) {
return dictBuffer.readInt();
}
protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer,
final int headerSize) {
final HashMap<String, String> attributes = new HashMap<String, String>();
while (dictBuffer.position() < headerSize) {
// We can avoid an infinite loop here since dictBuffer.position() is always
// increased by calling CharEncoding.readString.
final String key = CharEncoding.readString(dictBuffer);
final String value = CharEncoding.readString(dictBuffer);
attributes.put(key, value);
}
dictBuffer.position(headerSize);
return attributes;
}
}
/**
* Check whether the header contains the expected information. This is a no-error method,
* that will return an error code and never throw a checked exception.
@ -148,7 +84,7 @@ public abstract class AbstractDictDecoder implements DictDecoder {
}
@Override
public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) {
public PtNodeInfo readPtNode(final int ptNodePos) {
return null;
}
@ -165,14 +101,4 @@ public abstract class AbstractDictDecoder implements DictDecoder {
public int readPtNodeCount() {
return 0;
}
@Override
public boolean readAndFollowForwardLink() {
return false;
}
@Override
public boolean hasNextPtNodeArray() {
return false;
}
}

186
java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java
View File

@ -17,18 +17,12 @@
package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Map;
import java.util.TreeMap;
/**
* Decodes binary files for a FusionDictionary.
@ -47,8 +41,6 @@ public final class BinaryDictDecoderUtils {
// This utility class is not publicly instantiable.
}
private static final int MAX_JUMPS = 12;
@UsedForTesting
public interface DictBuffer {
public int readUnsignedByte();
@ -296,60 +288,21 @@ public final class BinaryDictDecoderUtils {
* @param dictDecoder the dict decoder.
* @param headerSize the size of the header.
* @param pos the position to seek.
* @param formatOptions file format options.
* @return the word with its frequency, as a weighted string.
*/
@UsedForTesting
/* package for tests */ static WeightedString getWordAtPosition(final DictDecoder dictDecoder,
final int headerSize, final int pos, final FormatOptions formatOptions) {
final int headerSize, final int pos) {
final WeightedString result;
final int originalPos = dictDecoder.getPosition();
dictDecoder.setPosition(pos);
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions);
} else {
result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos,
formatOptions);
}
result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos);
dictDecoder.setPosition(originalPos);
return result;
}
@SuppressWarnings("unused")
private static WeightedString getWordAtPositionWithParentAddress(final DictDecoder dictDecoder,
final int pos, final FormatOptions options) {
int currentPos = pos;
ProbabilityInfo probabilityInfo = null;
final StringBuilder builder = new StringBuilder();
// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
PtNodeInfo currentInfo;
int loopCounter = 0;
do {
dictDecoder.setPosition(currentPos);
currentInfo = dictDecoder.readPtNode(currentPos, options);
if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) {
currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
if (DBG && loopCounter++ > MAX_JUMPS) {
MakedictLog.d("Too many jumps - probably a bug");
}
} while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options));
if (probabilityInfo == null) {
probabilityInfo = currentInfo.mProbabilityInfo;
}
builder.insert(0,
new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length));
if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
}
return new WeightedString(builder.toString(), probabilityInfo);
}
private static WeightedString getWordAtPositionWithoutParentAddress(
final DictDecoder dictDecoder, final int headerSize, final int pos,
final FormatOptions options) {
final DictDecoder dictDecoder, final int headerSize, final int pos) {
dictDecoder.setPosition(headerSize);
final int count = dictDecoder.readPtNodeCount();
int groupPos = dictDecoder.getPosition();
@ -358,7 +311,7 @@ public final class BinaryDictDecoderUtils {
PtNodeInfo last = null;
for (int i = count - 1; i >= 0; --i) {
PtNodeInfo info = dictDecoder.readPtNode(groupPos, options);
PtNodeInfo info = dictDecoder.readPtNode(groupPos);
groupPos = info.mEndAddress;
if (info.mOriginalAddress == pos) {
builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
@ -389,135 +342,6 @@ public final class BinaryDictDecoderUtils {
return result;
}
/**
* Reads a single node array from a buffer.
*
* This methods reads the file at the current position. A node array is fully expected to start
* at the current position.
* This will recursively read other node arrays into the structure, populating the reverse
* maps on the fly and using them to keep track of already read nodes.
*
* @param dictDecoder the dict decoder, correctly positioned at the start of a node array.
* @param headerSize the size, in bytes, of the file header.
* @param reverseNodeArrayMap a mapping from addresses to already read node arrays.
* @param reversePtNodeMap a mapping from addresses to already read PtNodes.
* @param options file format options.
* @return the read node array with all his children already read.
*/
private static PtNodeArray readNodeArray(final DictDecoder dictDecoder,
final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap,
final Map<Integer, PtNode> reversePtNodeMap, final FormatOptions options)
throws IOException {
final ArrayList<PtNode> nodeArrayContents = new ArrayList<PtNode>();
final int nodeArrayOriginPos = dictDecoder.getPosition();
do { // Scan the linked-list node.
final int count = dictDecoder.readPtNodeCount();
int groupPos = dictDecoder.getPosition();
for (int i = count; i > 0; --i) { // Scan the array of PtNode.
PtNodeInfo info = dictDecoder.readPtNode(groupPos, options);
if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue;
ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets;
ArrayList<WeightedString> bigrams = null;
if (null != info.mBigrams) {
bigrams = new ArrayList<WeightedString>();
for (PendingAttribute bigram : info.mBigrams) {
final WeightedString word = getWordAtPosition(dictDecoder, headerSize,
bigram.mAddress, options);
final int reconstructedFrequency =
BinaryDictIOUtils.reconstructBigramFrequency(word.getProbability(),
bigram.mFrequency);
bigrams.add(new WeightedString(word.mWord, reconstructedFrequency));
}
}
if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress);
if (null == children) {
final int currentPosition = dictDecoder.getPosition();
dictDecoder.setPosition(info.mChildrenAddress);
children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap,
reversePtNodeMap, options);
dictDecoder.setPosition(currentPosition);
}
nodeArrayContents.add(
new PtNode(info.mCharacters, shortcutTargets, bigrams,
info.mProbabilityInfo,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
} else {
nodeArrayContents.add(
new PtNode(info.mCharacters, shortcutTargets, bigrams,
info.mProbabilityInfo,
0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
}
groupPos = info.mEndAddress;
}
// reach the end of the array.
if (options.supportsDynamicUpdate()) {
final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink();
if (!hasValidForwardLink) break;
}
} while (options.supportsDynamicUpdate() && dictDecoder.hasNextPtNodeArray());
final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents);
nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos;
nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos;
reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray);
return nodeArray;
}
/**
* Helper function to get the binary format version from the header.
* @throws IOException
*/
private static int getFormatVersion(final DictBuffer dictBuffer)
throws IOException {
final int magic = dictBuffer.readInt();
if (FormatSpec.MAGIC_NUMBER == magic) return dictBuffer.readUnsignedShort();
return FormatSpec.NOT_A_VERSION_NUMBER;
}
/**
* Helper function to get and validate the binary format version.
* @throws UnsupportedFormatException
* @throws IOException
*/
static int checkFormatVersion(final DictBuffer dictBuffer)
throws IOException, UnsupportedFormatException {
final int version = getFormatVersion(dictBuffer);
if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION
|| version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) {
throw new UnsupportedFormatException("This file has version " + version
+ ", but this implementation does not support versions above "
+ FormatSpec.MAXIMUM_SUPPORTED_VERSION);
}
return version;
}
/**
* Reads a buffer and returns the memory representation of the dictionary.
*
* This high-level method takes a buffer and reads its contents, populating a
* FusionDictionary structure.
*
* @param dictDecoder the dict decoder.
* @return the created dictionary.
*/
@UsedForTesting
/* package */ static FusionDictionary readDictionaryBinary(final DictDecoder dictDecoder)
throws IOException, UnsupportedFormatException {
// Read header
final DictionaryHeader fileHeader = dictDecoder.readHeader();
Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>();
Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>();
final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mBodyOffset,
reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions);
return new FusionDictionary(root, fileHeader.mDictionaryOptions);
}
/**
* Helper method to pass a file name instead of a File object to isBinaryDictionary.
*/

193
java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
View File

@ -122,18 +122,13 @@ public class BinaryDictEncoderUtils {
* Compute the maximum size of a PtNode, assuming 3-byte addresses for everything.
*
* @param ptNode the PtNode to compute the size of.
* @param options file format options.
* @return the maximum size of the PtNode.
*/
private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) {
int size = getNodeHeaderSize(ptNode, options);
private static int getPtNodeMaximumSize(final PtNode ptNode) {
int size = getNodeHeaderSize(ptNode);
if (ptNode.isTerminal()) {
// If terminal, one byte for the frequency or four bytes for the terminal id.
if (options.mHasTerminalId) {
size += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
} else {
size += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
// If terminal, one byte for the frequency.
size += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address
size += getShortcutListSize(ptNode.mShortcutTargets);
@ -151,19 +146,14 @@ public class BinaryDictEncoderUtils {
* the containing node array, and cache it it its 'mCachedSize' member.
*
* @param ptNodeArray the node array to compute the maximum size of.
* @param options file format options.
*/
private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray,
final FormatOptions options) {
private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray) {
int size = getPtNodeCountSize(ptNodeArray);
for (PtNode node : ptNodeArray.mData) {
final int nodeSize = getPtNodeMaximumSize(node, options);
final int nodeSize = getPtNodeMaximumSize(node);
node.mCachedSize = nodeSize;
size += nodeSize;
}
if (options.supportsDynamicUpdate()) {
size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
}
ptNodeArray.mCachedSize = size;
}
@ -171,15 +161,9 @@ public class BinaryDictEncoderUtils {
* Compute the size of the header (flag + [parent address] + characters size) of a PtNode.
*
* @param ptNode the PtNode of which to compute the size of the header
* @param options file format options.
*/
private static int getNodeHeaderSize(final PtNode ptNode, final FormatOptions options) {
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE
+ getPtNodeCharactersSize(ptNode);
} else {
return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode);
}
private static int getNodeHeaderSize(final PtNode ptNode) {
return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode);
}
/**
@ -379,11 +363,10 @@ public class BinaryDictEncoderUtils {
*
* @param ptNodeArray the node array to compute the size of.
* @param dict the dictionary in which the word/attributes are to be found.
* @param formatOptions file format options.
* @return false if none of the cached addresses inside the node array changed, true otherwise.
*/
private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray,
final FusionDictionary dict, final FormatOptions formatOptions) {
final FusionDictionary dict) {
boolean changed = false;
int size = getPtNodeCountSize(ptNodeArray);
for (PtNode ptNode : ptNodeArray.mData) {
@ -391,37 +374,26 @@ public class BinaryDictEncoderUtils {
if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) {
changed = true;
}
int nodeSize = getNodeHeaderSize(ptNode, formatOptions);
int nodeSize = getNodeHeaderSize(ptNode);
if (ptNode.isTerminal()) {
if (formatOptions.mHasTerminalId) {
nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
} else {
nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
if (formatOptions.supportsDynamicUpdate()) {
nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
} else if (null != ptNode.mChildren) {
if (null != ptNode.mChildren) {
nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray,
nodeSize + size, ptNode.mChildren));
}
if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) {
nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
if (null != ptNode.mBigrams) {
for (WeightedString bigram : ptNode.mBigrams) {
final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
}
nodeSize += getShortcutListSize(ptNode.mShortcutTargets);
if (null != ptNode.mBigrams) {
for (WeightedString bigram : ptNode.mBigrams) {
final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE,
FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord));
nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
}
}
ptNode.mCachedSize = nodeSize;
size += nodeSize;
}
if (formatOptions.supportsDynamicUpdate()) {
size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE;
}
if (ptNodeArray.mCachedSize != size) {
ptNodeArray.mCachedSize = size;
changed = true;
@ -433,11 +405,10 @@ public class BinaryDictEncoderUtils {
* Initializes the cached addresses of node arrays and their containing nodes from their size.
*
* @param flatNodes the list of node arrays.
* @param formatOptions file format options.
* @return the byte size of the entire stack.
*/
private static int initializePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes,
final FormatOptions formatOptions) {
private static int initializePtNodeArraysCachedAddresses(
final ArrayList<PtNodeArray> flatNodes) {
int nodeArrayOffset = 0;
for (final PtNodeArray nodeArray : flatNodes) {
nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset;
@ -467,28 +438,6 @@ public class BinaryDictEncoderUtils {
}
}
/**
* Compute the cached parent addresses after all has been updated.
*
* The parent addresses are used by some binary formats at write-to-disk time. Not all formats
* need them. In particular, version 2 does not need them, and version 3 does.
*
* @param flatNodes the flat array of node arrays to fill in
*/
private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) {
for (final PtNodeArray nodeArray : flatNodes) {
for (final PtNode ptNode : nodeArray.mData) {
if (null != ptNode.mChildren) {
// Assign my address to children's parent address
// Here BeforeUpdate and AfterUpdate addresses have the same value, so it
// does not matter which we use.
ptNode.mChildren.mCachedParentAddress = ptNode.mCachedAddressAfterUpdate
- ptNode.mChildren.mCachedAddressAfterUpdate;
}
}
}
}
/**
* Compute the addresses and sizes of an ordered list of PtNode arrays.
*
@ -501,14 +450,15 @@ public class BinaryDictEncoderUtils {
*
* @param dict the dictionary
* @param flatNodes the ordered list of PtNode arrays
* @param formatOptions file format options.
* @return the same array it was passed. The nodes have been updated for address and size.
*/
/* package */ static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict,
final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) {
final ArrayList<PtNodeArray> flatNodes) {
// First get the worst possible sizes and offsets
for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions);
final int offset = initializePtNodeArraysCachedAddresses(flatNodes, formatOptions);
for (final PtNodeArray n : flatNodes) {
calculatePtNodeArrayMaximumSize(n);
}
final int offset = initializePtNodeArraysCachedAddresses(flatNodes);
MakedictLog.i("Compressing the array addresses. Original size : " + offset);
MakedictLog.i("(Recursively seen size : " + offset + ")");
@ -521,8 +471,7 @@ public class BinaryDictEncoderUtils {
for (final PtNodeArray ptNodeArray : flatNodes) {
ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset;
final int oldNodeArraySize = ptNodeArray.mCachedSize;
final boolean changed =
computeActualPtNodeArraySize(ptNodeArray, dict, formatOptions);
final boolean changed = computeActualPtNodeArraySize(ptNodeArray, dict);
final int newNodeArraySize = ptNodeArray.mCachedSize;
if (oldNodeArraySize < newNodeArraySize) {
throw new RuntimeException("Increased size ?!");
@ -535,9 +484,6 @@ public class BinaryDictEncoderUtils {
if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug");
} while (changesDone);
if (formatOptions.supportsDynamicUpdate()) {
computeParentAddresses(flatNodes);
}
final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1);
MakedictLog.i("Compression complete in " + passes + " passes.");
MakedictLog.i("After address compression : "
@ -634,35 +580,29 @@ public class BinaryDictEncoderUtils {
* @param hasBigrams whether the PtNode has bigrams.
* @param isNotAWord whether the PtNode is not a word.
* @param isBlackListEntry whether the PtNode is a blacklist entry.
* @param formatOptions file format options.
* @return the flags
*/
static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal,
final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
final boolean isNotAWord, final boolean isBlackListEntry,
final FormatOptions formatOptions) {
final boolean isNotAWord, final boolean isBlackListEntry) {
byte flags = 0;
if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL;
if (formatOptions.supportsDynamicUpdate()) {
flags |= FormatSpec.FLAG_IS_NOT_MOVED;
} else if (true) {
switch (childrenAddressSize) {
case 1:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE;
break;
case 2:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES;
break;
case 3:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES;
break;
case 0:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS;
break;
default:
throw new RuntimeException("Node with a strange address");
}
switch (childrenAddressSize) {
case 1:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE;
break;
case 2:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES;
break;
case 3:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES;
break;
case 0:
flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS;
break;
default:
throw new RuntimeException("Node with a strange address");
}
if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
@ -671,12 +611,12 @@ public class BinaryDictEncoderUtils {
return flags;
}
/* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset,
final FormatOptions formatOptions) {
/* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) {
return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(),
getByteSize(childrenOffset),
node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(),
node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions);
node.mBigrams != null && !node.mBigrams.isEmpty(),
node.mIsNotAWord, node.mIsBlacklistEntry);
}
/**
@ -767,38 +707,14 @@ public class BinaryDictEncoderUtils {
+ (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY);
}
/* package */ static final int writeParentAddress(final byte[] buffer, final int index,
final int address, final FormatOptions formatOptions) {
if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) {
if (address == FormatSpec.NO_PARENT_ADDRESS) {
buffer[index] = buffer[index + 1] = buffer[index + 2] = 0;
} else {
final int absAddress = Math.abs(address);
assert(absAddress <= FormatSpec.SINT24_MAX);
buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0)
| ((absAddress >> 16) & 0xFF));
buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF);
buffer[index + 2] = (byte)(absAddress & 0xFF);
}
return index + 3;
} else {
return index;
}
}
/* package */ static final int getChildrenPosition(final PtNode ptNode,
final FormatOptions formatOptions) {
/* package */ static final int getChildrenPosition(final PtNode ptNode) {
int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate
+ getNodeHeaderSize(ptNode, formatOptions);
+ getNodeHeaderSize(ptNode);
if (ptNode.isTerminal()) {
// A terminal node has either the terminal id or the frequency.
// A terminal node has the frequency.
// If positionOfChildrenPosField is incorrect, we may crash when jumping to the children
// position.
if (formatOptions.mHasTerminalId) {
positionOfChildrenPosField += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
} else {
positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS
: ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField;
@ -810,12 +726,10 @@ public class BinaryDictEncoderUtils {
* @param dict the dictionary the node array is a part of (for relative offsets).
* @param dictEncoder the dictionary encoder.
* @param ptNodeArray the node array to write.
* @param formatOptions file format options.
*/
@SuppressWarnings("unused")
/* package */ static void writePlacedPtNodeArray(final FusionDictionary dict,
final DictEncoder dictEncoder, final PtNodeArray ptNodeArray,
final FormatOptions formatOptions) {
final DictEncoder dictEncoder, final PtNodeArray ptNodeArray) {
// TODO: Make the code in common with BinaryDictIOUtils#writePtNode
dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate);
@ -838,10 +752,7 @@ public class BinaryDictEncoderUtils {
+ FormatSpec.MAX_TERMINAL_FREQUENCY
+ " : " + ptNode.mProbabilityInfo.toString());
}
dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict);
}
if (formatOptions.supportsDynamicUpdate()) {
dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS);
dictEncoder.writePtNode(ptNode, dict);
}
if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate
+ ptNodeArray.mCachedSize) {

128
java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java
View File

@ -18,7 +18,6 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
@ -60,8 +59,7 @@ public final class BinaryDictIOUtils {
private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder,
final int bodyOffset, final Map<Integer, String> words,
final Map<Integer, Integer> frequencies,
final Map<Integer, ArrayList<PendingAttribute>> bigrams,
final FormatOptions formatOptions) {
final Map<Integer, ArrayList<PendingAttribute>> bigrams) {
int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1];
Stack<Position> stack = new Stack<Position>();
@ -90,17 +88,12 @@ public final class BinaryDictIOUtils {
stack.pop();
continue;
}
final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress, formatOptions);
final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress);
for (int i = 0; i < ptNodeInfo.mCharacters.length; ++i) {
pushedChars[index++] = ptNodeInfo.mCharacters[i];
}
p.mPosition++;
final boolean isMovedPtNode = isMovedPtNode(ptNodeInfo.mFlags,
formatOptions);
final boolean isDeletedPtNode = isDeletedPtNode(ptNodeInfo.mFlags,
formatOptions);
if (!isMovedPtNode && !isDeletedPtNode && ptNodeInfo.isTerminal()) {// found word
if (ptNodeInfo.isTerminal()) {// found word
words.put(ptNodeInfo.mOriginalAddress, new String(pushedChars, 0, index));
frequencies.put(
ptNodeInfo.mOriginalAddress, ptNodeInfo.mProbabilityInfo.mProbability);
@ -110,25 +103,13 @@ public final class BinaryDictIOUtils {
}
if (p.mPosition == p.mNumOfPtNode) {
if (formatOptions.supportsDynamicUpdate()) {
final boolean hasValidForwardLinkAddress =
dictDecoder.readAndFollowForwardLink();
if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) {
// The node array has a forward link.
p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT;
p.mAddress = dictDecoder.getPosition();
} else {
stack.pop();
}
} else {
stack.pop();
}
stack.pop();
} else {
// The Ptnode array has more PtNodes.
// The PtNode array has more PtNodes.
p.mAddress = dictDecoder.getPosition();
}
if (!isMovedPtNode && hasChildrenAddress(ptNodeInfo.mChildrenAddress)) {
if (hasChildrenAddress(ptNodeInfo.mChildrenAddress)) {
final Position childrenPos = new Position(ptNodeInfo.mChildrenAddress, index);
stack.push(childrenPos);
}
@ -153,7 +134,7 @@ public final class BinaryDictIOUtils {
// Read header
final DictionaryHeader header = dictDecoder.readHeader();
readUnigramsAndBigramsBinaryInner(dictDecoder, header.mBodyOffset, words,
frequencies, bigrams, header.mFormatOptions);
frequencies, bigrams);
}
/**
@ -171,8 +152,7 @@ public final class BinaryDictIOUtils {
final String word) throws IOException, UnsupportedFormatException {
if (word == null) return FormatSpec.NOT_VALID_WORD;
dictDecoder.setPosition(0);
final DictionaryHeader header = dictDecoder.readHeader();
dictDecoder.readHeader();
int wordPos = 0;
final int wordLen = word.codePointCount(0, word.length());
for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) {
@ -183,13 +163,7 @@ public final class BinaryDictIOUtils {
boolean foundNextPtNode = false;
for (int i = 0; i < ptNodeCount; ++i) {
final int ptNodePos = dictDecoder.getPosition();
final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos,
header.mFormatOptions);
final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags,
header.mFormatOptions);
final boolean isDeletedNode = isDeletedPtNode(currentInfo.mFlags,
header.mFormatOptions);
if (isMovedNode) continue;
final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos);
boolean same = true;
for (int p = 0, j = word.offsetByCodePoints(0, wordPos);
p < currentInfo.mCharacters.length;
@ -204,7 +178,7 @@ public final class BinaryDictIOUtils {
if (same) {
// found the PtNode matches the word.
if (wordPos + currentInfo.mCharacters.length == wordLen) {
if (!currentInfo.isTerminal() || isDeletedNode) {
if (!currentInfo.isTerminal()) {
return FormatSpec.NOT_VALID_WORD;
} else {
return ptNodePos;
@ -219,64 +193,13 @@ public final class BinaryDictIOUtils {
break;
}
}
// If we found the next PtNode, it is under the file pointer.
// But if not, we are at the end of this node array so we expect to have
// a forward link address that we need to consult and possibly resume
// search on the next node array in the linked list.
if (foundNextPtNode) break;
if (!header.mFormatOptions.supportsDynamicUpdate()) {
return FormatSpec.NOT_VALID_WORD;
}
final boolean hasValidForwardLinkAddress =
dictDecoder.readAndFollowForwardLink();
if (!hasValidForwardLinkAddress || !dictDecoder.hasNextPtNodeArray()) {
return FormatSpec.NOT_VALID_WORD;
}
return FormatSpec.NOT_VALID_WORD;
} while(true);
}
return FormatSpec.NOT_VALID_WORD;
}
/**
* @return the size written, in bytes. Always 3 bytes.
*/
@UsedForTesting
static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) {
final int absValue = Math.abs(value);
dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF));
dictBuffer.put((byte)((absValue >> 8) & 0xFF));
dictBuffer.put((byte)(absValue & 0xFF));
return 3;
}
/**
* @return the size written, in bytes. Always 3 bytes.
*/
@UsedForTesting
static int writeSInt24ToStream(final OutputStream destination, final int value)
throws IOException {
final int absValue = Math.abs(value);
destination.write((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF));
destination.write((byte)((absValue >> 8) & 0xFF));
destination.write((byte)(absValue & 0xFF));
return 3;
}
@UsedForTesting
static void skipString(final DictBuffer dictBuffer,
final boolean hasMultipleChars) {
if (hasMultipleChars) {
int character = CharEncoding.readChar(dictBuffer);
while (character != FormatSpec.INVALID_CHARACTER) {
character = CharEncoding.readChar(dictBuffer);
}
} else {
CharEncoding.readChar(dictBuffer);
}
}
/**
* Writes a PtNodeCount to the stream.
*
@ -355,30 +278,6 @@ public final class BinaryDictIOUtils {
return FormatSpec.NO_CHILDREN_ADDRESS != address;
}
/**
* Helper method to check whether the node is moved.
*/
public static boolean isMovedPtNode(final int flags, final FormatOptions options) {
return options.supportsDynamicUpdate()
&& ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED);
}
/**
* Helper method to check whether the dictionary can be updated dynamically.
*/
public static boolean supportsDynamicUpdate(final FormatOptions options) {
return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE
&& options.supportsDynamicUpdate();
}
/**
* Helper method to check whether the node is deleted.
*/
public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) {
return formatOptions.supportsDynamicUpdate()
&& ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED);
}
/**
* Compute the binary size of the node count
* @param count the node count
@ -396,9 +295,7 @@ public final class BinaryDictIOUtils {
}
}
static int getChildrenAddressSize(final int optionFlags,
final FormatOptions formatOptions) {
if (formatOptions.supportsDynamicUpdate()) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE;
static int getChildrenAddressSize(final int optionFlags) {
switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) {
case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE:
return 1;
@ -419,6 +316,7 @@ public final class BinaryDictIOUtils {
* @param bigramFrequency compressed frequency
* @return approximate bigram frequency
*/
@UsedForTesting
public static int reconstructBigramFrequency(final int unigramFrequency,
final int bigramFrequency) {
final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency)

12
java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
View File

@ -18,7 +18,6 @@ package com.android.inputmethod.latin.makedict;
import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.utils.ByteArrayDictBuffer;
import java.io.File;
@ -45,10 +44,9 @@ public interface DictDecoder {
/**
* Reads PtNode from ptNodePos.
* @param ptNodePos the position of PtNode.
* @param formatOptions the format options.
* @return PtNodeInfo.
*/
public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions);
public PtNodeInfo readPtNode(final int ptNodePos);
/**
* Reads a buffer and returns the memory representation of the dictionary.
@ -112,14 +110,6 @@ public interface DictDecoder {
*/
public int readPtNodeCount();
/**
* Reads the forward link and advances the position.
*
* @return true if this method moves the file pointer, false otherwise.
*/
public boolean readAndFollowForwardLink();
public boolean hasNextPtNodeArray();
/**
* Opens the dictionary file and makes DictBuffer.
*/

4
java/src/com/android/inputmethod/latin/makedict/DictEncoder.java
View File

@ -32,7 +32,5 @@ public interface DictEncoder {
public int getPosition();
public void writePtNodeCount(final int ptNodeCount);
public void writeForwardLinkAddress(final int forwardLinkAddress);
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict);
public void writePtNode(final PtNode ptNode, final FusionDictionary dict);
}

6
java/src/com/android/inputmethod/latin/makedict/FormatSpec.java
View File

@ -309,7 +309,6 @@ public final class FormatSpec {
*/
public static final class FormatOptions {
public final int mVersion;
public final boolean mHasTerminalId;
public final boolean mHasTimestamp;
@UsedForTesting
@ -319,13 +318,8 @@ public final class FormatSpec {
public FormatOptions(final int version, final boolean hasTimestamp) {
mVersion = version;
mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID);
mHasTimestamp = hasTimestamp;
}
public boolean supportsDynamicUpdate() {
return mVersion >= FIRST_VERSION_WITH_DYNAMIC_UPDATE;
}
}
/**

4
java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java
View File

@ -31,12 +31,11 @@ public final class PtNodeInfo {
public final int[] mCharacters;
public final ProbabilityInfo mProbabilityInfo;
public final int mChildrenAddress;
public final int mParentAddress;
public final ArrayList<WeightedString> mShortcutTargets;
public final ArrayList<PendingAttribute> mBigrams;
public PtNodeInfo(final int originalAddress, final int endAddress, final int flags,
final int[] characters, final ProbabilityInfo probabilityInfo, final int parentAddress,
final int[] characters, final ProbabilityInfo probabilityInfo,
final int childrenAddress, final ArrayList<WeightedString> shortcutTargets,
final ArrayList<PendingAttribute> bigrams) {
mOriginalAddress = originalAddress;
@ -44,7 +43,6 @@ public final class PtNodeInfo {
mFlags = flags;
mCharacters = characters;
mProbabilityInfo = probabilityInfo;
mParentAddress = parentAddress;
mChildrenAddress = childrenAddress;
mShortcutTargets = shortcutTargets;
mBigrams = bigrams;

36
java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
View File

@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.BinaryDictionary;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
import com.android.inputmethod.latin.utils.CollectionUtils;
@ -173,21 +172,23 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
@Override
public DictionaryHeader readHeader() throws IOException, UnsupportedFormatException {
if (mDictBuffer == null) {
final DictionaryHeader header = mBinaryDictionary.getHeader();
if (header.mFormatOptions.mVersion != FormatSpec.VERSION2) {
throw new UnsupportedFormatException("File header has a wrong version : "
+ header.mFormatOptions.mVersion);
}
if (!isDictBufferOpen()) {
openDictBuffer();
}
final DictionaryHeader header = super.readHeader(mDictBuffer);
final int version = header.mFormatOptions.mVersion;
if (version != FormatSpec.VERSION2) {
throw new UnsupportedFormatException("File header has a wrong version : " + version);
}
// Advance buffer reading position to the head of dictionary body.
setPosition(header.mBodyOffset);
return header;
}
// TODO: Make this buffer multi thread safe.
private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
@Override
public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) {
public PtNodeInfo readPtNode(final int ptNodePos) {
int addressPointer = ptNodePos;
final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
addressPointer += FormatSpec.PTNODE_FLAGS_SIZE;
@ -221,7 +222,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
childrenAddress += addressPointer;
}
addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags);
final ArrayList<WeightedString> shortcutTargets;
if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) {
// readShortcut will add shortcuts to shortcutTargets.
@ -244,7 +245,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
bigrams = null;
}
return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo,
FormatSpec.NO_PARENT_ADDRESS, childrenAddress, shortcutTargets, bigrams);
childrenAddress, shortcutTargets, bigrams);
}
@Override
@ -307,19 +308,4 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
public int readPtNodeCount() {
return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer);
}
@Override
public boolean readAndFollowForwardLink() {
final int nextAddress = mDictBuffer.readUnsignedInt24();
if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) {
mDictBuffer.position(nextAddress);
return true;
}
return false;
}
@Override
public boolean hasNextPtNodeArray() {
return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS;
}
}

42
java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
View File

@ -95,7 +95,7 @@ public class Ver2DictEncoder implements DictEncoder {
ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray);
MakedictLog.i("Computing addresses...");
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions);
BinaryDictEncoderUtils.computeAddresses(dict, flatNodes);
MakedictLog.i("Checking PtNode array...");
if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes);
@ -107,7 +107,7 @@ public class Ver2DictEncoder implements DictEncoder {
MakedictLog.i("Writing file...");
for (PtNodeArray nodeArray : flatNodes) {
BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions);
BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray);
}
if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes);
mOutStream.write(mBuffer, 0, mPosition);
@ -139,24 +139,13 @@ public class Ver2DictEncoder implements DictEncoder {
countSize);
}
private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
private void writePtNodeFlags(final PtNode ptNode) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode);
mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition,
BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions),
BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos),
FormatSpec.PTNODE_FLAGS_SIZE);
}
private void writeParentPosition(final int parentPosition, final PtNode ptNode,
final FormatOptions formatOptions) {
if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) {
mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition,
parentPosition, formatOptions);
} else {
mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition,
parentPosition - ptNode.mCachedAddressAfterUpdate, formatOptions);
}
}
private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) {
mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition);
if (hasSeveralChars) {
@ -171,15 +160,10 @@ public class Ver2DictEncoder implements DictEncoder {
}
}
private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions);
if (formatOptions.supportsDynamicUpdate()) {
mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition,
childrenPos);
} else {
mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
childrenPos);
}
private void writeChildrenPosition(final PtNode ptNode) {
final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode);
mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition,
childrenPos);
}
/**
@ -246,13 +230,11 @@ public class Ver2DictEncoder implements DictEncoder {
}
@Override
public void writePtNode(final PtNode ptNode, final int parentPosition,
final FormatOptions formatOptions, final FusionDictionary dict) {
writePtNodeFlags(ptNode, formatOptions);
writeParentPosition(parentPosition, ptNode, formatOptions);
public void writePtNode(final PtNode ptNode, final FusionDictionary dict) {
writePtNodeFlags(ptNode);
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars());
writeFrequency(ptNode.getProbability());
writeChildrenPosition(ptNode, formatOptions);
writeChildrenPosition(ptNode);
writeShortcuts(ptNode.mShortcutTargets);
writeBigrams(ptNode.mBigrams, dict);
}

3
java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
View File

@ -122,7 +122,6 @@ public class Ver4DictEncoder implements DictEncoder {
}
@Override
public void writePtNode(
PtNode ptNode, int parentPosition, FormatOptions formatOptions, FusionDictionary dict) {
public void writePtNode(PtNode ptNode, FusionDictionary dict) {
}
}

2
tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
View File

@ -501,7 +501,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
if (fileHeader == null) return null;
return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset,
address, fileHeader.mFormatOptions).mWord;
address).mWord;
}
private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word,