n/futokb
1
0
Fork 0
mirror of https://gitlab.futo.org/keyboard/latinime.git synced 2024-09-28 14:54:30 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Save language model in the body buffer.

Browse Source 
Bug: 14425059
Change-Id: Iaec277f7bed03d6c6780c6ce90fbe5fe799e175e
This commit is contained in:
Keisuke Kuroyanagi 2014-08-01 20:19:16 +09:00
parent c0c674cdc0
commit c4696b2eb6
6 changed files with 56 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
native/jni/NativeFileList.mk
View File

@ -72,6 +72,7 @@ LATIN_IME_CORE_SRC_FILES := \
ver4_pt_node_array_reader.cpp) \
$(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \
bigram_dict_content.cpp \
language_model_dict_content.cpp \
probability_dict_content.cpp \
shortcut_dict_content.cpp \
sparse_table_dict_content.cpp \

25
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright (C) 2014, The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"
namespace latinime {
bool LanguageModelDictContent::save(FILE *const file) const {
return mTrieMap.save(file);
}
} // namespace latinime

14
native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h
View File

@ -17,16 +17,28 @@
#ifndef LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H
#define LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H
#include <cstdio>
#include "defines.h"
#include "suggest/policyimpl/dictionary/utils/trie_map.h"
#include "utils/byte_array_view.h"
namespace latinime {
class LanguageModelDictContent {
public:
explicit LanguageModelDictContent(const bool hasHistoricalInfo) {}
LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer,
const bool hasHistoricalInfo)
: mTrieMap(trieMapBuffer) {}
explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {}
bool save(FILE *const file) const;
private:
DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);
TrieMap mTrieMap;
};
} // namespace latinime
#endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */

11
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp
View File

@ -162,6 +162,11 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const {
AKLOGE("Probability dict content cannot be written.");
return false;
}
// Write language model content.
if (!mLanguageModelDictContent.save(file)) {
AKLOGE("Language model dict content cannot be written.");
return false;
}
// Write bigram dict content.
if (!mBigramDictContent.flushToFile(file)) {
AKLOGE("Bigram dict content cannot be written.");
@ -195,7 +200,11 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer,
contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords()),
mLanguageModelDictContent(mHeaderPolicy.hasHistoricalInfoOfWords()),
mLanguageModelDictContent(
ReadWriteByteArrayView(
contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX],
contentBufferSizes[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX]),
mHeaderPolicy.hasHistoricalInfoOfWords()),
mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
&contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],
mHeaderPolicy.hasHistoricalInfoOfWords()),

6
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp
View File

@ -31,14 +31,17 @@ const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024;
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.
const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =
NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3
+ NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT
+ NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;
const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;
const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =
TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =
TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX =
PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =
LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =
BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
@ -73,5 +76,6 @@ const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3;
const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT = 1;
} // namespace latinime

2
native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h
View File

@ -36,6 +36,7 @@ class Ver4DictConstants {
static const int TRIE_BUFFER_INDEX;
static const int TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX;
static const int PROBABILITY_BUFFER_INDEX;
static const int LANGUAGE_MODEL_BUFFER_INDEX;
static const int BIGRAM_BUFFERS_INDEX;
static const int SHORTCUT_BUFFERS_INDEX;
@ -71,6 +72,7 @@ class Ver4DictConstants {
static const size_t NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
static const size_t NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;
};
} // namespace latinime
#endif /* LATINIME_VER4_DICT_CONSTANTS_H */