tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Korean external splitter: add some support for Mecab

Browse Source
This commit is contained in:
Jean-Francois Dockes
2020-03-23 16:20:32 +01:00
parent c9667b5ba7
commit 9719177c82
5 changed files with 92 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/textsplit.cpp
+18 -8
View File
@@ -44,8 +44,10 @@
// ngrams
#undef KATAKANA_AS_WORDS
// Same for Korean syllabic, and same problem, not used.
#undef HANGUL_AS_WORDS
// Same for Korean syllabic, and same problem. However we have a
// runtime option to use an external text analyser for hangul, so this
// is defined at compile time.
#define HANGUL_AS_WORDS
using namespace std;
@@ -246,7 +248,6 @@ static inline int whatcc(unsigned int c, char *asciirep = nullptr)
#define UNICODE_IS_KATAKANA(p) false
#endif
#define HANGUL_AS_WORDS
#ifdef HANGUL_AS_WORDS
#define UNICODE_IS_HANGUL(p) ( \
((p) >= 0x1100 && (p) <= 0x11FF) || \
@@ -290,6 +291,7 @@ bool TextSplit::o_noNumbers{false};
bool TextSplit::o_deHyphenate{false};
int TextSplit::o_maxWordLength{40};
static const int o_CJKMaxNgramLen{5};
bool o_exthangultagger{false};
void TextSplit::staticConfInit(RclConfig *config)
{
@@ -324,7 +326,13 @@ void TextSplit::staticConfInit(RclConfig *config)
charclasses[int('\\')] = SPACE;
}
}
koStaticConfInit(config);
string kotagger;
config->getConfParam("hangultagger", kotagger);
if (!kotagger.empty()) {
o_exthangultagger = true;
koStaticConfInit(config, kotagger);
}
}
// Final term checkpoint: do some checking (the kind which is simpler
@@ -627,7 +635,11 @@ bool TextSplit::text_to_words(const string &in)
if (UNICODE_IS_KATAKANA(c)) {
csc = CSC_KATAKANA;
} else if (UNICODE_IS_HANGUL(c)) {
csc = CSC_HANGUL;
if (o_exthangultagger) {
csc = CSC_HANGUL;
} else {
csc = CSC_CJK;
}
} else if (UNICODE_IS_CJK(c)) {
csc = CSC_CJK;
} else {
@@ -635,15 +647,13 @@ bool TextSplit::text_to_words(const string &in)
}
if (o_processCJK && (csc == CSC_CJK || csc == CSC_HANGUL)) {
// CJK character hit. Hangul processing may be special or
// not depending on how we were built.
// CJK character hit. Hangul processing may be special.
// Do like at EOF with the current non-cjk data.
if (m_wordLen || m_span.length()) {
if (!doemit(true, it.getBpos()))
return false;
}
// Hand off situation to the appropriate routine.
if (csc == CSC_HANGUL) {
if (!ko_to_words(&it, &c)) {
src/common/textsplit.h
+1 -1
View File
@@ -54,7 +54,7 @@ public:
/** Call at program initialization to read non default values from the
configuration */
static void staticConfInit(RclConfig *config);
static void koStaticConfInit(RclConfig *config);
static void koStaticConfInit(RclConfig *config, const std::string& tagger);
/** Split text, emit words and positions. */
virtual bool text_to_words(const std::string &in);
src/common/textsplitko.cpp
+34 -12
View File
@@ -15,6 +15,13 @@
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
// Specialized Korean text splitter using konlpy running in a Python
// subprocess. konlpy can use several different backends. We support
// Okt (Twitter) and Mecab at this point. Unfortunately the different
// backends have different POS TAG names, so that things are not
// completly transparent when using another (need to translate the tag
// names in the Python program).
#include "autoconfig.h"
#include <iostream>
@@ -33,16 +40,27 @@
using namespace std;
// Separator char used in words and tags lists.
static const string sepchars("\t");
static CmdTalk *o_talker;
static bool o_starterror{false};
static string o_cmdpath;
std::mutex o_mutex;
static string o_taggername{"Okt"};
void TextSplit::koStaticConfInit(RclConfig *config)
void TextSplit::koStaticConfInit(RclConfig *config, const string& tagger)
{
o_cmdpath = config->findFilter("kosplitter.py");
if (tagger == "Okt" && tagger == "Mecab") {
o_taggername = tagger;
} else {
LOGERR("TextSplit::koStaticConfInit: unknown tagger [" << tagger <<
"], using Okt\n");
}
}
// Start the Python subprocess
static bool initCmd()
{
if (o_starterror) {
@@ -68,8 +86,6 @@ static bool initCmd()
return true;
}
static const string sepchars("\t");
bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
{
std::unique_lock<std::mutex> mylock(o_mutex);
@@ -78,18 +94,28 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
return false;
}
}
LOGDEB1("k_to_words: m_wordpos " << m_wordpos << "\n");
Utf8Iter &it = *itp;
unsigned int c = 0;
unordered_map<string, string> args;
args.insert(pair<string,string>{"data", string()});
string& inputdata{args.begin()->second};
string::size_type orgbytepos = it.getBpos();
// We send the tagger name every time but it's only used the first
// one: can't change it after init. We could avoid sending it
// every time, but I don't think that the performance hit is
// significant
args.insert(pair<string,string>{"tagger", o_taggername});
// Gather all Korean characters and send the text to the analyser
// Walk the Korean characters section and send the text to the
// analyser
string::size_type orgbytepos = it.getBpos();
for (; !it.eof(); it++) {
c = *it;
if (!isHANGUL(c) && !(isascii(c) && (isspace(c) || ispunct(c)))) {
if (!isHANGUL(c) && !(isspace(c) || ispunct(c))) {
// Done with Korean stretch, process and go back to main routine
//std::cerr << "Broke on char " << int(c) << endl;
break;
@@ -97,10 +123,6 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
it.appendchartostring(inputdata);
}
}
// Need to convert white text spans to single space otherwise the
// byte offsets will be wrong
string::size_type textsize = inputdata.size();
LOGDEB1("TextSplit::k_to_words: sending out " << inputdata.size() <<
" bytes " << inputdata << endl);
unordered_map<string,string> result;
@@ -161,11 +183,11 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
}
#if DO_CHECK_THINGS
int sizediff = textsize - (bytepos - orgbytepos);
int sizediff = inputdata.size() - (bytepos - orgbytepos);
if (sizediff < 0)
sizediff = -sizediff;
if (sizediff > 1) {
LOGERR("ORIGINAL TEXT SIZE: " << textsize <<
LOGERR("ORIGINAL TEXT SIZE: " << inputdata.size() <<
" FINAL BYTE POS " << bytepos - orgbytepos <<
" TEXT [" << inputdata << "]\n");
}