tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

korean splitter: disable the noun+jx emitting thing

Browse Source
This commit is contained in:
Jean-Francois Dockes 2020-05-14 09:23:09 +02:00
parent 38f78e6f90
commit 97f3212f80
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/common/textsplitko.cpp
View File

@ -218,6 +218,10 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
" bytepos " << bytepos << " word from text: " << " bytepos " << bytepos << " word from text: " <<
inputdata.substr(bytepos, word.size()) << endl); inputdata.substr(bytepos, word.size()) << endl);
bool isNoun = (tags[i] == "Noun"); bool isNoun = (tags[i] == "Noun");
#if 0
// When Noun followed by JX, emit both Noun and Noun+JX at the
// same pos Experimental, it seems that this is sometimes
// problematic, so turned off for now.
if (isNoun) { if (isNoun) {
lastNoun = word; lastNoun = word;
lastNounWordPos = m_wordpos; lastNounWordPos = m_wordpos;
@ -231,6 +235,7 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
} }
lastNoun.clear(); lastNoun.clear();
} }
#endif
// 11/05/2020 For now index everything until more precise // 11/05/2020 For now index everything until more precise
// verification of what should be pruned // verification of what should be pruned
if (true || (isNoun || tags[i] == "Verb" || if (true || (isNoun || tags[i] == "Verb" ||