korean: for now dont filter tags, until it is better understood what should be done
This commit is contained in:
parent
d8edbcbc55
commit
d58fec0b81
@ -231,8 +231,10 @@ bool TextSplit::ko_to_words(Utf8Iter *itp, unsigned int *cp)
|
|||||||
}
|
}
|
||||||
lastNoun.clear();
|
lastNoun.clear();
|
||||||
}
|
}
|
||||||
if (isNoun || tags[i] == "Verb" ||
|
// 11/05/2020 For now index everything until more precise
|
||||||
tags[i] == "Adjective" || tags[i] == "Adverb") {
|
// verification of what should be pruned
|
||||||
|
if (true || (isNoun || tags[i] == "Verb" ||
|
||||||
|
tags[i] == "Adjective" || tags[i] == "Adverb")) {
|
||||||
string::size_type abspos = orgbytepos + bytepos - pagefix;
|
string::size_type abspos = orgbytepos + bytepos - pagefix;
|
||||||
if (!takeword(word, m_wordpos++, abspos, abspos + word.size())) {
|
if (!takeword(word, m_wordpos++, abspos, abspos + word.size())) {
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user