more utf-8 err checking prevents bogus terms in index

This commit is contained in:
Jean-Francois Dockes
2013-03-30 10:24:10 +01:00
parent 2018ef76b8
commit 0ae8ec99f6
6 changed files with 54 additions and 23 deletions
+3
View File
@@ -350,6 +350,9 @@ bool TextSplit::text_to_words(const string &in)
m_flags & TXTS_KEEPWILD ? " keepwild" : "",
in.substr(0,50).c_str()));
if (in.empty())
return true;
m_span.erase();
m_inNumber = false;
m_wordStart = m_wordLen = m_prevpos = m_prevlen = m_wordpos = m_spanpos = 0;