tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

get rid of a few garbage terms during indexing. Set a threshold for conversion errors after which we discard the doc. Stabilize the new termproc pipeline but no commongrams for now

Browse Source
This commit is contained in:
Jean-Francois Dockes
2011-10-12 17:55:58 +02:00
parent a2c9d2a82b
commit 0860b559ee
4 changed files with 167 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+18 -11
View File
@@ -897,8 +897,9 @@ private:
// Reimplement text_to_words to insert the begin and end anchor terms.
bool TextSplitDb::text_to_words(const string &in)
{
LOGDEB2(("TextSplitDb::text_to_words\n"));
bool ret = false;
string ermsg;
try {
// Index the possibly prefixed start term.
doc.add_posting(prefix + start_of_field_term, basepos, wdfinc);
@@ -906,14 +907,12 @@ bool TextSplitDb::text_to_words(const string &in)
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Db: xapian add_posting error %s\n", ermsg.c_str()));
basepos += curpos + 100;
return false;
goto out;
}
if (!TextSplitP::text_to_words(in)) {
LOGDEB(("TextSplitDb: TextSplit::text_to_words failed\n"));
basepos += curpos + 100;
return false;
goto out;
}
try {
@@ -923,10 +922,12 @@ bool TextSplitDb::text_to_words(const string &in)
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Db: xapian add_posting error %s\n", ermsg.c_str()));
basepos += curpos + 100;
return false;
goto out;
}
ret = true;
out:
basepos += curpos + 100;
return true;
}
@@ -961,6 +962,7 @@ public:
LOGERR(("Db: xapian add_posting error %s\n", ermsg.c_str()));
return false;
}
private:
TextSplitDb *m_ts;
};
@@ -1028,12 +1030,17 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
Doc doc = idoc;
Xapian::Document newdocument;
// The term processing pipeline:
TermProcIdx tpidx;
// TermProcStop tpstop(&tpidx, m_stops);
TermProcCommongrams tpstop(&tpidx, m_stops);
TermProcPrep tpprep(&tpstop);
TextSplitDb splitter(m_ndb->xwdb, newdocument, &tpprep);
TermProc *nxt = &tpidx;
TermProcStop tpstop(nxt, m_stops);nxt = &tpstop;
// TermProcCommongrams tpcommon(nxt, m_stops); nxt = &tpcommon;
TermProcPrep tpprep(nxt); nxt = &tpprep;
TextSplitDb splitter(m_ndb->xwdb, newdocument, nxt);
tpidx.setTSD(&splitter);
// Split and index file name as document term(s)
LOGDEB2(("Db::add: split file name [%s]\n", fn.c_str()));
if (!splitter.text_to_words(doc.utf8fn))
src/rcldb/searchdata.cpp
+41 -36
View File
@@ -478,7 +478,8 @@ void SearchData::getUTerms(vector<string>& terms) const
class TextSplitQ : public TextSplitP {
public:
TextSplitQ(Flags flags, const StopList &_stops, TermProc *prc)
: TextSplitP(prc, flags), stops(_stops), alltermcount(0), lastpos(0)
: TextSplitP(prc, flags),
curnostemexp(false), stops(_stops), alltermcount(0), lastpos(0)
{}
bool takeword(const std::string &term, int pos, int bs, int be)
@@ -509,16 +510,30 @@ public:
bool takeword(const std::string &term, int pos, int bs, int be)
{
m_ts->alltermcount++;
m_ts->lastpos = pos;
if (m_ts->lastpos < pos)
m_ts->lastpos = pos;
bool noexpand = be ? m_ts->curnostemexp : true;
LOGDEB(("TermProcQ::takeword: pushing [%s] noexp %d\n",
term.c_str(), noexpand));
m_ts->terms.push_back(term);
m_ts->nostemexps.push_back(noexpand);
LOGDEB(("TermProcQ::takeword: pushing [%s] pos %d noexp %d\n",
term.c_str(), pos, noexpand));
if (m_terms[pos].size() < term.size()) {
m_terms[pos] = term;
m_nste[pos] = noexpand;
}
return true;
}
bool flush()
{
for (map<int, string>::const_iterator it = m_terms.begin();
it != m_terms.end(); it++) {
m_ts->terms.push_back(it->second);
m_ts->nostemexps.push_back(m_nste[it->first]);
}
return true;
}
private:
TextSplitQ *m_ts;
map<int, string> m_terms;
map<int, bool> m_nste;
};
// A class used to translate a user compound string (*not* a query
@@ -783,7 +798,7 @@ void StringToXapianQ::processPhraseOrNear(TextSplitQ *splitData,
// Generate an appropriate PHRASE/NEAR query with adjusted slack
// For phrases, give a relevance boost like we do for original terms
LOGDEB2(("PHRASE/NEAR: alltermcount %d lastpos %d\n",
LOGDEB2(("PHRASE/NEAR: alltermcount %d lastpos %d\n",
splitData->alltermcount, splitData->lastpos));
Xapian::Query xq(op, orqueries.begin(), orqueries.end(),
splitData->lastpos + 1 + slack);
@@ -839,7 +854,7 @@ bool StringToXapianQ::processUserString(const string &iq,
bool useNear
)
{
LOGDEB(("StringToXapianQ:: query string: [%s]\n", iq.c_str()));
LOGDEB(("StringToXapianQ:: query string: [%s], slack %d, near %d\n", iq.c_str(), slack, useNear));
ermsg.erase();
m_uterms.clear();
m_terms.clear();
@@ -874,45 +889,35 @@ bool StringToXapianQ::processUserString(const string &iq,
// We used to do word split, searching for
// "term0 term1 term2" instead, which may have worse
// performance, but will succeed.
// We now adjust the phrase/near slack by the term count
// difference (this is mainly better for cjk where this is a very
// common occurrence because of the ngrams thing.
// We now adjust the phrase/near slack by comparing the term count
// and the last position
// The term processing pipeline:
TermProcQ tpq;
// TermProcStop tpstop(&tpidx, stops);
TermProcCommongrams tpstop(&tpq, stops);
tpstop.onlygrams(true);
TermProcPrep tpprep(&tpstop);
TermProc *nxt = &tpq;
TermProcStop tpstop(nxt, stops); nxt = &tpstop;
//TermProcCommongrams tpcommon(nxt, stops); nxt = &tpcommon;
//tpcommon.onlygrams(true);
TermProcPrep tpprep(nxt); nxt = &tpprep;
TextSplitQ splitterS(TextSplit::Flags(TextSplit::TXTS_ONLYSPANS |
TextSplitQ splitter(TextSplit::Flags(TextSplit::TXTS_ONLYSPANS |
TextSplit::TXTS_KEEPWILD),
stops, &tpprep);
tpq.setTSQ(&splitterS);
splitterS.text_to_words(*it);
LOGDEB(("SplitterS has %d terms\n", splitterS.terms.size()));
TextSplitQ splitterW(TextSplit::Flags(TextSplit::TXTS_NOSPANS |
TextSplit::TXTS_KEEPWILD),
stops, &tpprep);
tpq.setTSQ(&splitterW);
tpstop.onlygrams(false);
splitterW.text_to_words(*it);
stops, nxt);
tpq.setTSQ(&splitter);
splitter.text_to_words(*it);
if (splitterS.terms.size() > 1 &&
splitterS.terms.size() != splitterW.terms.size()) {
slack += splitterW.terms.size() - splitterS.terms.size();
}
slack += splitter.lastpos - splitter.terms.size() + 1;
TextSplitQ *splitter = &splitterS;
LOGDEB0(("strToXapianQ: termcount: %d\n", splitter->terms.size()));
switch (splitter->terms.size() + terminc) {
LOGDEB0(("strToXapianQ: termcount: %d\n", splitter.terms.size()));
switch (splitter.terms.size() + terminc) {
case 0:
continue;// ??
case 1:
processSimpleSpan(splitter->terms.front(),
splitter->nostemexps.front(), pqueries);
processSimpleSpan(splitter.terms.front(),
splitter.nostemexps.front(), pqueries);
break;
default:
processPhraseOrNear(splitter, pqueries, useNear, slack, mods);
processPhraseOrNear(&splitter, pqueries, useNear, slack, mods);
}
}
} catch (const Xapian::Error &e) {
src/rcldb/termproc.h
+37 -12
View File
@@ -66,10 +66,10 @@ private:
};
/**
* Intermediary specialized texsplit class: this will probably replace the base
* textsplit when we've converted all the code. The takeword() routine in this
* calls a TextProc's instead of being specialized in a derived class by the
* user module. The text_to_word() method also takes care of flushing.
* Specialized TextSplit class: this will probably replace the base
* TextSplit when we've converted all the code. The takeword() routine in this
* calls a TermProc's instead of being overriden in a user derived class.
* The text_to_word() method also takes care of flushing.
*/
class TextSplitP : public TextSplit {
public:
@@ -99,18 +99,39 @@ private:
/** Unaccent and lowercase term. This is usually the first in the pipeline */
class TermProcPrep : public TermProc {
public:
TermProcPrep(TermProc *nxt) : TermProc(nxt) {}
TermProcPrep(TermProc *nxt)
: TermProc(nxt), m_totalterms(0), m_unacerrors(0) {}
virtual bool takeword(const string& itrm, int pos, int bs, int be)
{
m_totalterms++;
string otrm;
if (!unacmaybefold(itrm, otrm, "UTF-8", true)) {
LOGINFO(("splitter::takeword: unac [%s] failed\n", itrm.c_str()));
// We don't generate a fatal error because of a bad term
LOGDEB(("splitter::takeword: unac [%s] failed\n", itrm.c_str()));
m_unacerrors++;
// We don't generate a fatal error because of a bad term,
// but one has to put the limit somewhere
if (m_unacerrors > 500 &&
(double(m_totalterms) / double(m_unacerrors)) < 2.0) {
// More than 1 error for every other term
LOGERR(("splitter::takeword: too many unac errors %d/%d\n",
m_unacerrors, m_totalterms));
return false;
}
return true;
}
return TermProc::takeword(otrm, pos, bs, be);
}
virtual bool flush()
{
m_totalterms = m_unacerrors = 0;
return TermProc::flush();
}
private:
int m_totalterms;
int m_unacerrors;
};
/** Compare to stop words list and discard if match found */
@@ -119,19 +140,23 @@ public:
TermProcStop(TermProc *nxt, const Rcl::StopList& stops)
: TermProc(nxt), m_stops(stops) {}
virtual bool takeword(const string& term, int pos, int bts, int bte)
virtual bool takeword(const string& term, int pos, int bs, int be)
{
if (m_stops.isStop(term)) {
return true;
}
return TermProc::takeword(term, pos, bts, bte);
return TermProc::takeword(term, pos, bs, be);
}
private:
const Rcl::StopList& m_stops;
};
/** Handle common-gram generation: combine frequent terms with neighbours to
* shorten the positions lists for phrase searches.
* NOTE: This does not currently work because of bad interaction with the
* spans (ie john@domain.com) generation in textsplit. Not used, kept for
* testing only
*/
class TermProcCommongrams : public TermProc {
public:
@@ -147,7 +172,7 @@ public:
if (!m_prevterm.empty() && (m_prevstop || isstop)) {
// create 2-gram. space unnecessary but improves
// lisibility of queries
// the readability of queries
string twogram;
twogram.swap(m_prevterm);
twogram.append(1, ' ');
@@ -164,7 +189,7 @@ public:
}
#endif
}
m_prevterm = term;
m_prevstop = isstop;
m_prevpos = pos;
@@ -181,7 +206,7 @@ public:
return true;
}
bool flush()
virtual bool flush()
{
if (!m_prevsent && !m_prevterm.empty())
if (!TermProc::takeword(m_prevterm, m_prevpos, m_prevbs, m_prevbe))