tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

abstract: we used to discard snippets too early, before they might get a phrase weight boost

Browse Source
This commit is contained in:
Jean-Francois Dockes 2019-05-24 08:51:03 +02:00
parent 15dc419fec
commit 2a945c9443
1 changed files with 33 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/rcldb/rclabsfromtext.cpp
View File

@ -136,6 +136,14 @@ public:
// abstract will be incorrect or inexistant, but this is // abstract will be incorrect or inexistant, but this is
// better than taking forever (the default cutoff is 10E6) // better than taking forever (the default cutoff is 10E6)
if (maxtermcount && termcount++ > maxtermcount) { if (maxtermcount && termcount++ > maxtermcount) {
LOGINF("Rclabsfromtext: stopping because maxtermcount reached: "<<
maxtermcount << endl);
return false;
}
// Also limit the number of fragments (just in case safety)
if (m_fragments.size() > maxtermcount / 100) {
LOGINF("Rclabsfromtext: stopping because maxfragments reached: "<<
maxtermcount/100 << endl);
return false; return false;
} }
// Remember recent past // Remember recent past
@ -223,8 +231,16 @@ public:
m_remainingWords--; m_remainingWords--;
m_curfrag.second = bte; m_curfrag.second = bte;
if (m_remainingWords == 0) { if (m_remainingWords == 0) {
if (m_totalcoef < 5.0 || m_curfragcoef >= 1.0) { // We used to not push weak fragments if we had a lot
// Don't push bad fragments if we have a lot already // already. This can cause problems if the fragments
// we drop are actually group fragments (which have
// not got their boost yet). The right cut value is
// difficult to determine, because the absolute values
// of the coefs depend on many things (index size,
// etc.) The old test was if (m_totalcoef < 5.0 ||
// m_curfragcoef >= 1.0) We now just avoid creating a
// monster by testing the current fragments count at
// the top of the function
m_fragments.push_back(MatchFragment(m_curfrag.first, m_fragments.push_back(MatchFragment(m_curfrag.first,
m_curfrag.second, m_curfrag.second,
m_curfragcoef, m_curfragcoef,
@ -234,7 +250,6 @@ public:
m_curhitpos, m_curhitpos,
m_curterm m_curterm
)); ));
}
m_totalcoef += m_curfragcoef; m_totalcoef += m_curfragcoef;
m_curfragcoef = 0.0; m_curfragcoef = 0.0;
m_curtermcoef = 0.0; m_curtermcoef = 0.0;
@ -252,6 +267,8 @@ public:
// find the group matches. We process everything as NEAR (no // find the group matches. We process everything as NEAR (no
// PHRASE specific processing). // PHRASE specific processing).
void updgroups() { void updgroups() {
LOGDEB("TextSplitABS: stored total " << m_fragments.size() <<
" fragments" << endl);
vector<GroupMatchEntry> tboffs; vector<GroupMatchEntry> tboffs;
// Look for matches to PHRASE and NEAR term groups and finalize // Look for matches to PHRASE and NEAR term groups and finalize
@ -283,7 +300,7 @@ public:
); );
// Give a boost to fragments which contain a group match // Give a boost to fragments which contain a group match
// (phrase/near), they are dear to the user's heart. list are // (phrase/near), they are dear to the user's heart. Lists are
// sorted, so we never go back in the fragment list (can // sorted, so we never go back in the fragment list (can
// always start the search where we previously stopped). // always start the search where we previously stopped).
if (m_fragments.empty()) { if (m_fragments.empty()) {
@ -413,7 +430,6 @@ int Query::Native::abstractFromText(
} }
); );
vector<int> vpbreaks; vector<int> vpbreaks;
ndb->getPagePositions(docid, vpbreaks); ndb->getPagePositions(docid, vpbreaks);