tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

unified retrying for databaseModified errors

Browse Source
This commit is contained in:
dockes 2009-10-24 15:02:49 +00:00
parent 0bd1b1a674
commit 709be40f5c
4 changed files with 250 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
src/rcldb/rcldb.cpp
View File

@ -108,25 +108,20 @@ static inline string make_parentterm(const string& udi)
bool Db::Native::subDocs(const string &udi, vector<Xapian::docid>& docids) bool Db::Native::subDocs(const string &udi, vector<Xapian::docid>& docids)
{ {
LOGDEB2(("subDocs: [%s]\n", uniterm.c_str())); LOGDEB2(("subDocs: [%s]\n", uniterm.c_str()));
string ermsg;
string pterm = make_parentterm(udi); string pterm = make_parentterm(udi);
for (int tries = 0; tries < 2; tries++) {
try { XAPTRY(docids.clear();
Xapian::PostingIterator it = xrdb.postlist_begin(pterm); docids.insert(docids.begin(), xrdb.postlist_begin(pterm),
for (; it != xrdb.postlist_end(pterm); it++) { xrdb.postlist_end(pterm)),
docids.push_back(*it); xrdb, m_rcldb->m_reason);
}
LOGDEB(("Db::Native::subDocs: returning %d ids\n", docids.size())); if (!m_rcldb->m_reason.empty()) {
return true; LOGERR(("Rcl::Db::subDocs: %s\n", m_rcldb->m_reason.c_str()));
} catch (const Xapian::DatabaseModifiedError &e) { return false;
LOGDEB(("Db::subDocs: got modified error. reopen/retry\n")); } else {
xrdb.reopen(); LOGDEB(("Db::Native::subDocs: returning %d ids\n", docids.size()));
} XCATCHERROR(ermsg); return true;
if (!ermsg.empty())
break;
} }
LOGERR(("Rcl::Db::subDocs: %s\n", ermsg.c_str()));
return false;
} }
// Only ONE field name inside the index data record differs from the // Only ONE field name inside the index data record differs from the
@ -206,6 +201,9 @@ static list<string> noPrefixList(const list<string>& in)
// Build a document abstract by extracting text chunks around the query terms // Build a document abstract by extracting text chunks around the query terms
// This uses the db termlists, not the original document. // This uses the db termlists, not the original document.
//
// DatabaseModified and other general exceptions are catched and
// possibly retried by our caller
string Db::Native::makeAbstract(Xapian::docid docid, Query *query) string Db::Native::makeAbstract(Xapian::docid docid, Query *query)
{ {
Chrono chron; Chrono chron;
@ -655,20 +653,14 @@ bool Db::adjustdbs()
int Db::docCnt() int Db::docCnt()
{ {
int res = -1; int res = -1;
string ermsg; if (!m_ndb || !m_ndb->m_isopen)
if (m_ndb && m_ndb->m_isopen) { return -1;
try {
res = m_ndb->xdb().get_doccount(); XAPTRY(res = m_ndb->xdb().get_doccount(), m_ndb->xrdb, m_reason);
} catch (const Xapian::DatabaseModifiedError &e) {
LOGDEB(("Db::docCnt: got modified error. reopen/retry\n")); if (!m_reason.empty()) {
// Doesn't make sense if we are the writer ! LOGERR(("Db::docCnt: got error: %s\n", m_reason.c_str()));
if (m_ndb->m_iswritable) return -1;
return -1;
m_ndb->xdb().reopen();
res = m_ndb->xdb().get_doccount();
} XCATCHERROR(ermsg);
if (!ermsg.empty())
LOGERR(("Db::docCnt: got error: %s\n", ermsg.c_str()));
} }
return res; return res;
} }
@ -1125,15 +1117,14 @@ bool Db::needUpdate(const string &udi, const string& sig)
// the actual document file, or, for a multi-document file, the // the actual document file, or, for a multi-document file, the
// pseudo-doc we create to stand for the file itself. // pseudo-doc we create to stand for the file itself.
// We try twice in case database needs to be reopened. (Ulterior // We try twice in case database needs to be reopened.
// note: this does not make sense as we are the sole writer!)
for (int tries = 0; tries < 2; tries++) { for (int tries = 0; tries < 2; tries++) {
try { try {
// Get the doc or pseudo-doc // Get the doc or pseudo-doc
Xapian::PostingIterator docid =m_ndb->xrdb.postlist_begin(uniterm); Xapian::PostingIterator docid =m_ndb->xrdb.postlist_begin(uniterm);
if (docid == m_ndb->xrdb.postlist_end(uniterm)) { if (docid == m_ndb->xrdb.postlist_end(uniterm)) {
// If no document exist with this path, we do need update // If no document exist with this path, we do need update
LOGDEB(("Db::needUpdate: no path: [%s]\n", uniterm.c_str())); LOGDEB(("Db::needUpdate:yes (new): [%s]\n", uniterm.c_str()));
return true; return true;
} }
Xapian::Document doc = m_ndb->xrdb.get_document(*docid); Xapian::Document doc = m_ndb->xrdb.get_document(*docid);
@ -1148,9 +1139,9 @@ bool Db::needUpdate(const string &udi, const string& sig)
osig.c_str(), sig.c_str())); osig.c_str(), sig.c_str()));
// Db is not up to date. Let's index the file // Db is not up to date. Let's index the file
return true; return true;
} }
LOGDEB(("Db::needUpdate: uptodate: [%s]\n", uniterm.c_str())); LOGDEB(("Db::needUpdate:no: [%s]\n", uniterm.c_str()));
// Up to date. // Up to date.
@ -1173,13 +1164,14 @@ bool Db::needUpdate(const string &udi, const string& sig)
return false; return false;
} catch (const Xapian::DatabaseModifiedError &e) { } catch (const Xapian::DatabaseModifiedError &e) {
LOGDEB(("Db::needUpdate: got modified error. reopen/retry\n")); LOGDEB(("Db::needUpdate: got modified error. reopen/retry\n"));
m_ndb->xdb().reopen(); m_reason = e.get_msg();
} XCATCHERROR(ermsg); m_ndb->xrdb.reopen();
if (!ermsg.empty()) continue;
break; } XCATCHERROR(m_reason);
break;
} }
LOGERR(("Db::needUpdate: error while checking existence: %s\n", LOGERR(("Db::needUpdate: error while checking existence: %s\n",
ermsg.c_str())); m_reason.c_str()));
return true; return true;
} }
@ -1393,7 +1385,7 @@ bool Db::termMatch(MatchType typ, const string &lang,
{ {
if (!m_ndb || !m_ndb->m_isopen) if (!m_ndb || !m_ndb->m_isopen)
return false; return false;
Xapian::Database db = m_ndb->xdb(); Xapian::Database xdb = m_ndb->xdb();
res.clear(); res.clear();
@ -1417,7 +1409,10 @@ bool Db::termMatch(MatchType typ, const string &lang,
res.unique(); res.unique();
for (list<TermMatchEntry>::iterator it = res.begin(); for (list<TermMatchEntry>::iterator it = res.begin();
it != res.end(); it++) { it != res.end(); it++) {
it->wcf = db.get_collection_freq(it->term); XAPTRY(it->wcf = xdb.get_collection_freq(it->term),
xdb, m_reason);
if (!m_reason.empty())
return false;
LOGDEB1(("termMatch: %d [%s]\n", it->wcf, it->term.c_str())); LOGDEB1(("termMatch: %d [%s]\n", it->wcf, it->term.c_str()));
} }
} else { } else {
@ -1446,34 +1441,44 @@ bool Db::termMatch(MatchType typ, const string &lang,
} }
LOGDEB(("termMatch: initsec: [%s]\n", is.c_str())); LOGDEB(("termMatch: initsec: [%s]\n", is.c_str()));
string ermsg; for (int tries = 0; tries < 2; tries++) {
try { try {
Xapian::TermIterator it = db.allterms_begin(); Xapian::TermIterator it = xdb.allterms_begin();
if (!is.empty()) if (!is.empty())
it.skip_to(is.c_str()); it.skip_to(is.c_str());
for (int n = 0; it != db.allterms_end(); it++) { for (int n = 0; it != xdb.allterms_end(); it++) {
// If we're beyond the terms matching the initial string, end // If we're beyond the terms matching the initial
if (!is.empty() && (*it).find(is) != 0) // string, end
break; if (!is.empty() && (*it).find(is) != 0)
string term; break;
if (!prefix.empty()) string term;
term = (*it).substr(prefix.length()); if (!prefix.empty())
else term = (*it).substr(prefix.length());
term = *it; else
if (typ == ET_WILD) { term = *it;
if (fnmatch(droot.c_str(), term.c_str(), 0) == FNM_NOMATCH) if (typ == ET_WILD) {
continue; if (fnmatch(droot.c_str(), term.c_str(), 0) ==
} else { FNM_NOMATCH)
if (regexec(&reg, term.c_str(), 0, 0, 0)) continue;
continue; } else {
} if (regexec(&reg, term.c_str(), 0, 0, 0))
// Do we want stem expansion here? We don't do it for now continue;
res.push_back(TermMatchEntry(term, it.get_termfreq())); }
++n; // Do we want stem expansion here? We don't do it for now
} res.push_back(TermMatchEntry(term, it.get_termfreq()));
} XCATCHERROR(ermsg); ++n;
if (!ermsg.empty()) { }
LOGERR(("termMatch: %s\n", ermsg.c_str())); m_reason.erase();
break;
} catch (const Xapian::DatabaseModifiedError &e) {
m_reason = e.get_msg();
xdb.reopen();
continue;
} XCATCHERROR(m_reason);
break;
}
if (!m_reason.empty()) {
LOGERR(("termMatch: %s\n", m_reason.c_str()));
return false; return false;
} }
@ -1508,12 +1513,9 @@ TermIter *Db::termWalkOpen()
TermIter *tit = new TermIter; TermIter *tit = new TermIter;
if (tit) { if (tit) {
tit->db = m_ndb->xdb(); tit->db = m_ndb->xdb();
string ermsg; XAPTRY(tit->it = tit->db.allterms_begin(), tit->db, m_reason);
try { if (!m_reason.empty()) {
tit->it = tit->db.allterms_begin(); LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", ermsg.c_str()));
return 0; return 0;
} }
} }
@ -1521,15 +1523,15 @@ TermIter *Db::termWalkOpen()
} }
bool Db::termWalkNext(TermIter *tit, string &term) bool Db::termWalkNext(TermIter *tit, string &term)
{ {
string ermsg; XAPTRY(
try {
if (tit && tit->it != tit->db.allterms_end()) { if (tit && tit->it != tit->db.allterms_end()) {
term = *(tit->it)++; term = *(tit->it)++;
return true; return true;
} }
} XCATCHERROR(ermsg); , tit->db, m_reason);
if (!ermsg.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", ermsg.c_str())); if (!m_reason.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
} }
return false; return false;
} }
@ -1544,13 +1546,12 @@ bool Db::termExists(const string& word)
{ {
if (!m_ndb || !m_ndb->m_isopen) if (!m_ndb || !m_ndb->m_isopen)
return 0; return 0;
string ermsg;
try { XAPTRY(if (!m_ndb->xdb().term_exists(word)) return false,
if (!m_ndb->xdb().term_exists(word)) m_ndb->xrdb, m_reason);
return false;
} XCATCHERROR(ermsg); if (!m_reason.empty()) {
if (!ermsg.empty()) { LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
LOGERR(("Db::termWalkOpen: xapian error: %s\n", ermsg.c_str()));
return false; return false;
} }
return true; return true;
@ -1573,27 +1574,14 @@ bool Db::stemDiffers(const string& lang, const string& word,
bool Db::makeDocAbstract(Doc &doc, Query *query, string& abstract) bool Db::makeDocAbstract(Doc &doc, Query *query, string& abstract)
{ {
LOGDEB1(("Db::makeDocAbstract: exti %d\n", exti)); LOGDEB1(("Db::makeDocAbstract: exti %d\n", exti));
if (!m_ndb) { if (!m_ndb || !m_ndb->m_isopen) {
LOGERR(("Db::makeDocAbstract: no db\n")); LOGERR(("Db::makeDocAbstract: no db\n"));
return false; return false;
} }
m_reason.erase();
for (int i = 0; i < 2; i++) { XAPTRY(abstract = m_ndb->makeAbstract(doc.xdocid, query),
try { m_ndb->xrdb, m_reason);
abstract = m_ndb->makeAbstract(doc.xdocid, query);
m_reason.erase();
break;
} catch (const Xapian::DatabaseModifiedError &error) {
LOGDEB(("Db:makeDocAbstract: caught DatabaseModified\n"));
m_reason = error.get_msg();
m_ndb->xdb().reopen();
} catch (const Xapian::Error & error) {
LOGERR(("Db::makeDocAbstract: exception: %s\n",
error.get_msg().c_str()));
m_reason = error.get_msg();
break;
}
}
return m_reason.empty() ? true : false; return m_reason.empty() ? true : false;
} }
@ -1609,25 +1597,32 @@ bool Db::getDoc(const string &udi, Doc &doc)
doc.pc = 100; doc.pc = 100;
string uniterm = make_uniterm(udi); string uniterm = make_uniterm(udi);
string ermsg; for (int tries = 0; tries < 2; tries++) {
try { try {
if (!m_ndb->xrdb.term_exists(uniterm)) { if (!m_ndb->xrdb.term_exists(uniterm)) {
// Document found in history no longer in the database. // Document found in history no longer in the
// We return true (because their might be other ok docs further) // database. We return true (because their might be
// but indicate the error with pc = -1 // other ok docs further) but indicate the error with
doc.pc = -1; // pc = -1
LOGINFO(("Db:getDoc: no such doc in index: [%s] (len %d)\n", doc.pc = -1;
uniterm.c_str(), uniterm.length())); LOGINFO(("Db:getDoc: no such doc in index: [%s] (len %d)\n",
return true; uniterm.c_str(), uniterm.length()));
} return true;
Xapian::PostingIterator docid = m_ndb->xrdb.postlist_begin(uniterm); }
Xapian::Document xdoc = m_ndb->xrdb.get_document(*docid); Xapian::PostingIterator docid =
string data = xdoc.get_data(); m_ndb->xrdb.postlist_begin(uniterm);
return m_ndb->dbDataToRclDoc(*docid, data, doc, 100); Xapian::Document xdoc = m_ndb->xrdb.get_document(*docid);
} XCATCHERROR(ermsg); string data = xdoc.get_data();
if (!ermsg.empty()) { return m_ndb->dbDataToRclDoc(*docid, data, doc, 100);
LOGERR(("Db::getDoc: %s\n", ermsg.c_str())); } catch (const Xapian::DatabaseModifiedError &e) {
m_reason = e.get_msg();
m_ndb->xrdb.reopen();
continue;
} XCATCHERROR(m_reason);
break;
} }
LOGERR(("Db::getDoc: %s\n", m_reason.c_str()));
return false; return false;
} }

14
src/rcldb/rcldb_p.h
View File

@ -31,6 +31,20 @@ enum value_slot {
MSG = "Caught unknown xapian exception"; \ MSG = "Caught unknown xapian exception"; \
} }
#define XAPTRY(STMTTOTRY, XAPDB, ERSTR) \
for (int tries = 0; tries < 2; tries++) { \
try { \
STMTTOTRY; \
ERSTR.erase(); \
break; \
} catch (const Xapian::DatabaseModifiedError &e) { \
ERSTR = e.get_msg(); \
XAPDB.reopen(); \
continue; \
} XCATCHERROR(ERSTR); \
break; \
}
class Query; class Query;
// A class for data and methods that would have to expose // A class for data and methods that would have to expose

214
src/rcldb/rclquery.cpp
View File

@ -151,34 +151,44 @@ bool Query::setQuery(RefCntr<SearchData> sdata)
m_reason += sdata->getReason(); m_reason += sdata->getReason();
return false; return false;
} }
m_nq->query = xq; m_nq->xquery = xq;
string ermsg;
string d; string d;
try { for (int tries = 0; tries < 2; tries++) {
m_nq->enquire = new Xapian::Enquire(m_db->m_ndb->xrdb); try {
if (m_collapseDuplicates) { m_nq->xenquire = new Xapian::Enquire(m_db->m_ndb->xrdb);
m_nq->enquire->set_collapse_key(Rcl::VALUE_MD5); if (m_collapseDuplicates) {
} else { m_nq->xenquire->set_collapse_key(Rcl::VALUE_MD5);
m_nq->enquire->set_collapse_key(Xapian::BAD_VALUENO); } else {
} m_nq->xenquire->set_collapse_key(Xapian::BAD_VALUENO);
if (!m_sortField.empty()) { }
if (m_sorter) { if (!m_sortField.empty()) {
delete (QSorter*)m_sorter; if (m_sorter) {
m_sorter = 0; delete (QSorter*)m_sorter;
} m_sorter = 0;
m_sorter = new QSorter(m_sortField); }
// It really seems there is a xapian bug about sort order, we m_sorter = new QSorter(m_sortField);
// invert here. // It really seems there is a xapian bug about sort order, we
m_nq->enquire->set_sort_by_key((QSorter*)m_sorter, // invert here.
!m_sortAscending); m_nq->xenquire->set_sort_by_key((QSorter*)m_sorter,
} !m_sortAscending);
m_nq->enquire->set_query(m_nq->query); }
m_nq->mset = Xapian::MSet(); m_nq->xenquire->set_query(m_nq->xquery);
// Get the query description and trim the "Xapian::Query" m_nq->xmset = Xapian::MSet();
d = m_nq->query.get_description(); // Get the query description and trim the "Xapian::Query"
} XCATCHERROR(ermsg); d = m_nq->xquery.get_description();
if (!ermsg.empty()) { m_reason.erase();
LOGDEB(("Query::SetQuery: xapian error %s\n", ermsg.c_str())); break;
} catch (const Xapian::DatabaseModifiedError &e) {
m_reason = e.get_msg();
m_db->m_ndb->xrdb.reopen();
continue;
} XCATCHERROR(m_reason);
break;
}
if (!m_reason.empty()) {
LOGDEB(("Query::SetQuery: xapian error %s\n", m_reason.c_str()));
return false; return false;
} }
@ -202,8 +212,8 @@ bool Query::getQueryTerms(list<string>& terms)
Xapian::TermIterator it; Xapian::TermIterator it;
string ermsg; string ermsg;
try { try {
for (it = m_nq->query.get_terms_begin(); for (it = m_nq->xquery.get_terms_begin();
it != m_nq->query.get_terms_end(); it++) { it != m_nq->xquery.get_terms_end(); it++) {
terms.push_back(*it); terms.push_back(*it);
} }
} XCATCHERROR(ermsg); } XCATCHERROR(ermsg);
@ -216,7 +226,7 @@ bool Query::getQueryTerms(list<string>& terms)
bool Query::getMatchTerms(const Doc& doc, list<string>& terms) bool Query::getMatchTerms(const Doc& doc, list<string>& terms)
{ {
if (ISNULL(m_nq) || !m_nq->enquire) { if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::getMatchTerms: no query opened\n")); LOGERR(("Query::getMatchTerms: no query opened\n"));
return -1; return -1;
} }
@ -224,15 +234,14 @@ bool Query::getMatchTerms(const Doc& doc, list<string>& terms)
terms.clear(); terms.clear();
Xapian::TermIterator it; Xapian::TermIterator it;
Xapian::docid id = Xapian::docid(doc.xdocid); Xapian::docid id = Xapian::docid(doc.xdocid);
string ermsg;
try { XAPTRY(terms.insert(terms.begin(),
for (it=m_nq->enquire->get_matching_terms_begin(id); m_nq->xenquire->get_matching_terms_begin(id),
it != m_nq->enquire->get_matching_terms_end(id); it++) { m_nq->xenquire->get_matching_terms_end(id)),
terms.push_back(*it); m_db->m_ndb->xrdb, m_reason);
}
} XCATCHERROR(ermsg); if (!m_reason.empty()) {
if (!ermsg.empty()) { LOGERR(("getQueryTerms: xapian error: %s\n", m_reason.c_str()));
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
return false; return false;
} }
@ -246,25 +255,23 @@ static const int qquantum = 30;
// the search job in there, this can be long // the search job in there, this can be long
int Query::getResCnt() int Query::getResCnt()
{ {
if (ISNULL(m_nq) || !m_nq->enquire) { if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::getResCnt: no query opened\n")); LOGERR(("Query::getResCnt: no query opened\n"));
return -1; return -1;
} }
string ermsg;
int ret = -1; int ret = -1;
if (m_nq->mset.size() <= 0) { if (m_nq->xmset.size() <= 0) {
Chrono chron; Chrono chron;
try {
m_nq->mset = m_nq->enquire->get_mset(0, qquantum,0, m_nq->decider); XAPTRY(m_nq->xmset =
ret = m_nq->mset.get_matches_lower_bound(); m_nq->xenquire->get_mset(0, qquantum,0, m_nq->decider);
} catch (const Xapian::DatabaseModifiedError &error) { ret = m_nq->xmset.get_matches_lower_bound(),
m_db->m_ndb->xrdb.reopen(); m_db->m_ndb->xrdb, m_reason);
m_nq->mset = m_nq->enquire->get_mset(0, qquantum,0, m_nq->decider);
ret = m_nq->mset.get_matches_lower_bound();
} XCATCHERROR(ermsg);
LOGDEB(("Query::getResCnt: %d mS\n", chron.millis())); LOGDEB(("Query::getResCnt: %d mS\n", chron.millis()));
if (!ermsg.empty()) if (!m_reason.empty())
LOGERR(("enquire->get_mset: exception: %s\n", ermsg.c_str())); LOGERR(("xenquire->get_mset: exception: %s\n", m_reason.c_str()));
} }
return ret; return ret;
} }
@ -282,7 +289,7 @@ int Query::getResCnt()
bool Query::getDoc(int exti, Doc &doc) bool Query::getDoc(int exti, Doc &doc)
{ {
LOGDEB1(("Query::getDoc: exti %d\n", exti)); LOGDEB1(("Query::getDoc: exti %d\n", exti));
if (ISNULL(m_nq) || !m_nq->enquire) { if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::getDoc: no query opened\n")); LOGERR(("Query::getDoc: no query opened\n"));
return false; return false;
} }
@ -301,31 +308,29 @@ bool Query::getDoc(int exti, Doc &doc)
while (exti >= (int)m_nq->m_dbindices.size()) { while (exti >= (int)m_nq->m_dbindices.size()) {
LOGDEB(("Query::getDoc: fetching %d starting at %d\n", LOGDEB(("Query::getDoc: fetching %d starting at %d\n",
qquantum, first)); qquantum, first));
try {
m_nq->mset = m_nq->enquire->get_mset(first, qquantum); XAPTRY(m_nq->xmset = m_nq->xenquire->get_mset(first, qquantum),
} catch (const Xapian::DatabaseModifiedError &error) { m_db->m_ndb->xrdb, m_reason);
m_db->m_ndb->xrdb.reopen();
m_nq->mset = m_nq->enquire->get_mset(first, qquantum); if (!m_reason.empty()) {
} catch (const Xapian::Error & error) { LOGERR(("enquire->get_mset: exception: %s\n",
LOGERR(("enquire->get_mset: exception: %s\n", m_reason.c_str()));
error.get_msg().c_str())); return false;
m_reason = error.get_msg();
return false;
} }
if (m_nq->mset.empty()) { if (m_nq->xmset.empty()) {
LOGDEB(("Query::getDoc: got empty mset\n")); LOGDEB(("Query::getDoc: got empty mset\n"));
return false; return false;
} }
first = m_nq->mset.get_firstitem(); first = m_nq->xmset.get_firstitem();
for (unsigned int i = 0; i < m_nq->mset.size() ; i++) { for (unsigned int i = 0; i < m_nq->xmset.size() ; i++) {
LOGDEB(("Query::getDoc: [%d]\n", i)); LOGDEB(("Query::getDoc: [%d]\n", i));
Xapian::Document xdoc = m_nq->mset[i].get_document(); Xapian::Document xdoc = m_nq->xmset[i].get_document();
if ((*m_nq->postfilter)(xdoc)) { if ((*m_nq->postfilter)(xdoc)) {
m_nq->m_dbindices.push_back(first + i); m_nq->m_dbindices.push_back(first + i);
} }
} }
first = first + m_nq->mset.size(); first = first + m_nq->xmset.size();
} }
} }
xapi = m_nq->m_dbindices[exti]; xapi = m_nq->m_dbindices[exti];
@ -334,34 +339,31 @@ bool Query::getDoc(int exti, Doc &doc)
} }
// From there on, we work with a xapian enquire item number. Fetch it // From there on, we work with a xapian enquire item number. Fetch it
int first = m_nq->mset.get_firstitem(); int first = m_nq->xmset.get_firstitem();
int last = first + m_nq->mset.size() -1; int last = first + m_nq->xmset.size() -1;
if (!(xapi >= first && xapi <= last)) { if (!(xapi >= first && xapi <= last)) {
LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum)); LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));
try {
m_nq->mset = m_nq->enquire->get_mset(xapi, qquantum,
0, m_nq->decider);
} catch (const Xapian::DatabaseModifiedError &error) {
m_db->m_ndb->xrdb.reopen();
m_nq->mset = m_nq->enquire->get_mset(xapi, qquantum,
0, m_nq->decider);
} catch (const Xapian::Error & error) { XAPTRY(m_nq->xmset = m_nq->xenquire->get_mset(xapi, qquantum,
LOGERR(("enquire->get_mset: exception: %s\n", 0, m_nq->decider),
error.get_msg().c_str())); m_db->m_ndb->xrdb, m_reason);
m_reason = error.get_msg();
return false; if (!m_reason.empty()) {
LOGERR(("enquire->get_mset: exception: %s\n", m_reason.c_str()));
return false;
} }
if (m_nq->mset.empty()) if (m_nq->xmset.empty()) {
LOGDEB(("enquire->get_mset: got empty result\n"));
return false; return false;
first = m_nq->mset.get_firstitem(); }
last = first + m_nq->mset.size() -1; first = m_nq->xmset.get_firstitem();
last = first + m_nq->xmset.size() -1;
} }
LOGDEB1(("Query::getDoc: Qry [%s] win [%d-%d] Estimated results: %d", LOGDEB1(("Query::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",
m_nq->query.get_description().c_str(), m_nq->query.get_description().c_str(),
first, last, m_nq->mset.get_matches_lower_bound())); first, last, m_nq->xmset.get_matches_lower_bound()));
Xapian::Document xdoc; Xapian::Document xdoc;
Xapian::docid docid = 0; Xapian::docid docid = 0;
@ -370,41 +372,42 @@ bool Query::getDoc(int exti, Doc &doc)
m_reason.erase(); m_reason.erase();
for (int xaptries=0; xaptries < 2; xaptries++) { for (int xaptries=0; xaptries < 2; xaptries++) {
try { try {
xdoc = m_nq->mset[xapi-first].get_document(); xdoc = m_nq->xmset[xapi-first].get_document();
docid = *(m_nq->mset[xapi-first]); docid = *(m_nq->xmset[xapi-first]);
pc = m_nq->mset.convert_to_percent(m_nq->mset[xapi-first]); pc = m_nq->xmset.convert_to_percent(m_nq->xmset[xapi-first]);
data = xdoc.get_data(); data = xdoc.get_data();
m_reason.erase(); m_reason.erase();
break; break;
} catch (Xapian::DatabaseModifiedError &error) { } catch (Xapian::DatabaseModifiedError &error) {
// retry or end of loop // retry or end of loop
LOGDEB(("getDoc: caught DatabaseModified\n"));
m_reason = error.get_msg(); m_reason = error.get_msg();
continue; continue;
} }
XCATCHERROR(m_reason); XCATCHERROR(m_reason);
break; break;
} }
if (!m_reason.empty()) {
LOGERR(("Query::getDoc: %s\n", m_reason.c_str()));
return false;
}
// Parse xapian document's data and populate doc fields // Parse xapian document's data and populate doc fields
return m_reason.empty() ? return m_db->m_ndb->dbDataToRclDoc(docid, data, doc, pc);
m_db->m_ndb->dbDataToRclDoc(docid, data, doc, pc) : false;
} }
list<string> Query::expand(const Doc &doc) list<string> Query::expand(const Doc &doc)
{ {
list<string> res; list<string> res;
if (ISNULL(m_nq) || !m_nq->enquire) { if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::expand: no query opened\n")); LOGERR(("Query::expand: no query opened\n"));
return res; return res;
} }
string ermsg;
for (int tries = 0; tries < 2; tries++) { for (int tries = 0; tries < 2; tries++) {
try { try {
Xapian::RSet rset; Xapian::RSet rset;
rset.add_document(Xapian::docid(doc.xdocid)); rset.add_document(Xapian::docid(doc.xdocid));
// We don't exclude the original query terms. // We don't exclude the original query terms.
Xapian::ESet eset = m_nq->enquire->get_eset(20, rset, false); Xapian::ESet eset = m_nq->xenquire->get_eset(20, rset, false);
LOGDEB(("ESet terms:\n")); LOGDEB(("ESet terms:\n"));
// We filter out the special terms // We filter out the special terms
for (Xapian::ESetIterator it = eset.begin(); for (Xapian::ESetIterator it = eset.begin();
@ -416,16 +419,21 @@ list<string> Query::expand(const Doc &doc)
if (res.size() >= 10) if (res.size() >= 10)
break; break;
} }
} catch (const Xapian::DatabaseModifiedError &error) { m_reason.erase();
continue; break;
} XCATCHERROR(ermsg); } catch (const Xapian::DatabaseModifiedError &e) {
if (!ermsg.empty()) { m_reason = e.get_msg();
LOGERR(("Query::expand: xapian error %s\n", ermsg.c_str())); m_db->m_ndb->xrdb.reopen();
res.clear(); continue;
} } XCATCHERROR(m_reason);
break; break;
} }
if (!m_reason.empty()) {
LOGERR(("Query::expand: xapian error %s\n", m_reason.c_str()));
res.clear();
}
return res; return res;
} }

10
src/rcldb/rclquery_p.h
View File

@ -20,7 +20,7 @@ public:
/** query descriptor: terms and subqueries joined by operators /** query descriptor: terms and subqueries joined by operators
* (or/and etc...) * (or/and etc...)
*/ */
Xapian::Query query; Xapian::Query xquery;
/** In case there is a postq filter: sequence of db indices that match */ /** In case there is a postq filter: sequence of db indices that match */
vector<int> m_dbindices; vector<int> m_dbindices;
@ -47,13 +47,13 @@ public:
Xapian::MatchDecider *decider; // Xapian does the filtering Xapian::MatchDecider *decider; // Xapian does the filtering
Xapian::MatchDecider *postfilter; // Result filtering done by Recoll Xapian::MatchDecider *postfilter; // Result filtering done by Recoll
Xapian::Enquire *enquire; // Open query descriptor. Xapian::Enquire *xenquire; // Open query descriptor.
Xapian::MSet mset; // Partial result set Xapian::MSet xmset; // Partial result set
// Term frequencies for current query. See makeAbstract, setQuery // Term frequencies for current query. See makeAbstract, setQuery
map<string, double> termfreqs; map<string, double> termfreqs;
Native(Query *q) Native(Query *q)
: m_q(q), decider(0), postfilter(0), enquire(0) : m_q(q), decider(0), postfilter(0), xenquire(0)
{ } { }
~Native() { ~Native() {
clear(); clear();
@ -62,7 +62,7 @@ public:
m_dbindices.clear(); m_dbindices.clear();
delete decider; decider = 0; delete decider; decider = 0;
delete postfilter; postfilter = 0; delete postfilter; postfilter = 0;
delete enquire; enquire = 0; delete xenquire; xenquire = 0;
termfreqs.clear(); termfreqs.clear();
} }
}; };