tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Avoid purging documents from absent mountable volumes

Browse Source
This commit is contained in:
Jean-Francois Dockes
2019-02-03 18:51:52 +01:00
parent c2691f68bf
commit 399c633efd
5 changed files with 197 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+40 -22
View File
@@ -46,7 +46,6 @@ using namespace std;
#include "rclutil.h"
#include "smallut.h"
#include "chrono.h"
#include "utf8iter.h"
#include "searchdata.h"
#include "rclquery.h"
#include "rclquery_p.h"
@@ -144,21 +143,6 @@ static inline string make_parentterm(const string& udi)
return pterm;
}
static void utf8truncate(string& s, int maxlen)
{
if (s.size() <= string::size_type(maxlen)) {
return;
}
Utf8Iter iter(s);
string::size_type pos = 0;
while (iter++ != string::npos)
if (iter.getBpos() < string::size_type(maxlen)) {
pos = iter.getBpos();
}
s.erase(pos);
}
Db::Native::Native(Db *db)
: m_rcldb(db), m_isopen(false), m_iswritable(false),
m_noversionwrite(false)
@@ -2023,11 +2007,10 @@ void Db::i_setExistingFlags(const string& udi, unsigned int docid)
LOGERR("Rcl::Db::needUpdate: can't get subdocs\n");
return;
}
for (vector<Xapian::docid>::iterator it = docids.begin();
it != docids.end(); it++) {
if (*it < updated.size()) {
LOGDEB2("Db::needUpdate: docid " << (*it) << " set\n");
updated[*it] = true;
for (auto docid : docids) {
if (docid < updated.size()) {
LOGDEB2("Db::needUpdate: docid " << docid << " set\n");
updated[docid] = true;
}
}
}
@@ -2556,5 +2539,40 @@ bool Db::getSubDocs(const Doc &idoc, vector<Doc>& subdocs)
return false;
}
} // End namespace Rcl
// Walk an UDI section (all UDIs beginning with input prefix), and
// mark all docs and subdocs as existing. Caller beware: Makes sense
// or not depending on the UDI structure for the data store. In practise,
// used for absent FS mountable volumes.
bool Db::udiTreeMarkExisting(const string& udi)
{
LOGDEB("Db::udiTreeWalk: " << udi << endl);
string wrapd = wrap_prefix(udi_prefix);
string expr = udi + "*";
#ifdef IDX_THREADS
std::unique_lock<std::mutex> lock(m_ndb->m_mutex);
#endif
bool ret = m_ndb->idxTermMatch_p(
int(ET_WILD), cstr_null, expr,
[this, &udi](const string& term, Xapian::termcount, Xapian::doccount) {
Xapian::PostingIterator docid;
XAPTRY(docid = m_ndb->xrdb.postlist_begin(term), m_ndb->xrdb,
m_reason);
if (!m_reason.empty()) {
LOGERR("Db::udiTreeWalk: xapian::postlist_begin failed: " <<
m_reason << "\n");
return false;
}
if (docid == m_ndb->xrdb.postlist_end(term)) {
LOGDEB("Db::udiTreeWalk:no doc for " << term << " ??\n");
return false;
}
i_setExistingFlags(udi, *docid);
LOGDEB("Db::udiTreeWalk: uniterm: " << term << endl);
return true;
}, wrapd);
return ret;
}
} // End namespace Rcl
src/rcldb/rcldb.h
+8 -1
View File
@@ -483,7 +483,14 @@ public:
// Use empty fn for no synonyms
bool setSynGroupsFile(const std::string& fn);
// Mark all documents with an UDI having input as prefix as
// existing. Only works if the UDIs for the store are
// hierarchical of course. Used by FsIndexer to avoid purging
// files for a topdir which is on a removable file system and
// currently unmounted (topdir does not exist or is empty.
bool udiTreeMarkExisting(const string& udi);
/* This has to be public for access by embedded Query::Native */
Native *m_ndb;
private:
src/rcldb/rcldb_p.h
+7
View File
@@ -181,6 +181,13 @@ class Db::Native {
*/
bool subDocs(const string &udi, int idxi, vector<Xapian::docid>& docids);
/** Matcher */
bool idxTermMatch_p(int typ_sens,const string &lang,const std::string &term,
std::function<bool(const std::string& term,
Xapian::termcount colfreq,
Xapian::doccount termfreq)> client,
const string& field);
/** Check if a page position list is defined */
bool hasPages(Xapian::docid id);
src/rcldb/rclterms.cpp
+61 -42
View File
@@ -330,34 +330,14 @@ bool Db::termMatch(int typ_sens, const string &lang, const string &_term,
return true;
}
// Second phase of wildcard/regexp term expansion after case/diac
// expansion: expand against main index terms
bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
TermMatchResult& res, int max, const string& field)
bool Db::Native::idxTermMatch_p(
int typ, const string &lang, const string &root,
std::function<bool(const string& term,
Xapian::termcount colfreq,
Xapian::doccount termfreq)> client,
const string& prefix)
{
int typ = matchTypeTp(typ_sens);
LOGDEB1("Db::idxTermMatch: typ " << tmtptostr(typ) << " lang [" <<
lang << "] term [" << root << "] max " << max << " field [" <<
field << "] init res.size " << res.entries.size() << "\n");
if (typ == ET_STEM) {
LOGFATAL("RCLDB: internal error: idxTermMatch called with ET_STEM\n");
abort();
}
Xapian::Database xdb = m_ndb->xrdb;
string prefix;
if (!field.empty()) {
const FieldTraits *ftp = 0;
if (!fieldToTraits(field, &ftp, true) || ftp->pfx.empty()) {
LOGDEB("Db::termMatch: field is not indexed (no prefix): [" <<
field << "]\n");
} else {
prefix = wrap_prefix(ftp->pfx);
}
}
res.prefix = prefix;
Xapian::Database xdb = xrdb;
std::shared_ptr<StrMatcher> matcher;
if (typ == ET_REGEXP) {
@@ -418,35 +398,74 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
if (matcher && !matcher->match(term))
continue;
res.entries.push_back(
TermMatchEntry(ixterm, xdb.get_collection_freq(ixterm),
it.get_termfreq()));
// The problem with truncating here is that this is done
// alphabetically and we may not keep the most frequent
// terms. OTOH, not doing it may stall the program if
// we are walking the whole term list. We compromise
// by cutting at 2*max
if (max > 0 && ++rcnt >= 2*max)
if (!client(ixterm, xdb.get_collection_freq(ixterm),
it.get_termfreq())) {
break;
}
}
m_reason.erase();
m_rcldb->m_reason.erase();
break;
} catch (const Xapian::DatabaseModifiedError &e) {
m_reason = e.get_msg();
m_rcldb->m_reason = e.get_msg();
xdb.reopen();
continue;
} XCATCHERROR(m_reason);
} XCATCHERROR(m_rcldb->m_reason);
break;
}
if (!m_reason.empty()) {
LOGERR("termMatch: " << m_reason << "\n");
if (!m_rcldb->m_reason.empty()) {
LOGERR("termMatch: " << m_rcldb->m_reason << "\n");
return false;
}
return true;
}
// Second phase of wildcard/regexp term expansion after case/diac
// expansion: expand against main index terms
bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
TermMatchResult& res, int max, const string& field)
{
int typ = matchTypeTp(typ_sens);
LOGDEB1("Db::idxTermMatch: typ " << tmtptostr(typ) << " lang [" <<
lang << "] term [" << root << "] max " << max << " field [" <<
field << "] init res.size " << res.entries.size() << "\n");
if (typ == ET_STEM) {
LOGFATAL("RCLDB: internal error: idxTermMatch called with ET_STEM\n");
abort();
}
string prefix;
if (!field.empty()) {
const FieldTraits *ftp = 0;
if (!fieldToTraits(field, &ftp, true) || ftp->pfx.empty()) {
LOGDEB("Db::termMatch: field is not indexed (no prefix): [" <<
field << "]\n");
} else {
prefix = wrap_prefix(ftp->pfx);
}
}
res.prefix = prefix;
int rcnt = 0;
bool ret = m_ndb->idxTermMatch_p(
typ, lang, root,
[&res, &rcnt, max](const string& term,
Xapian::termcount cf, Xapian::doccount tf) {
res.entries.push_back(TermMatchEntry(term, cf, tf));
// The problem with truncating here is that this is done
// alphabetically and we may not keep the most frequent
// terms. OTOH, not doing it may stall the program if
// we are walking the whole term list. We compromise
// by cutting at 2*max
if (max > 0 && ++rcnt >= 2*max)
return false;
return true;
}, prefix);
return ret;
}
/** Term list walking. */
class TermIter {
public: