tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

separated rcldb and rclquery

Browse Source
This commit is contained in:
dockes
2008-06-13 18:22:47 +00:00
parent e5e8249ad3
commit 0e7a78d688
20 changed files with 707 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+22 -430
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.132 2008-05-20 10:09:54 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.133 2008-06-13 18:22:46 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -36,6 +36,7 @@ using namespace std;
#include "rclconfig.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "stemdb.h"
#include "textsplit.h"
#include "transcode.h"
@@ -47,8 +48,9 @@ using namespace std;
#include "pathhash.h"
#include "utf8iter.h"
#include "searchdata.h"
#include "rclquery.h"
#include "rclquery_p.h"
#include "xapian.h"
#ifndef MAX
#define MAX(A,B) (A>B?A:B)
@@ -88,125 +90,8 @@ namespace Rcl {
const static string rclSyntAbs = "?!#@";
const static string emptystring;
// A class for data and methods that would have to expose
// Xapian-specific stuff if they were in Rcl::Db. There could actually be
// 2 different ones for indexing or query as there is not much in
// common.
class Native {
public:
Db *m_db;
bool m_isopen;
bool m_iswritable;
// Indexing
Xapian::WritableDatabase wdb;
// Querying
Xapian::Database db;
Xapian::Query query; // query descriptor: terms and subqueries
// joined by operators (or/and etc...)
// Filtering results on location. There are 2 possible approaches
// for this:
// - Set a "MatchDecider" to be used by Xapian during the query
// - Filter the results out of Xapian (this also uses a
// Xapian::MatchDecider object, but applied to the results by Recoll.
//
// The result filtering approach was the first implemented.
//
// The efficiency of both methods depend on the searches, so the code
// for both has been kept. A nice point for the Xapian approach is that
// the result count estimate are correct (they are wrong with
// the postfilter approach). It is also faster in some worst case scenarios
// so this now the default (but the post-filtering is faster in many common
// cases).
//
// Which is used is decided in SetQuery(), by setting either of
// the two following members. This in turn is controlled by a
// preprocessor directive.
#define XAPIAN_FILTERING 1
Xapian::MatchDecider *decider; // Xapian does the filtering
Xapian::MatchDecider *postfilter; // Result filtering done by Recoll
Xapian::Enquire *enquire; // Open query descriptor.
Xapian::MSet mset; // Partial result set
// Term frequencies for current query. See makeAbstract, setQuery
map<string, double> m_termfreqs;
Native(Db *db)
: m_db(db),
m_isopen(false), m_iswritable(false), decider(0), postfilter(0),
enquire(0)
{ }
~Native() {
delete decider;
delete postfilter;
delete enquire;
}
string makeAbstract(Xapian::docid id, const list<string>& terms);
bool dbDataToRclDoc(Xapian::docid docid, std::string &data, Doc &doc);
/** Compute list of subdocuments for a given path (given by hash)
* We look for all Q terms beginning with the path/hash
* As suggested by James Aylett, a better method would be to add
* a single term (ie: XP/path/to/file) to all subdocs, then finding
* them would be a simple matter of retrieving the posting list for the
* term. There would still be a need for the current Qterm though, as a
* unique term for replace_document, and for retrieving by
* path/ipath (history)
*/
bool subDocs(const string &hash, vector<Xapian::docid>& docids);
};
class FilterMatcher : public Xapian::MatchDecider {
public:
FilterMatcher(const string &topdir)
: m_topdir(topdir)
{}
virtual ~FilterMatcher() {}
virtual
#if XAPIAN_MAJOR_VERSION < 1
int
#else
bool
#endif
operator()(const Xapian::Document &xdoc) const
{
m_cnt++;
// Parse xapian document's data and populate doc fields
string data = xdoc.get_data();
ConfSimple parms(&data);
// The only filtering for now is on file path (subtree)
string url;
parms.get(string("url"), url);
LOGDEB2(("FilterMatcher topdir [%s] url [%s]\n",
m_topdir.c_str(), url.c_str()));
if (url.find(m_topdir, 7) == 7) {
LOGDEB2(("FilterMatcher: MATCH %d\n", m_cnt));
return true;
} else {
LOGDEB2(("FilterMatcher: NO MATCH %d\n", m_cnt));
return false;
}
}
static int m_cnt;
private:
string m_topdir;
};
int FilterMatcher::m_cnt;
/* See comment in class declaration */
bool Native::subDocs(const string &hash, vector<Xapian::docid>& docids)
bool Db::Native::subDocs(const string &hash, vector<Xapian::docid>& docids)
{
docids.clear();
string qterm = "Q"+ hash + "|";
@@ -250,7 +135,7 @@ bool Native::subDocs(const string &hash, vector<Xapian::docid>& docids)
}
// Turn data record from db into document fields
bool Native::dbDataToRclDoc(Xapian::docid docid, std::string &data, Doc &doc)
bool Db::Native::dbDataToRclDoc(Xapian::docid docid, std::string &data, Doc &doc)
{
LOGDEB1(("Db::dbDataToRclDoc: data: %s\n", data.c_str()));
ConfSimple parms(&data);
@@ -306,26 +191,29 @@ static list<string> noPrefixList(const list<string>& in)
// Build a document abstract by extracting text chunks around the query terms
// This uses the db termlists, not the original document.
string Native::makeAbstract(Xapian::docid docid, const list<string>& iterms)
string Db::Native::makeAbstract(Xapian::docid docid, Query *query)
{
Chrono chron;
LOGDEB(("makeAbstract:%d: maxlen %d wWidth %d\n", chron.ms(),
m_db->m_synthAbsLen, m_db->m_synthAbsWordCtxLen));
list<string> iterms;
query->getQueryTerms(iterms);
list<string> terms = noPrefixList(iterms);
if (terms.empty()) {
return "";
}
// Retrieve db-wide frequencies for the query terms
if (m_termfreqs.empty()) {
if (query->m_nq->termfreqs.empty()) {
double doccnt = db.get_doccount();
if (doccnt == 0) doccnt = 1;
for (list<string>::const_iterator qit = terms.begin();
qit != terms.end(); qit++) {
m_termfreqs[*qit] = db.get_termfreq(*qit) / doccnt;
query->m_nq->termfreqs[*qit] = db.get_termfreq(*qit) / doccnt;
LOGABS(("makeAbstract: [%s] db freq %.1e\n", qit->c_str(),
m_termfreqs[*qit]));
query->m_nq->termfreqs[*qit]));
}
LOGABS(("makeAbstract:%d: got termfreqs\n", chron.ms()));
}
@@ -343,7 +231,7 @@ string Native::makeAbstract(Xapian::docid docid, const list<string>& iterms)
Xapian::TermIterator term = db.termlist_begin(docid);
term.skip_to(*qit);
if (term != db.termlist_end(docid) && *term == *qit) {
double q = (term.get_wdf() / doclen) * m_termfreqs[*qit];
double q = (term.get_wdf() / doclen) * query->m_nq->termfreqs[*qit];
q = -log10(q);
if (q < 3) {
q = 0.05;
@@ -556,7 +444,7 @@ string Native::makeAbstract(Xapian::docid docid, const list<string>& iterms)
/* Rcl::Db methods ///////////////////////////////// */
Db::Db()
: m_ndb(0), m_qOpts(QO_NONE), m_idxAbsTruncLen(250), m_synthAbsLen(250),
: m_ndb(0), m_idxAbsTruncLen(250), m_synthAbsLen(250),
m_synthAbsWordCtxLen(4), m_flushMb(-1),
m_curtxtsz(0), m_flushtxtsz(0), m_occtxtsz(0),
m_maxFsOccupPc(0), m_mode(Db::DbRO)
@@ -586,28 +474,9 @@ Db::~Db()
return res;
}
// Generic Xapian exception catching code. We do this quite often,
// and I have no idea how to do this except for a macro
#define XCATCHERROR(MSG) \
catch (const Xapian::Error &e) { \
MSG = e.get_msg(); \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (const string &s) { \
MSG = s; \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (const char *s) { \
MSG = s; \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (...) { \
MSG = "Caught unknown xapian exception"; \
}
bool Db::open(const string& dir, const string &stops, OpenMode mode, int qops)
bool Db::open(const string& dir, const string &stops, OpenMode mode,
bool keep_updated)
{
bool keep_updated = (qops & QO_KEEP_UPDATED) != 0;
qops &= ~QO_KEEP_UPDATED;
if (m_ndb == 0)
return false;
LOGDEB(("Db::open: m_isopen %d m_iswritable %d\n", m_ndb->m_isopen,
@@ -724,7 +593,7 @@ bool Db::reOpen()
if (m_ndb && m_ndb->m_isopen) {
if (!close())
return false;
if (!open(m_basedir, "", m_mode, m_qOpts | QO_KEEP_UPDATED)) {
if (!open(m_basedir, "", m_mode, true)) {
return false;
}
}
@@ -1467,64 +1336,6 @@ bool Db::filenameWildExp(const string& fnexp, list<string>& names)
return true;
}
// Prepare query out of user search data
bool Db::setQuery(RefCntr<SearchData> sdata, int opts,
const string& stemlang)
{
if (!m_ndb) {
LOGERR(("Db::setQuery: no db!\n"));
return false;
}
m_reason.erase();
LOGDEB(("Db::setQuery:\n"));
m_filterTopDir = sdata->getTopdir();
deleteZ(m_ndb->decider);
deleteZ(m_ndb->postfilter);
if (!m_filterTopDir.empty()) {
#if XAPIAN_FILTERING
m_ndb->decider =
#else
m_ndb->postfilter =
#endif
new FilterMatcher(m_filterTopDir);
}
m_dbindices.clear();
m_qOpts = opts;
m_ndb->m_termfreqs.clear();
FilterMatcher::m_cnt = 0;
Xapian::Query xq;
if (!sdata->toNativeQuery(*this, &xq,
(opts & Db::QO_STEM) ? stemlang : "")) {
m_reason += sdata->getReason();
return false;
}
m_ndb->query = xq;
string ermsg;
string d;
try {
delete m_ndb->enquire;
m_ndb->enquire = new Xapian::Enquire(m_ndb->db);
m_ndb->enquire->set_query(m_ndb->query);
m_ndb->mset = Xapian::MSet();
// Get the query description and trim the "Xapian::Query"
d = m_ndb->query.get_description();
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGDEB(("Db::SetQuery: xapian error %s\n", ermsg.c_str()));
return false;
}
if (d.find("Xapian::Query") == 0)
d.erase(0, strlen("Xapian::Query"));
if (!m_filterTopDir.empty()) {
d += string(" [dir: ") + m_filterTopDir + "]";
}
sdata->setDescription(d);
LOGDEB(("Db::SetQuery: Q: %s\n", sdata->getDescription().c_str()));
return true;
}
class TermMatchCmpByWcf {
public:
int operator()(const TermMatchEntry& l, const TermMatchEntry& r) {
@@ -1735,195 +1546,15 @@ bool Db::stemDiffers(const string& lang, const string& word,
return true;
}
bool Db::getQueryTerms(list<string>& terms)
{
if (!m_ndb)
return false;
terms.clear();
Xapian::TermIterator it;
string ermsg;
try {
for (it = m_ndb->query.get_terms_begin();
it != m_ndb->query.get_terms_end(); it++) {
terms.push_back(*it);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
return false;
}
return true;
}
bool Db::getMatchTerms(const Doc& doc, list<string>& terms)
{
if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::getMatchTerms: no query opened\n"));
return -1;
}
terms.clear();
Xapian::TermIterator it;
Xapian::docid id = Xapian::docid(doc.xdocid);
string ermsg;
try {
for (it=m_ndb->enquire->get_matching_terms_begin(id);
it != m_ndb->enquire->get_matching_terms_end(id); it++) {
terms.push_back(*it);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
return false;
}
return true;
}
// Mset size
static const int qquantum = 30;
int Db::getResCnt()
{
if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::getResCnt: no query opened\n"));
return -1;
}
string ermsg;
if (m_ndb->mset.size() <= 0) {
try {
m_ndb->mset = m_ndb->enquire->get_mset(0, qquantum,
0, m_ndb->decider);
} catch (const Xapian::DatabaseModifiedError &error) {
m_ndb->db.reopen();
m_ndb->mset = m_ndb->enquire->get_mset(0, qquantum,
0, m_ndb->decider);
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("enquire->get_mset: exception: %s\n", ermsg.c_str()));
return -1;
}
}
int ret = -1;
try {
ret = m_ndb->mset.get_matches_lower_bound();
} catch (...) {}
return ret;
}
// Get document at rank i in query (i is the index in the whole result
// set, as in the enquire class. We check if the current mset has the
// doc, else ask for an other one. We use msets of 10 documents. Don't
// know if the whole thing makes sense at all but it seems to work.
//
// If there is a postquery filter (ie: file names), we have to
// maintain a correspondance from the sequential external index
// sequence to the internal Xapian hole-y one (the holes being the documents
// that dont match the filter).
bool Db::getDoc(int exti, Doc &doc, int *percent)
{
LOGDEB1(("Db::getDoc: exti %d\n", exti));
if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::getDoc: no query opened\n"));
return false;
}
int xapi;
if (m_ndb->postfilter) {
// There is a postquery filter, does this fall in already known area ?
if (exti >= (int)m_dbindices.size()) {
// Have to fetch xapian docs and filter until we get
// enough or fail
m_dbindices.reserve(exti+1);
// First xapian doc we fetch is the one after last stored
int first = m_dbindices.size() > 0 ? m_dbindices.back() + 1 : 0;
// Loop until we get enough docs
while (exti >= (int)m_dbindices.size()) {
LOGDEB(("Db::getDoc: fetching %d starting at %d\n",
qquantum, first));
try {
m_ndb->mset = m_ndb->enquire->get_mset(first, qquantum);
} catch (const Xapian::DatabaseModifiedError &error) {
m_ndb->db.reopen();
m_ndb->mset = m_ndb->enquire->get_mset(first, qquantum);
} catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str()));
abort();
}
if (m_ndb->mset.empty()) {
LOGDEB(("Db::getDoc: got empty mset\n"));
return false;
}
first = m_ndb->mset.get_firstitem();
for (unsigned int i = 0; i < m_ndb->mset.size() ; i++) {
LOGDEB(("Db::getDoc: [%d]\n", i));
Xapian::Document xdoc = m_ndb->mset[i].get_document();
if ((*m_ndb->postfilter)(xdoc)) {
m_dbindices.push_back(first + i);
}
}
first = first + m_ndb->mset.size();
}
}
xapi = m_dbindices[exti];
} else {
xapi = exti;
}
// From there on, we work with a xapian enquire item number. Fetch it
int first = m_ndb->mset.get_firstitem();
int last = first + m_ndb->mset.size() -1;
if (!(xapi >= first && xapi <= last)) {
LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));
try {
m_ndb->mset = m_ndb->enquire->get_mset(xapi, qquantum,
0, m_ndb->decider);
} catch (const Xapian::DatabaseModifiedError &error) {
m_ndb->db.reopen();
m_ndb->mset = m_ndb->enquire->get_mset(xapi, qquantum,
0, m_ndb->decider);
} catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str()));
abort();
}
if (m_ndb->mset.empty())
return false;
first = m_ndb->mset.get_firstitem();
last = first + m_ndb->mset.size() -1;
}
LOGDEB1(("Db::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",
m_ndb->query.get_description().c_str(),
first, last,
m_ndb->mset.get_matches_lower_bound()));
Xapian::Document xdoc = m_ndb->mset[xapi-first].get_document();
Xapian::docid docid = *(m_ndb->mset[xapi-first]);
if (percent)
*percent = m_ndb->mset.convert_to_percent(m_ndb->mset[xapi-first]);
// Parse xapian document's data and populate doc fields
string data = xdoc.get_data();
return m_ndb->dbDataToRclDoc(docid, data, doc);
}
bool Db::makeDocAbstract(Doc &doc, string& abstract)
bool Db::makeDocAbstract(Doc &doc, Query *query, string& abstract)
{
LOGDEB1(("Db::makeDocAbstract: exti %d\n", exti));
if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::makeDocAbstract: no query opened\n"));
if (!m_ndb) {
LOGERR(("Db::makeDocAbstract: no db\n"));
return false;
}
list<string> terms;
getQueryTerms(terms);
abstract = m_ndb->makeAbstract(doc.xdocid, terms);
abstract = m_ndb->makeAbstract(doc.xdocid, query);
return true;
}
@@ -1969,45 +1600,6 @@ bool Db::getDoc(const string &fn, const string &ipath, Doc &doc, int *pc)
return false;
}
list<string> Db::expand(const Doc &doc)
{
list<string> res;
if (!m_ndb || !m_ndb->enquire) {
LOGERR(("Db::expand: no query opened\n"));
return res;
}
string ermsg;
for (int tries = 0; tries < 2; tries++) {
try {
Xapian::RSet rset;
rset.add_document(Xapian::docid(doc.xdocid));
// We don't exclude the original query terms.
Xapian::ESet eset = m_ndb->enquire->get_eset(20, rset, false);
LOGDEB(("ESet terms:\n"));
// We filter out the special terms
for (Xapian::ESetIterator it = eset.begin();
it != eset.end(); it++) {
LOGDEB((" [%s]\n", (*it).c_str()));
if ((*it).empty() || ((*it).at(0)>='A' && (*it).at(0)<='Z'))
continue;
res.push_back(*it);
if (res.size() >= 10)
break;
}
} catch (const Xapian::DatabaseModifiedError &error) {
continue;
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Db::expand: xapian error %s\n", ermsg.c_str()));
res.clear();
}
break;
}
return res;
}
#ifndef NO_NAMESPACES
}
#endif
src/rcldb/rcldb.h
+17 -45
View File
@@ -16,7 +16,7 @@
*/
#ifndef _DB_H_INCLUDED_
#define _DB_H_INCLUDED_
/* @(#$Id: rcldb.h,v 1.54 2007-07-10 09:23:28 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: rcldb.h,v 1.55 2008-06-13 18:22:46 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
@@ -52,8 +52,8 @@ namespace Rcl {
#endif
class SearchData;
class Native;
class TermIter;
class Query;
class TermMatchEntry {
public:
@@ -71,17 +71,17 @@ public:
*/
class Db {
public:
// A place for things we don't want visible here.
class Native;
friend class Native;
/* General stuff (valid for query or update) ****************************/
Db();
~Db();
enum OpenMode {DbRO, DbUpd, DbTrunc};
// KEEP_UPDATED is internal use by reOpen() only
enum QueryOpts {QO_NONE=0, QO_STEM = 1, QO_KEEP_UPDATED = 8};
bool open(const string &dbdir, const string &stoplistfn,
OpenMode mode, int qops = QO_NONE);
OpenMode mode, bool keep_updated = false);
bool close();
bool isopen();
@@ -130,11 +130,12 @@ class Db {
/** Return total docs in db */
int docCnt();
// Parse query string and initialize query
bool setQuery(RefCntr<SearchData> q, int opts = QO_NONE,
const string& stemlang = "english");
bool getQueryTerms(list<string>& terms);
bool getMatchTerms(const Doc& doc, list<string>& terms);
/** Add extra database for querying */
bool addQueryDb(const string &dir);
/** Remove extra database. if dir == "", remove all. */
bool rmQueryDb(const string &dir);
/** Tell if directory seems to hold xapian db */
static bool testDbDir(const string &dir);
/** Return a list of index terms that match the input string
* Expansion is performed either with either wildcard or regexp processing
@@ -143,33 +144,12 @@ class Db {
bool termMatch(MatchType typ, const string &lang, const string &s,
list<TermMatchEntry>& result, int max = -1);
/** Add extra database for querying */
bool addQueryDb(const string &dir);
/** Remove extra database. if dir == "", remove all. */
bool rmQueryDb(const string &dir);
/** Tell if directory seems to hold xapian db */
static bool testDbDir(const string &dir);
/** Get document at rank i in current query.
This is probably vastly inferior to the type of interface in
Xapian, but we have to start with something simple to
experiment with the GUI. i is sequential from 0 to some value.
*/
bool getDoc(int i, Doc &doc, int *percent = 0);
/* Build synthetic abstract out of query terms and term position data */
bool makeDocAbstract(Doc &doc, string& abstract);
bool makeDocAbstract(Doc &doc, Query *query, string& abstract);
/** Get document for given filename and ipath */
bool getDoc(const string &fn, const string &ipath, Doc &doc, int *percent);
/** Expand query */
list<string> expand(const Doc &doc);
/** Get results count for current query */
int getResCnt();
/** Get a list of existing stemming databases */
std::list<std::string> getStemLangs();
@@ -189,22 +169,16 @@ class Db {
/** Filename wildcard expansion */
bool filenameWildExp(const string& exp, list<string>& names);
/** This has to be public for access by embedded Query::Native */
Native *m_ndb;
private:
// Internal form of close, can be called during destruction
bool i_close(bool final);
string m_filterTopDir; // Current query filter on subtree top directory
vector<int> m_dbindices; // In case there is a postq filter: sequence of
// db indices that match
string m_reason; // Error explanation
// A place for things we don't want visible here.
friend class Native;
Native *m_ndb;
unsigned int m_qOpts;
/* Parameters cached out of the configuration files */
// This is how long an abstract we keep or build from beginning of
// text when indexing. It only has an influence on the size of the
// db as we are free to shorten it again when displaying
@@ -215,7 +189,6 @@ private:
// This is how many words (context size) we keep around query terms
// when building the abstract
int m_synthAbsWordCtxLen;
// Flush threshold. Megabytes of text indexed before we flush.
int m_flushMb;
// Text bytes indexed since beginning
@@ -224,7 +197,6 @@ private:
long long m_flushtxtsz;
// Text bytes at last fsoccup check
long long m_occtxtsz;
// Maximum file system occupation percentage
int m_maxFsOccupPc;
src/rcldb/rcldb_p.h
+67
View File
@@ -0,0 +1,67 @@
#ifndef _rcldb_p_h_included_
#define _rcldb_p_h_included_
#include "xapian.h"
namespace Rcl {
/* @(#$Id: rcldb_p.h,v 1.1 2008-06-13 18:22:46 dockes Exp $ (C) 2007 J.F.Dockes */
// Generic Xapian exception catching code. We do this quite often,
// and I have no idea how to do this except for a macro
#define XCATCHERROR(MSG) \
catch (const Xapian::Error &e) { \
MSG = e.get_msg(); \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (const string &s) { \
MSG = s; \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (const char *s) { \
MSG = s; \
if (MSG.empty()) MSG = "Empty error message"; \
} catch (...) { \
MSG = "Caught unknown xapian exception"; \
}
class Query;
// A class for data and methods that would have to expose
// Xapian-specific stuff if they were in Rcl::Db. There could actually be
// 2 different ones for indexing or query as there is not much in
// common.
class Db::Native {
public:
Db *m_db;
bool m_isopen;
bool m_iswritable;
// Indexing
Xapian::WritableDatabase wdb;
// Querying
Xapian::Database db;
Native(Db *db)
: m_db(db), m_isopen(false), m_iswritable(false)
{ }
~Native() {
}
string makeAbstract(Xapian::docid id, Query *query);
bool dbDataToRclDoc(Xapian::docid docid, std::string &data, Doc &doc);
/** Compute list of subdocuments for a given path (given by hash)
* We look for all Q terms beginning with the path/hash
* As suggested by James Aylett, a better method would be to add
* a single term (ie: XP/path/to/file) to all subdocs, then finding
* them would be a simple matter of retrieving the posting list for the
* term. There would still be a need for the current Qterm though, as a
* unique term for replace_document, and for retrieving by
* path/ipath (history)
*/
bool subDocs(const string &hash, vector<Xapian::docid>& docids);
};
}
#endif /* _rcldb_p_h_included_ */
src/rcldb/rclquery.cpp
+354
View File
@@ -0,0 +1,354 @@
#ifndef lint
static char rcsid[] = "@(#$Id: rclquery.cpp,v 1.1 2008-06-13 18:22:46 dockes Exp $ (C) 2008 J.F.Dockes";
#endif
#include <list>
#include <vector>
#include "rcldb.h"
#include "rcldb_p.h"
#include "rclquery.h"
#include "rclquery_p.h"
#include "debuglog.h"
#include "conftree.h"
#include "smallut.h"
#include "searchdata.h"
#ifndef NO_NAMESPACES
namespace Rcl {
#endif
class FilterMatcher : public Xapian::MatchDecider {
public:
FilterMatcher(const string &topdir)
: m_topdir(topdir)
{}
virtual ~FilterMatcher() {}
virtual
#if XAPIAN_MAJOR_VERSION < 1
int
#else
bool
#endif
operator()(const Xapian::Document &xdoc) const
{
// Parse xapian document's data and populate doc fields
string data = xdoc.get_data();
ConfSimple parms(&data);
// The only filtering for now is on file path (subtree)
string url;
parms.get(string("url"), url);
LOGDEB2(("FilterMatcher topdir [%s] url [%s]\n",
m_topdir.c_str(), url.c_str()));
if (url.find(m_topdir, 7) == 7) {
return true;
} else {
return false;
}
}
private:
string m_topdir;
};
Query::Query(Db *db)
: m_nq(new Native(this)), m_db(db)
{
}
Query::~Query()
{
deleteZ(m_nq);
}
string Query::getReason() const
{
return m_reason;
}
Db *Query::whatDb()
{
return m_db;
}
//#define ISNULL(X) (X).isNull()
#define ISNULL(X) !(X)
// Prepare query out of user search data
bool Query::setQuery(RefCntr<SearchData> sdata, int opts,
const string& stemlang)
{
if (!m_db || ISNULL(m_nq)) {
LOGERR(("Query::setQuery: not initialised!\n"));
return false;
}
m_reason.erase();
LOGDEB(("Query::setQuery:\n"));
m_filterTopDir = sdata->getTopdir();
deleteZ(m_nq->decider);
deleteZ(m_nq->postfilter);
if (!m_filterTopDir.empty()) {
#if XAPIAN_FILTERING
m_nq->decider =
#else
m_nq->postfilter =
#endif
new FilterMatcher(m_filterTopDir);
}
m_nq->m_dbindices.clear();
m_qOpts = opts;
m_nq->termfreqs.clear();
Xapian::Query xq;
if (!sdata->toNativeQuery(*m_db, &xq,
(opts & QO_STEM) ? stemlang : "")) {
m_reason += sdata->getReason();
return false;
}
m_nq->query = xq;
string ermsg;
string d;
try {
delete m_nq->enquire;
m_nq->enquire = new Xapian::Enquire(m_db->m_ndb->db);
m_nq->enquire->set_query(m_nq->query);
m_nq->mset = Xapian::MSet();
// Get the query description and trim the "Xapian::Query"
d = m_nq->query.get_description();
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGDEB(("Query::SetQuery: xapian error %s\n", ermsg.c_str()));
return false;
}
if (d.find("Xapian::Query") == 0)
d.erase(0, strlen("Xapian::Query"));
if (!m_filterTopDir.empty()) {
d += string(" [dir: ") + m_filterTopDir + "]";
}
sdata->setDescription(d);
LOGDEB(("Query::SetQuery: Q: %s\n", sdata->getDescription().c_str()));
return true;
}
bool Query::getQueryTerms(list<string>& terms)
{
if (ISNULL(m_nq))
return false;
terms.clear();
Xapian::TermIterator it;
string ermsg;
try {
for (it = m_nq->query.get_terms_begin();
it != m_nq->query.get_terms_end(); it++) {
terms.push_back(*it);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
return false;
}
return true;
}
bool Query::getMatchTerms(const Doc& doc, list<string>& terms)
{
if (ISNULL(m_nq) || !m_nq->enquire) {
LOGERR(("Query::getMatchTerms: no query opened\n"));
return -1;
}
terms.clear();
Xapian::TermIterator it;
Xapian::docid id = Xapian::docid(doc.xdocid);
string ermsg;
try {
for (it=m_nq->enquire->get_matching_terms_begin(id);
it != m_nq->enquire->get_matching_terms_end(id); it++) {
terms.push_back(*it);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
return false;
}
return true;
}
// Mset size
static const int qquantum = 30;
int Query::getResCnt()
{
if (ISNULL(m_nq) || !m_nq->enquire) {
LOGERR(("Query::getResCnt: no query opened\n"));
return -1;
}
string ermsg;
if (m_nq->mset.size() <= 0) {
try {
m_nq->mset = m_nq->enquire->get_mset(0, qquantum,
0, m_nq->decider);
} catch (const Xapian::DatabaseModifiedError &error) {
m_db->m_ndb->db.reopen();
m_nq->mset = m_nq->enquire->get_mset(0, qquantum,
0, m_nq->decider);
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("enquire->get_mset: exception: %s\n", ermsg.c_str()));
return -1;
}
}
int ret = -1;
try {
ret = m_nq->mset.get_matches_lower_bound();
} catch (...) {}
return ret;
}
// Get document at rank i in query (i is the index in the whole result
// set, as in the enquire class. We check if the current mset has the
// doc, else ask for an other one. We use msets of 10 documents. Don't
// know if the whole thing makes sense at all but it seems to work.
//
// If there is a postquery filter (ie: file names), we have to
// maintain a correspondance from the sequential external index
// sequence to the internal Xapian hole-y one (the holes being the documents
// that dont match the filter).
bool Query::getDoc(int exti, Doc &doc, int *percent)
{
LOGDEB1(("Query::getDoc: exti %d\n", exti));
if (ISNULL(m_nq) || !m_nq->enquire) {
LOGERR(("Query::getDoc: no query opened\n"));
return false;
}
int xapi;
if (m_nq->postfilter) {
// There is a postquery filter, does this fall in already known area ?
if (exti >= (int)m_nq->m_dbindices.size()) {
// Have to fetch xapian docs and filter until we get
// enough or fail
m_nq->m_dbindices.reserve(exti+1);
// First xapian doc we fetch is the one after last stored
int first = m_nq->m_dbindices.size() > 0 ?
m_nq->m_dbindices.back() + 1 : 0;
// Loop until we get enough docs
while (exti >= (int)m_nq->m_dbindices.size()) {
LOGDEB(("Query::getDoc: fetching %d starting at %d\n",
qquantum, first));
try {
m_nq->mset = m_nq->enquire->get_mset(first, qquantum);
} catch (const Xapian::DatabaseModifiedError &error) {
m_db->m_ndb->db.reopen();
m_nq->mset = m_nq->enquire->get_mset(first, qquantum);
} catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str()));
abort();
}
if (m_nq->mset.empty()) {
LOGDEB(("Query::getDoc: got empty mset\n"));
return false;
}
first = m_nq->mset.get_firstitem();
for (unsigned int i = 0; i < m_nq->mset.size() ; i++) {
LOGDEB(("Query::getDoc: [%d]\n", i));
Xapian::Document xdoc = m_nq->mset[i].get_document();
if ((*m_nq->postfilter)(xdoc)) {
m_nq->m_dbindices.push_back(first + i);
}
}
first = first + m_nq->mset.size();
}
}
xapi = m_nq->m_dbindices[exti];
} else {
xapi = exti;
}
// From there on, we work with a xapian enquire item number. Fetch it
int first = m_nq->mset.get_firstitem();
int last = first + m_nq->mset.size() -1;
if (!(xapi >= first && xapi <= last)) {
LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));
try {
m_nq->mset = m_nq->enquire->get_mset(xapi, qquantum,
0, m_nq->decider);
} catch (const Xapian::DatabaseModifiedError &error) {
m_db->m_ndb->db.reopen();
m_nq->mset = m_nq->enquire->get_mset(xapi, qquantum,
0, m_nq->decider);
} catch (const Xapian::Error & error) {
LOGERR(("enquire->get_mset: exception: %s\n",
error.get_msg().c_str()));
abort();
}
if (m_nq->mset.empty())
return false;
first = m_nq->mset.get_firstitem();
last = first + m_nq->mset.size() -1;
}
LOGDEB1(("Query::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",
m_nq->query.get_description().c_str(),
first, last,
m_nq->mset.get_matches_lower_bound()));
Xapian::Document xdoc = m_nq->mset[xapi-first].get_document();
Xapian::docid docid = *(m_nq->mset[xapi-first]);
if (percent)
*percent = m_nq->mset.convert_to_percent(m_nq->mset[xapi-first]);
// Parse xapian document's data and populate doc fields
string data = xdoc.get_data();
return m_db->m_ndb->dbDataToRclDoc(docid, data, doc);
}
list<string> Query::expand(const Doc &doc)
{
list<string> res;
if (ISNULL(m_nq) || !m_nq->enquire) {
LOGERR(("Query::expand: no query opened\n"));
return res;
}
string ermsg;
for (int tries = 0; tries < 2; tries++) {
try {
Xapian::RSet rset;
rset.add_document(Xapian::docid(doc.xdocid));
// We don't exclude the original query terms.
Xapian::ESet eset = m_nq->enquire->get_eset(20, rset, false);
LOGDEB(("ESet terms:\n"));
// We filter out the special terms
for (Xapian::ESetIterator it = eset.begin();
it != eset.end(); it++) {
LOGDEB((" [%s]\n", (*it).c_str()));
if ((*it).empty() || ((*it).at(0)>='A' && (*it).at(0)<='Z'))
continue;
res.push_back(*it);
if (res.size() >= 10)
break;
}
} catch (const Xapian::DatabaseModifiedError &error) {
continue;
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Query::expand: xapian error %s\n", ermsg.c_str()));
res.clear();
}
break;
}
return res;
}
}
src/rcldb/rclquery.h
+92
View File
@@ -0,0 +1,92 @@
#ifndef _rclquery_h_included_
#define _rclquery_h_included_
/* @(#$Id: rclquery.h,v 1.1 2008-06-13 18:22:46 dockes Exp $ (C) 2008 J.F.Dockes */
/*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include <string>
#include <list>
#include <vector>
#ifndef NO_NAMESPACES
using std::string;
using std::list;
using std::vector;
#endif
#include "refcntr.h"
#ifndef NO_NAMESPACES
namespace Rcl {
#endif
class SearchData;
class Db;
class Doc;
/**
* An Rcl::Query is a question (SearchData) applied to a
* database. Handles access to the results. Somewhat equivalent to a
* cursor in an rdb.
*/
class Query {
public:
enum QueryOpts {QO_NONE=0, QO_STEM = 1};
Query(Db *db);
~Query();
/** Get explanation about last error */
string getReason() const;
/** Parse query string and initialize query */
bool setQuery(RefCntr<SearchData> q, int opts = QO_NONE,
const string& stemlang = "english");
bool getQueryTerms(list<string>& terms);
bool getMatchTerms(const Doc& doc, list<string>& terms);
/** Get document at rank i in current query. */
bool getDoc(int i, Doc &doc, int *percent = 0);
/** Expand query */
list<string> expand(const Doc &doc);
/** Get results count for current query */
int getResCnt();
Db *whatDb();
/** make this public for access from embedded Db::Native */
class Native;
Native *m_nq;
private:
string m_filterTopDir; // Current query filter on subtree top directory
string m_reason; // Error explanation
Db *m_db;
unsigned int m_qOpts;
/* Copyconst and assignemt private and forbidden */
Query(const Query &) {}
Query & operator=(const Query &) {return *this;};
};
#ifndef NO_NAMESPACES
}
#endif // NO_NAMESPACES
#endif /* _rclquery_h_included_ */
src/rcldb/rclquery_p.h
+66
View File
@@ -0,0 +1,66 @@
#ifndef _rclquery_p_h_included_
#define _rclquery_p_h_included_
/* @(#$Id: rclquery_p.h,v 1.1 2008-06-13 18:22:46 dockes Exp $ (C) 2007 J.F.Dockes */
#include <map>
#include <vector>
using std::map;
using std::vector;
#include <xapian.h>
#include "rclquery.h"
namespace Rcl {
class Query::Native {
public:
Xapian::Query query; // query descriptor: terms and subqueries
// joined by operators (or/and etc...)
vector<int> m_dbindices; // In case there is a postq filter: sequence of
// db indices that match
// Filtering results on location. There are 2 possible approaches
// for this:
// - Set a "MatchDecider" to be used by Xapian during the query
// - Filter the results out of Xapian (this also uses a
// Xapian::MatchDecider object, but applied to the results by Recoll.
//
// The result filtering approach was the first implemented.
//
// The efficiency of both methods depend on the searches, so the code
// for both has been kept. A nice point for the Xapian approach is that
// the result count estimate are correct (they are wrong with
// the postfilter approach). It is also faster in some worst case scenarios
// so this now the default (but the post-filtering is faster in many common
// cases).
//
// Which is used is decided in SetQuery(), by setting either of
// the two following members. This in turn is controlled by a
// preprocessor directive.
#define XAPIAN_FILTERING 1
Xapian::MatchDecider *decider; // Xapian does the filtering
Xapian::MatchDecider *postfilter; // Result filtering done by Recoll
Xapian::Enquire *enquire; // Open query descriptor.
Xapian::MSet mset; // Partial result set
Query *m_q;
// Term frequencies for current query. See makeAbstract, setQuery
map<string, double> termfreqs;
Native(Query *q)
: decider(0), postfilter(0), enquire(0), m_q(q)
{ }
~Native() {
delete decider;
delete postfilter;
delete enquire;
}
};
}
#endif /* _rclquery_p_h_included_ */
src/rcldb/searchdata.h
+2 -2
View File
@@ -16,7 +16,7 @@
*/
#ifndef _SEARCHDATA_H_INCLUDED_
#define _SEARCHDATA_H_INCLUDED_
/* @(#$Id: searchdata.h,v 1.13 2008-05-08 10:00:20 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: searchdata.h,v 1.14 2008-06-13 18:22:46 dockes Exp $ (C) 2004 J.F.Dockes */
/**
* Structures to hold data coming almost directly from the gui
@@ -47,7 +47,7 @@ class SearchDataClause;
/**
Data structure representing a Recoll user query, for translation
into a Xapian query tree.
into a Xapian query tree. This could probably better called a 'question'.
This is a list of search clauses combined through either OR or AND.