tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logging now uses c++ streams

Browse Source
This commit is contained in:
Jean-Francois Dockes
2016-07-12 09:41:04 +02:00
parent 7c4223088b
commit f6a999de84
123 changed files with 1974 additions and 2760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/daterange.cpp
+2 -1
View File
@@ -31,7 +31,7 @@ using namespace std;
#include <xapian.h>
#include "debuglog.h"
#include "log.h"
#include "rclconfig.h"
namespace Rcl {
@@ -124,3 +124,4 @@ Xapian::Query date_range_filter(int y1, int m1, int d1, int y2, int m2, int d2)
}
src/rcldb/expansiondbs.cpp
+7 -7
View File
@@ -20,7 +20,7 @@
#include MEMORY_INCLUDE
#include "debuglog.h"
#include "log.h"
#include "utf8iter.h"
#include "smallut.h"
#include "chrono.h"
@@ -41,8 +41,7 @@ namespace Rcl {
bool createExpansionDbs(Xapian::WritableDatabase& wdb,
const vector<string>& langs)
{
LOGDEB(("StemDb::createExpansionDbs: languages: %s\n",
stringsToString(langs).c_str()));
LOGDEB("StemDb::createExpansionDbs: languages: " << (stringsToString(langs)) << "\n" );
Chrono cron;
// Erase and recreate all the expansion groups
@@ -102,7 +101,7 @@ bool createExpansionDbs(Xapian::WritableDatabase& wdb,
if (utfit.eof()) // Empty term?? Seems to happen.
continue;
if (TextSplit::isCJK(*utfit)) {
// LOGDEB(("stemskipped: Skipping CJK\n"));
// LOGDEB("stemskipped: Skipping CJK\n" );
continue;
}
@@ -119,7 +118,7 @@ bool createExpansionDbs(Xapian::WritableDatabase& wdb,
// Dont' apply stemming to terms which don't look like
// natural language words.
if (!Db::isSpellingCandidate(*it)) {
LOGDEB1(("createExpansionDbs: skipped: [%s]\n", (*it).c_str()));
LOGDEB1("createExpansionDbs: skipped: [" << ((*it)) << "]\n" );
continue;
}
@@ -145,12 +144,13 @@ bool createExpansionDbs(Xapian::WritableDatabase& wdb,
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("Db::createStemDb: map build failed: %s\n", ermsg.c_str()));
LOGERR("Db::createStemDb: map build failed: " << (ermsg) << "\n" );
return false;
}
LOGDEB(("StemDb::createExpansionDbs: done: %.2f S\n", cron.secs()));
LOGDEB("StemDb::createExpansionDbs: done: " << (cron.secs()) << " S\n" );
return true;
}
}
src/rcldb/rclabstract.cpp
+19 -21
View File
@@ -20,7 +20,7 @@
#include <map>
#include "debuglog.h"
#include "log.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "rclquery.h"
@@ -50,7 +50,7 @@ static void listList(const string& what, const vector<string>&l)
for (vector<string>::const_iterator it = l.begin(); it != l.end(); it++) {
a = a + *it + " ";
}
LOGDEB(("%s: %s\n", what.c_str(), a.c_str()));
LOGDEB("" << (what) << ": " << (a) << "\n" );
}
#else
#define LOGABS LOGDEB2
@@ -82,7 +82,7 @@ static void noPrefixList(const vector<string>& in, vector<string>& out)
bool Query::Native::getMatchTerms(unsigned long xdocid, vector<string>& terms)
{
if (!xenquire) {
LOGERR(("Query::getMatchTerms: no query opened\n"));
LOGERR("Query::getMatchTerms: no query opened\n" );
return false;
}
@@ -95,7 +95,7 @@ bool Query::Native::getMatchTerms(unsigned long xdocid, vector<string>& terms)
xenquire->get_matching_terms_end(id)),
m_q->m_db->m_ndb->xrdb, m_q->m_reason);
if (!m_q->m_reason.empty()) {
LOGERR(("getMatchTerms: xapian error: %s\n", m_q->m_reason.c_str()));
LOGERR("getMatchTerms: xapian error: " << (m_q->m_reason) << "\n" );
return false;
}
noPrefixList(iterms, terms);
@@ -178,7 +178,7 @@ double Query::Native::qualityTerms(Xapian::docid docid,
if (eit != hld.terms.end()) {
byRoot[eit->second].push_back(*qit);
} else {
LOGDEB0(("qualityTerms: [%s] not found in hld\n", (*qit).c_str()));
LOGDEB0("qualityTerms: [" << ((*qit)) << "] not found in hld\n" );
byRoot[*qit].push_back(*qit);
}
}
@@ -257,7 +257,7 @@ double Query::Native::qualityTerms(Xapian::docid docid,
int Query::Native::getFirstMatchPage(Xapian::docid docid, string& term)
{
if (!m_q|| !m_q->m_db || !m_q->m_db->m_ndb || !m_q->m_db->m_ndb->m_isopen) {
LOGERR(("Query::getFirstMatchPage: no db\n"));
LOGERR("Query::getFirstMatchPage: no db\n" );
return -1;
}
Rcl::Db::Native *ndb(m_q->m_db->m_ndb);
@@ -267,7 +267,7 @@ int Query::Native::getFirstMatchPage(Xapian::docid docid, string& term)
getMatchTerms(docid, terms);
if (terms.empty()) {
LOGDEB(("getFirstMatchPage: empty match term list (field match?)\n"));
LOGDEB("getFirstMatchPage: empty match term list (field match?)\n" );
return -1;
}
@@ -322,7 +322,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
vector<string> matchedTerms;
getMatchTerms(docid, matchedTerms);
if (matchedTerms.empty()) {
LOGDEB(("makeAbstract::Empty term list\n"));
LOGDEB("makeAbstract::Empty term list\n" );
return ABSRES_ERROR;
}
@@ -342,7 +342,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
LOGABS(("makeAbstract:%d: computed Qcoefs.\n", chron.ms()));
// This can't happen, but would crash us
if (totalweight == 0.0) {
LOGERR(("makeAbstract: totalweight == 0.0 !\n"));
LOGERR("makeAbstract: totalweight == 0.0 !\n" );
return ABSRES_ERROR;
}
@@ -494,7 +494,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
// This can happen if there are term occurences in the keywords
// etc. but not elsewhere ?
if (totaloccs == 0) {
LOGDEB(("makeAbstract: no occurrences\n"));
LOGDEB("makeAbstract: no occurrences\n" );
return ABSRES_OK;
}
@@ -513,8 +513,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
continue;
if (m_q->m_snipMaxPosWalk > 0 && cutoff-- < 0) {
ret |= ABSRES_TERMMISS;
LOGDEB0(("makeAbstract: max term count cutoff %d\n",
m_q->m_snipMaxPosWalk));
LOGDEB0("makeAbstract: max term count cutoff " << (m_q->m_snipMaxPosWalk) << "\n" );
break;
}
@@ -524,8 +523,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
pos != xrdb.positionlist_end(docid, *term); pos++) {
if (m_q->m_snipMaxPosWalk > 0 && cutoff-- < 0) {
ret |= ABSRES_TERMMISS;
LOGDEB0(("makeAbstract: max term count cutoff %d\n",
m_q->m_snipMaxPosWalk));
LOGDEB0("makeAbstract: max term count cutoff " << (m_q->m_snipMaxPosWalk) << "\n" );
break;
}
// If we are beyond the max possible position, stop
@@ -539,8 +537,7 @@ int Query::Native::makeAbstract(Xapian::docid docid,
// at the same position, we want to keep only the
// first one (ie: dockes and dockes@wanadoo.fr)
if (vit->second.empty()) {
LOGDEB2(("makeAbstract: populating: [%s] at %d\n",
(*term).c_str(), *pos));
LOGDEB2("makeAbstract: populating: [" << ((*term)) << "] at " << (*pos) << "\n" );
sparseDoc[*pos] = *term;
}
}
@@ -557,11 +554,11 @@ int Query::Native::makeAbstract(Xapian::docid docid,
it++, ipos++) {
if (it->empty()) {
if (!epty)
LOGDEB(("makeAbstract:vec[%d]: [%s]\n", ipos, it->c_str()));
LOGDEB("makeAbstract:vec[" << (ipos) << "]: [" << (it) << "]\n" );
epty=true;
} else {
epty = false;
LOGDEB(("makeAbstract:vec[%d]: [%s]\n", ipos, it->c_str()));
LOGDEB("makeAbstract:vec[" << (ipos) << "]: [" << (it) << "]\n" );
}
}
#endif
@@ -579,9 +576,9 @@ int Query::Native::makeAbstract(Xapian::docid docid,
string term;
for (map<unsigned int, string>::const_iterator it = sparseDoc.begin();
it != sparseDoc.end(); it++) {
LOGDEB2(("Abtract:output %u -> [%s]\n", it->first, it->second.c_str()));
LOGDEB2("Abtract:output " << (it->first) << " -> [" << (it->second) << "]\n" );
if (!occupiedmarker.compare(it->second)) {
LOGDEB(("Abstract: qtrm position not filled ??\n"));
LOGDEB("Abstract: qtrm position not filled ??\n" );
continue;
}
if (chunk.empty() && !vpbreaks.empty()) {
@@ -611,9 +608,10 @@ int Query::Native::makeAbstract(Xapian::docid docid,
if (!chunk.empty())
vabs.push_back(Snippet(page, chunk).setTerm(term));
LOGDEB2(("makeAbtract: done in %d mS\n", chron.millis()));
LOGDEB2("makeAbtract: done in " << (chron.millis()) << " mS\n" );
return ret;
}
}
src/rcldb/rcldb.cpp
+148 -198
View File
File diff suppressed because it is too large Load Diff
src/rcldb/rcldb_p.h
-2
View File
@@ -27,7 +27,6 @@
#ifdef IDX_THREADS
#include "workqueue.h"
#endif // IDX_THREADS
#include "debuglog.h"
#include "xmacros.h"
#include "ptmutex.h"
@@ -82,7 +81,6 @@ class Db::Native {
bool m_noversionwrite; //Set if open failed because of version mismatch!
#ifdef IDX_THREADS
WorkQueue<DbUpdTask*> m_wqueue;
int m_loglevel;
PTMutexInit m_mutex;
long long m_totalworkns;
bool m_havewriteq;
src/rcldb/rcldoc.cpp
+18 -18
View File
@@ -17,7 +17,7 @@
#include "autoconfig.h"
#include "rcldoc.h"
#include "debuglog.h"
#include "log.h"
#include "rclutil.h"
namespace Rcl {
@@ -50,27 +50,26 @@ namespace Rcl {
void Doc::dump(bool dotext) const
{
LOGDEB(("Rcl::Doc::dump: url: [%s]\n", url.c_str()));
LOGDEB(("Rcl::Doc::dump: idxurl: [%s]\n", idxurl.c_str()));
LOGDEB(("Rcl::Doc::dump: ipath: [%s]\n", ipath.c_str()));
LOGDEB(("Rcl::Doc::dump: mimetype: [%s]\n", mimetype.c_str()));
LOGDEB(("Rcl::Doc::dump: fmtime: [%s]\n", fmtime.c_str()));
LOGDEB(("Rcl::Doc::dump: dmtime: [%s]\n", dmtime.c_str()));
LOGDEB(("Rcl::Doc::dump: origcharset: [%s]\n", origcharset.c_str()));
LOGDEB(("Rcl::Doc::dump: syntabs: [%d]\n", syntabs));
LOGDEB(("Rcl::Doc::dump: pcbytes: [%s]\n", pcbytes.c_str()));
LOGDEB(("Rcl::Doc::dump: fbytes: [%s]\n", fbytes.c_str()));
LOGDEB(("Rcl::Doc::dump: dbytes: [%s]\n", dbytes.c_str()));
LOGDEB(("Rcl::Doc::dump: sig: [%s]\n", sig.c_str()));
LOGDEB(("Rcl::Doc::dump: pc: [%d]\n", pc));
LOGDEB(("Rcl::Doc::dump: xdocid: [%lu]\n", (unsigned long)xdocid));
LOGDEB("Rcl::Doc::dump: url: [" << (url) << "]\n" );
LOGDEB("Rcl::Doc::dump: idxurl: [" << (idxurl) << "]\n" );
LOGDEB("Rcl::Doc::dump: ipath: [" << (ipath) << "]\n" );
LOGDEB("Rcl::Doc::dump: mimetype: [" << (mimetype) << "]\n" );
LOGDEB("Rcl::Doc::dump: fmtime: [" << (fmtime) << "]\n" );
LOGDEB("Rcl::Doc::dump: dmtime: [" << (dmtime) << "]\n" );
LOGDEB("Rcl::Doc::dump: origcharset: [" << (origcharset) << "]\n" );
LOGDEB("Rcl::Doc::dump: syntabs: [" << (syntabs) << "]\n" );
LOGDEB("Rcl::Doc::dump: pcbytes: [" << (pcbytes) << "]\n" );
LOGDEB("Rcl::Doc::dump: fbytes: [" << (fbytes) << "]\n" );
LOGDEB("Rcl::Doc::dump: dbytes: [" << (dbytes) << "]\n" );
LOGDEB("Rcl::Doc::dump: sig: [" << (sig) << "]\n" );
LOGDEB("Rcl::Doc::dump: pc: [" << (pc) << "]\n" );
LOGDEB("Rcl::Doc::dump: xdocid: [" << ((unsigned long)xdocid) << "]\n" );
for (map<string, string>::const_iterator it = meta.begin();
it != meta.end(); it++) {
LOGDEB(("Rcl::Doc::dump: meta[%s]: [%s]\n",
(*it).first.c_str(), (*it).second.c_str()));
LOGDEB("Rcl::Doc::dump: meta[" << ((*it).first) << "]: [" << ((*it).second) << "]\n" );
}
if (dotext)
LOGDEB(("Rcl::Doc::dump: text: \n[%s]\n", text.c_str()));
LOGDEB("Rcl::Doc::dump: text: \n[" << (text) << "]\n" );
}
// Copy ensuring no shared string data, for threading issues.
@@ -100,3 +99,4 @@ namespace Rcl {
}
}
src/rcldb/rcldups.cpp
+13 -12
View File
@@ -24,7 +24,7 @@ using namespace std;
#include <xapian.h>
#include "debuglog.h"
#include "log.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "xmacros.h"
@@ -39,11 +39,11 @@ namespace Rcl {
bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
{
if (m_ndb == 0) {
LOGERR(("Db::docDups: no db\n"));
LOGERR("Db::docDups: no db\n" );
return false;
}
if (idoc.xdocid == 0) {
LOGERR(("Db::docDups: null xdocid in input doc\n"));
LOGERR("Db::docDups: null xdocid in input doc\n" );
return false;
}
// Get the xapian doc
@@ -51,7 +51,7 @@ bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
XAPTRY(xdoc = m_ndb->xrdb.get_document(Xapian::docid(idoc.xdocid)),
m_ndb->xrdb, m_reason);
if (!m_reason.empty()) {
LOGERR(("Db::docDups: xapian error: %s\n", m_reason.c_str()));
LOGERR("Db::docDups: xapian error: " << (m_reason) << "\n" );
return false;
}
@@ -59,11 +59,11 @@ bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
string digest;
XAPTRY(digest = xdoc.get_value(VALUE_MD5), m_ndb->xrdb, m_reason);
if (!m_reason.empty()) {
LOGERR(("Db::docDups: xapian error: %s\n", m_reason.c_str()));
LOGERR("Db::docDups: xapian error: " << (m_reason) << "\n" );
return false;
}
if (digest.empty()) {
LOGDEB(("Db::docDups: doc has no md5\n"));
LOGDEB("Db::docDups: doc has no md5\n" );
return false;
}
string md5;
@@ -79,14 +79,14 @@ bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
Query query(this);
query.setCollapseDuplicates(0);
if (!query.setQuery(sd)) {
LOGERR(("Db::docDups: setQuery failed\n"));
LOGERR("Db::docDups: setQuery failed\n" );
return false;
}
int cnt = query.getResCnt();
for (int i = 0; i < cnt; i++) {
Doc doc;
if (!query.getDoc(i, doc)) {
LOGERR(("Db::docDups: getDoc failed at %d (cnt %d)\n", i, cnt));
LOGERR("Db::docDups: getDoc failed at " << (i) << " (cnt " << (cnt) << ")\n" );
return false;
}
odocs.push_back(doc);
@@ -98,18 +98,19 @@ bool Db::docDups(const Doc& idoc, vector<Doc>& odocs)
{
vector<Doc> dups;
bool ret;
LOGDEB(("DOCDUPS\n"));
LOGDEB("DOCDUPS\n" );
ret = m_db->docDups(doc, dups);
if (!ret) {
LOGDEB(("docDups failed\n"));
LOGDEB("docDups failed\n" );
} else if (dups.size() == 1) {
LOGDEB(("No dups\n"));
LOGDEB("No dups\n" );
} else {
for (unsigned int i = 0; i < dups.size(); i++) {
LOGDEB(("Dup: %s\n", dups[i].url.c_str()));
LOGDEB("Dup: " << (dups[i].url) << "\n" );
}
}
}
#endif
}
src/rcldb/rclquery.cpp
+29 -35
View File
@@ -27,7 +27,7 @@
#include "cstr.h"
#include "rclconfig.h"
#include "debuglog.h"
#include "log.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "rclquery.h"
@@ -132,7 +132,7 @@ public:
sortterm = sortterm.substr(i1, sortterm.size()-i1);
}
LOGDEB2(("QSorter: [%s] -> [%s]\n", term.c_str(), sortterm.c_str()));
LOGDEB2("QSorter: [" << (term) << "] -> [" << (sortterm) << "]\n" );
return sortterm;
}
@@ -166,8 +166,7 @@ void Query::setSortBy(const string& fld, bool ascending) {
m_sortField = m_db->getConf()->fieldQCanon(fld);
m_sortAscending = ascending;
}
LOGDEB0(("RclQuery::setSortBy: [%s] %s\n", m_sortField.c_str(),
m_sortAscending ? "ascending" : "descending"));
LOGDEB0("RclQuery::setSortBy: [" << (m_sortField) << "] " << (m_sortAscending ? "ascending" : "descending") << "\n" );
}
//#define ISNULL(X) (X).isNull()
@@ -176,10 +175,10 @@ void Query::setSortBy(const string& fld, bool ascending) {
// Prepare query out of user search data
bool Query::setQuery(STD_SHARED_PTR<SearchData> sdata)
{
LOGDEB(("Query::setQuery:\n"));
LOGDEB("Query::setQuery:\n" );
if (!m_db || ISNULL(m_nq)) {
LOGERR(("Query::setQuery: not initialised!\n"));
LOGERR("Query::setQuery: not initialised!\n" );
return false;
}
m_resCnt = -1;
@@ -233,7 +232,7 @@ bool Query::setQuery(STD_SHARED_PTR<SearchData> sdata)
}
if (!m_reason.empty()) {
LOGDEB(("Query::SetQuery: xapian error %s\n", m_reason.c_str()));
LOGDEB("Query::SetQuery: xapian error " << (m_reason) << "\n" );
return false;
}
@@ -242,7 +241,7 @@ bool Query::setQuery(STD_SHARED_PTR<SearchData> sdata)
sdata->setDescription(d);
m_sd = sdata;
LOGDEB(("Query::SetQuery: Q: %s\n", sdata->getDescription().c_str()));
LOGDEB("Query::SetQuery: Q: " << (sdata->getDescription()) << "\n" );
return true;
}
@@ -261,7 +260,7 @@ bool Query::getQueryTerms(vector<string>& terms)
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("getQueryTerms: xapian error: %s\n", ermsg.c_str()));
LOGERR("getQueryTerms: xapian error: " << (ermsg) << "\n" );
return false;
}
return true;
@@ -271,17 +270,16 @@ int Query::makeDocAbstract(const Doc &doc,
vector<Snippet>& abstract,
int maxoccs, int ctxwords)
{
LOGDEB(("makeDocAbstract: maxoccs %d ctxwords %d\n", maxoccs, ctxwords));
LOGDEB("makeDocAbstract: maxoccs " << (maxoccs) << " ctxwords " << (ctxwords) << "\n" );
if (!m_db || !m_db->m_ndb || !m_db->m_ndb->m_isopen || !m_nq) {
LOGERR(("Query::makeDocAbstract: no db or no nq\n"));
LOGERR("Query::makeDocAbstract: no db or no nq\n" );
return ABSRES_ERROR;
}
int ret = ABSRES_ERROR;
XAPTRY(ret = m_nq->makeAbstract(doc.xdocid, abstract, maxoccs, ctxwords),
m_db->m_ndb->xrdb, m_reason);
if (!m_reason.empty()) {
LOGDEB(("makeDocAbstract: makeAbstract error, reason: %s\n",
m_reason.c_str()));
LOGDEB("makeDocAbstract: makeAbstract error, reason: " << (m_reason) << "\n" );
return ABSRES_ERROR;
}
return ret;
@@ -321,9 +319,9 @@ bool Query::makeDocAbstract(const Doc &doc, string& abstract)
int Query::getFirstMatchPage(const Doc &doc, string& term)
{
LOGDEB1(("Db::getFirstMatchPage\n"));;
LOGDEB1("Db::getFirstMatchPage\n" );;
if (!m_nq) {
LOGERR(("Query::getFirstMatchPage: no nq\n"));
LOGERR("Query::getFirstMatchPage: no nq\n" );
return false;
}
int pagenum = -1;
@@ -341,7 +339,7 @@ static const int qquantum = 50;
int Query::getResCnt()
{
if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::getResCnt: no query opened\n"));
LOGERR("Query::getResCnt: no query opened\n" );
return -1;
}
if (m_resCnt >= 0)
@@ -356,9 +354,9 @@ int Query::getResCnt()
m_resCnt = m_nq->xmset.get_matches_lower_bound(),
m_db->m_ndb->xrdb, m_reason);
LOGDEB(("Query::getResCnt: %d %d mS\n", m_resCnt, chron.millis()));
LOGDEB("Query::getResCnt: " << (m_resCnt) << " " << (chron.millis()) << " mS\n" );
if (!m_reason.empty())
LOGERR(("xenquire->get_mset: exception: %s\n", m_reason.c_str()));
LOGERR("xenquire->get_mset: exception: " << (m_reason) << "\n" );
} else {
m_resCnt = m_nq->xmset.get_matches_lower_bound();
}
@@ -375,9 +373,9 @@ int Query::getResCnt()
// on subsequent calls is probably only due to disk caching.
bool Query::getDoc(int xapi, Doc &doc)
{
LOGDEB1(("Query::getDoc: xapian enquire index %d\n", xapi));
LOGDEB1("Query::getDoc: xapian enquire index " << (xapi) << "\n" );
if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::getDoc: no query opened\n"));
LOGERR("Query::getDoc: no query opened\n" );
return false;
}
@@ -385,28 +383,24 @@ bool Query::getDoc(int xapi, Doc &doc)
int last = first + m_nq->xmset.size() -1;
if (!(xapi >= first && xapi <= last)) {
LOGDEB(("Fetching for first %d, count %d\n", xapi, qquantum));
LOGDEB("Fetching for first " << (xapi) << ", count " << (qquantum) << "\n" );
XAPTRY(m_nq->xmset = m_nq->xenquire->get_mset(xapi, qquantum,
(const Xapian::RSet *)0),
m_db->m_ndb->xrdb, m_reason);
if (!m_reason.empty()) {
LOGERR(("enquire->get_mset: exception: %s\n", m_reason.c_str()));
LOGERR("enquire->get_mset: exception: " << (m_reason) << "\n" );
return false;
}
if (m_nq->xmset.empty()) {
LOGDEB(("enquire->get_mset: got empty result\n"));
LOGDEB("enquire->get_mset: got empty result\n" );
return false;
}
first = m_nq->xmset.get_firstitem();
last = first + m_nq->xmset.size() -1;
}
LOGDEB1(("Query::getDoc: Qry [%s] win [%d-%d] Estimated results: %d",
m_nq->query.get_description().c_str(),
first, last, m_nq->xmset.get_matches_lower_bound()));
Xapian::Document xdoc;
Xapian::docid docid = 0;
int pc = 0;
@@ -424,8 +418,7 @@ bool Query::getDoc(int xapi, Doc &doc)
m_reason.erase();
Chrono chron;
m_db->m_ndb->xdocToUdi(xdoc, udi);
LOGDEB2(("Query::getDoc: %d ms for udi [%s], collapse count %d\n",
chron.millis(), udi.c_str(), collapsecount));
LOGDEB2("Query::getDoc: " << (chron.millis()) << " ms for udi [" << (udi) << "], collapse count " << (collapsecount) << "\n" );
break;
} catch (Xapian::DatabaseModifiedError &error) {
// retry or end of loop
@@ -436,7 +429,7 @@ bool Query::getDoc(int xapi, Doc &doc)
break;
}
if (!m_reason.empty()) {
LOGERR(("Query::getDoc: %s\n", m_reason.c_str()));
LOGERR("Query::getDoc: " << (m_reason) << "\n" );
return false;
}
doc.meta[Rcl::Doc::keyudi] = udi;
@@ -461,10 +454,10 @@ bool Query::getDoc(int xapi, Doc &doc)
vector<string> Query::expand(const Doc &doc)
{
LOGDEB(("Rcl::Query::expand()\n"));
LOGDEB("Rcl::Query::expand()\n" );
vector<string> res;
if (ISNULL(m_nq) || !m_nq->xenquire) {
LOGERR(("Query::expand: no query opened\n"));
LOGERR("Query::expand: no query opened\n" );
return res;
}
@@ -474,11 +467,11 @@ vector<string> Query::expand(const Doc &doc)
rset.add_document(Xapian::docid(doc.xdocid));
// We don't exclude the original query terms.
Xapian::ESet eset = m_nq->xenquire->get_eset(20, rset, false);
LOGDEB(("ESet terms:\n"));
LOGDEB("ESet terms:\n" );
// We filter out the special terms
for (Xapian::ESetIterator it = eset.begin();
it != eset.end(); it++) {
LOGDEB((" [%s]\n", (*it).c_str()));
LOGDEB(" [" << ((*it)) << "]\n" );
if ((*it).empty() || has_prefix(*it))
continue;
res.push_back(*it);
@@ -496,7 +489,7 @@ vector<string> Query::expand(const Doc &doc)
}
if (!m_reason.empty()) {
LOGERR(("Query::expand: xapian error %s\n", m_reason.c_str()));
LOGERR("Query::expand: xapian error " << (m_reason) << "\n" );
res.clear();
}
@@ -504,3 +497,4 @@ vector<string> Query::expand(const Doc &doc)
}
}
src/rcldb/rclterms.cpp
+20 -32
View File
@@ -22,7 +22,7 @@
#include <string>
#include "debuglog.h"
#include "log.h"
#include "rcldb.h"
#include "rcldb_p.h"
#include "stemdb.h"
@@ -49,7 +49,7 @@ bool Db::filenameWildExp(const string& fnexp, vector<string>& names, int max)
pattern = "*" + pattern + "*";
} // else let it be
LOGDEB(("Rcl::Db::filenameWildExp: pattern: [%s]\n", pattern.c_str()));
LOGDEB("Rcl::Db::filenameWildExp: pattern: [" << (pattern) << "]\n" );
// We inconditionnally lowercase and strip the pattern, as is done
// during indexing. This seems to be the only sane possible
@@ -79,12 +79,12 @@ bool Db::filenameWildExp(const string& fnexp, vector<string>& names, int max)
// Walk the Y terms and return min/max
bool Db::maxYearSpan(int *minyear, int *maxyear)
{
LOGDEB(("Rcl::Db:maxYearSpan\n"));
LOGDEB("Rcl::Db:maxYearSpan\n" );
*minyear = 1000000;
*maxyear = -1000000;
TermMatchResult result;
if (!idxTermMatch(ET_WILD, string(), "*", result, -1, "xapyear")) {
LOGINFO(("Rcl::Db:maxYearSpan: termMatch failed\n"));
LOGINFO("Rcl::Db:maxYearSpan: termMatch failed\n" );
return false;
}
for (vector<TermMatchEntry>::const_iterator it = result.entries.begin();
@@ -165,11 +165,7 @@ bool Db::termMatch(int typ_sens, const string &lang, const string &_term,
bool diac_sensitive = (typ_sens & ET_DIACSENS) != 0;
bool case_sensitive = (typ_sens & ET_CASESENS) != 0;
LOGDEB0(("Db::TermMatch: typ %s diacsens %d casesens %d lang [%s] term [%s]"
" max %d field [%s] stripped %d init res.size %u\n",
tmtptostr(matchtyp), diac_sensitive, case_sensitive, lang.c_str(),
_term.c_str(), max, field.c_str(), o_index_stripchars,
res.entries.size()));
LOGDEB0("Db::TermMatch: typ " << (tmtptostr(matchtyp)) << " diacsens " << (diac_sensitive) << " casesens " << (case_sensitive) << " lang [" << (lang) << "] term [" << (_term) << "] max " << (max) << " field [" << (field) << "] stripped " << (o_index_stripchars) << " init res.size " << (res.entries.size()) << "\n" );
// If index is stripped, no case or diac expansion can be needed:
// for the processing inside this routine, everything looks like
@@ -179,7 +175,7 @@ bool Db::termMatch(int typ_sens, const string &lang, const string &_term,
if (o_index_stripchars) {
diac_sensitive = case_sensitive = true;
if (!unacmaybefold(_term, term, "UTF-8", UNACOP_UNACFOLD)) {
LOGERR(("Db::termMatch: unac failed for [%s]\n", _term.c_str()));
LOGERR("Db::termMatch: unac failed for [" << (_term) << "]\n" );
return false;
}
}
@@ -270,19 +266,17 @@ bool Db::termMatch(int typ_sens, const string &lang, const string &_term,
exp1.swap(lexp);
sort(lexp.begin(), lexp.end());
lexp.erase(unique(lexp.begin(), lexp.end()), lexp.end());
LOGDEB(("ExpTerm: stemexp: %s\n",
stringsToString(lexp).c_str()));
LOGDEB("ExpTerm: stemexp: " << (stringsToString(lexp)) << "\n" );
}
if (m_syngroups.ok() && (typ_sens & ET_SYNEXP)) {
LOGDEB(("ExpTerm: got syngroups\n"));
LOGDEB("ExpTerm: got syngroups\n" );
vector<string> exp1(lexp);
for (vector<string>::const_iterator it = lexp.begin();
it != lexp.end(); it++) {
vector<string> sg = m_syngroups.getgroup(*it);
if (!sg.empty()) {
LOGDEB(("ExpTerm: syns: %s -> %s\n",
it->c_str(), stringsToString(sg).c_str()));
LOGDEB("ExpTerm: syns: " << *it << " -> " << (stringsToString(sg)) << "\n" );
for (vector<string>::const_iterator it1 = sg.begin();
it1 != sg.end(); it1++) {
if (it1->find_first_of(" ") != string::npos) {
@@ -313,7 +307,7 @@ bool Db::termMatch(int typ_sens, const string &lang, const string &_term,
}
// Filter the result and get the stats, possibly add prefixes.
LOGDEB(("ExpandTerm:TM: lexp: %s\n", stringsToString(lexp).c_str()));
LOGDEB("ExpandTerm:TM: lexp: " << (stringsToString(lexp)) << "\n" );
for (vector<string>::const_iterator it = lexp.begin();
it != lexp.end(); it++) {
idxTermMatch(Rcl::Db::ET_WILD, "", *it, res, max, field);
@@ -341,13 +335,10 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
TermMatchResult& res, int max, const string& field)
{
int typ = matchTypeTp(typ_sens);
LOGDEB1(("Db::idxTermMatch: typ %s lang [%s] term [%s] "
"max %d field [%s] init res.size %u\n",
tmtptostr(typ), lang.c_str(), root.c_str(),
max, field.c_str(), res.entries.size()));
LOGDEB1("Db::idxTermMatch: typ " << (tmtptostr(typ)) << " lang [" << (lang) << "] term [" << (root) << "] max " << (max) << " field [" << (field) << "] init res.size " << (res.entries.size()) << "\n" );
if (typ == ET_STEM) {
LOGFATAL(("RCLDB: internal error: idxTermMatch called with ET_STEM\n"));
LOGFATAL("RCLDB: internal error: idxTermMatch called with ET_STEM\n" );
abort();
}
@@ -357,8 +348,7 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
if (!field.empty()) {
const FieldTraits *ftp = 0;
if (!fieldToTraits(field, &ftp, true) || ftp->pfx.empty()) {
LOGDEB(("Db::termMatch: field is not indexed (no prefix): [%s]\n",
field.c_str()));
LOGDEB("Db::termMatch: field is not indexed (no prefix): [" << (field) << "]\n" );
} else {
prefix = wrap_prefix(ftp->pfx);
}
@@ -369,8 +359,7 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
if (typ == ET_REGEXP) {
matcher = STD_SHARED_PTR<StrMatcher>(new StrRegexpMatcher(root));
if (!matcher->ok()) {
LOGERR(("termMatch: regcomp failed: %s\n",
matcher->getreason().c_str()))
LOGERR("termMatch: regcomp failed: " << (matcher->getreason()));
return false;
}
} else if (typ == ET_WILD) {
@@ -394,7 +383,7 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
} else {
is = prefix + root.substr(0, es);
}
LOGDEB2(("termMatch: initsec: [%s]\n", is.c_str()));
LOGDEB2("termMatch: initsec: [" << (is) << "]\n" );
for (int tries = 0; tries < 2; tries++) {
try {
@@ -446,7 +435,7 @@ bool Db::idxTermMatch(int typ_sens, const string &lang, const string &root,
break;
}
if (!m_reason.empty()) {
LOGERR(("termMatch: %s\n", m_reason.c_str()));
LOGERR("termMatch: " << (m_reason) << "\n" );
return false;
}
@@ -468,7 +457,7 @@ TermIter *Db::termWalkOpen()
tit->db = m_ndb->xrdb;
XAPTRY(tit->it = tit->db.allterms_begin(), tit->db, m_reason);
if (!m_reason.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
LOGERR("Db::termWalkOpen: xapian error: " << (m_reason) << "\n" );
return 0;
}
}
@@ -484,7 +473,7 @@ bool Db::termWalkNext(TermIter *tit, string &term)
, tit->db, m_reason);
if (!m_reason.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
LOGERR("Db::termWalkOpen: xapian error: " << (m_reason) << "\n" );
}
return false;
}
@@ -504,7 +493,7 @@ bool Db::termExists(const string& word)
m_ndb->xrdb, m_reason);
if (!m_reason.empty()) {
LOGERR(("Db::termWalkOpen: xapian error: %s\n", m_reason.c_str()));
LOGERR("Db::termWalkOpen: xapian error: " << (m_reason) << "\n" );
return false;
}
return true;
@@ -515,8 +504,7 @@ bool Db::stemDiffers(const string& lang, const string& word,
{
Xapian::Stem stemmer(lang);
if (!stemmer(word).compare(stemmer(base))) {
LOGDEB2(("Rcl::Db::stemDiffers: same for %s and %s\n",
word.c_str(), base.c_str()));
LOGDEB2("Rcl::Db::stemDiffers: same for " << (word) << " and " << (base) << "\n" );
return false;
}
return true;
src/rcldb/searchdata.cpp
+13 -12
View File
@@ -34,7 +34,7 @@ using namespace std;
#include "rcldb.h"
#include "rcldb_p.h"
#include "searchdata.h"
#include "debuglog.h"
#include "log.h"
#include "smallut.h"
#include "textsplit.h"
#include "unacpp.h"
@@ -68,7 +68,7 @@ void SearchData::commoninit()
SearchData::~SearchData()
{
LOGDEB0(("SearchData::~SearchData\n"));
LOGDEB0("SearchData::~SearchData\n" );
for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++)
delete *it;
}
@@ -79,13 +79,13 @@ SearchData::~SearchData()
// We remove very common terms from the query to avoid performance issues.
bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
{
LOGDEB0(("SearchData::maybeAddAutoPhrase()\n"));
LOGDEB0("SearchData::maybeAddAutoPhrase()\n" );
// cerr << "BEFORE SIMPLIFY\n"; dump(cerr);
simplify();
// cerr << "AFTER SIMPLIFY\n"; dump(cerr);
if (!m_query.size()) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: empty query\n"));
LOGDEB2("SearchData::maybeAddAutoPhrase: empty query\n" );
return false;
}
@@ -96,27 +96,27 @@ bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) {
SClType tp = (*it)->m_tp;
if (tp != SCLT_AND) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: wrong tp %d\n", tp));
LOGDEB2("SearchData::maybeAddAutoPhrase: wrong tp " << (tp) << "\n" );
return false;
}
SearchDataClauseSimple *clp =
dynamic_cast<SearchDataClauseSimple*>(*it);
if (clp == 0) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: dyncast failed\n"));
LOGDEB2("SearchData::maybeAddAutoPhrase: dyncast failed\n" );
return false;
}
if (it == m_query.begin()) {
field = clp->getfield();
} else {
if (clp->getfield().compare(field)) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: diff. fields\n"));
LOGDEB2("SearchData::maybeAddAutoPhrase: diff. fields\n" );
return false;
}
}
// If there are wildcards or quotes in there, bail out
if (clp->gettext().find_first_of("\"*[?") != string::npos) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: wildcards\n"));
LOGDEB2("SearchData::maybeAddAutoPhrase: wildcards\n" );
return false;
}
@@ -145,8 +145,8 @@ bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
swords.append(1, ' ');
swords += *it;
} else {
LOGDEB0(("SearchData::Autophrase: [%s] too frequent (%.2f %%)\n",
it->c_str(), 100 * freq));
LOGDEB0("SearchData::Autophrase: [" << *it << "] too frequent ("
<< (100 * freq) << " %" << ")\n" );
slack++;
}
}
@@ -154,7 +154,7 @@ bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
// We can't make a phrase with a single word :)
int nwords = TextSplit::countWords(swords);
if (nwords <= 1) {
LOGDEB2(("SearchData::maybeAddAutoPhrase: ended with 1 word\n"));
LOGDEB2("SearchData::maybeAddAutoPhrase: ended with 1 word\n" );
return false;
}
@@ -171,7 +171,7 @@ bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
bool SearchData::addClause(SearchDataClause* cl)
{
if (m_tp == SCLT_OR && cl->getexclude()) {
LOGERR(("SearchData::addClause: cant add EXCL to OR list\n"));
LOGERR("SearchData::addClause: cant add EXCL to OR list\n" );
m_reason = "No Negative (AND_NOT) clauses allowed in OR queries";
return false;
}
@@ -373,3 +373,4 @@ void SearchDataClauseSub::dump(ostream& o) const
}
} // Namespace Rcl
src/rcldb/searchdatatox.cpp
+40 -55
View File
@@ -34,7 +34,7 @@ using namespace std;
#include "rcldb.h"
#include "rcldb_p.h"
#include "searchdata.h"
#include "debuglog.h"
#include "log.h"
#include "smallut.h"
#include "textsplit.h"
#include "unacpp.h"
@@ -62,7 +62,7 @@ bool SearchData::expandFileTypes(Db &db, vector<string>& tps)
{
const RclConfig *cfg = db.getConf();
if (!cfg) {
LOGFATAL(("Db::expandFileTypes: null configuration!!\n"));
LOGFATAL("Db::expandFileTypes: null configuration!!\n" );
return false;
}
vector<string> exptps;
@@ -113,13 +113,12 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
for (qlist_it_t it = query.begin(); it != query.end(); it++) {
Xapian::Query nq;
if (!(*it)->toNativeQuery(db, &nq)) {
LOGERR(("SearchData::clausesToQuery: toNativeQuery failed: %s\n",
(*it)->getReason().c_str()));
LOGERR("SearchData::clausesToQuery: toNativeQuery failed: " << ((*it)->getReason()) << "\n" );
reason += (*it)->getReason() + " ";
return false;
}
if (nq.empty()) {
LOGDEB(("SearchData::clausesToQuery: skipping empty clause\n"));
LOGDEB("SearchData::clausesToQuery: skipping empty clause\n" );
continue;
}
// If this structure is an AND list, must use AND_NOT for excl clauses.
@@ -144,7 +143,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
xq = Xapian::Query(op, xq, nq);
}
if (int(xq.get_length()) >= getMaxCl()) {
LOGERR(("%s\n", maxXapClauseMsg));
LOGERR("" << (maxXapClauseMsg) << "\n" );
m_reason += maxXapClauseMsg;
if (!o_index_stripchars)
m_reason += maxXapClauseCaseDiacMsg;
@@ -152,7 +151,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
}
}
LOGDEB0(("SearchData::clausesToQuery: got %d clauses\n", xq.get_length()));
LOGDEB0("SearchData::clausesToQuery: got " << (xq.get_length()) << " clauses\n" );
if (xq.empty())
xq = Xapian::Query::MatchAll;
@@ -163,7 +162,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
{
LOGDEB(("SearchData::toNativeQuery: stemlang [%s]\n", m_stemlang.c_str()));
LOGDEB("SearchData::toNativeQuery: stemlang [" << (m_stemlang) << "]\n" );
m_reason.erase();
db.getConf()->getConfParam("maxTermExpand", &m_maxexp);
@@ -175,8 +174,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
// Xapian query tree
Xapian::Query xq;
if (!clausesToQuery(db, m_tp, m_query, m_reason, &xq)) {
LOGERR(("SearchData::toNativeQuery: clausesToQuery failed. reason: %s\n",
m_reason.c_str()));
LOGERR("SearchData::toNativeQuery: clausesToQuery failed. reason: " << (m_reason) << "\n" );
return false;
}
@@ -185,7 +183,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
if (m_dates.y1 == 0 || m_dates.y2 == 0) {
int minyear = 1970, maxyear = 2100;
if (!db.maxYearSpan(&minyear, &maxyear)) {
LOGERR(("Can't retrieve index min/max dates\n"));
LOGERR("Can't retrieve index min/max dates\n" );
//whatever, go on.
}
@@ -200,18 +198,16 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
m_dates.d2 = 31;
}
}
LOGDEB(("Db::toNativeQuery: date interval: %d-%d-%d/%d-%d-%d\n",
m_dates.y1, m_dates.m1, m_dates.d1,
m_dates.y2, m_dates.m2, m_dates.d2));
LOGDEB("Db::toNativeQuery: date interval: " << (m_dates.y1) << "-" << (m_dates.m1) << "-" << (m_dates.d1) << "/" << (m_dates.y2) << "-" << (m_dates.m2) << "-" << (m_dates.d2) << "\n" );
Xapian::Query dq = date_range_filter(m_dates.y1, m_dates.m1, m_dates.d1,
m_dates.y2, m_dates.m2, m_dates.d2);
if (dq.empty()) {
LOGINFO(("Db::toNativeQuery: date filter is empty\n"));
LOGINFO("Db::toNativeQuery: date filter is empty\n" );
}
// If no probabilistic query is provided then promote the daterange
// filter to be THE query instead of filtering an empty query.
if (xq.empty()) {
LOGINFO(("Db::toNativeQuery: proba query is empty\n"));
LOGINFO("Db::toNativeQuery: proba query is empty\n" );
xq = dq;
} else {
xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, dq);
@@ -243,7 +239,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
// If no probabilistic query is provided then promote the
// filter to be THE query instead of filtering an empty query.
if (xq.empty()) {
LOGINFO(("Db::toNativeQuery: proba query is empty\n"));
LOGINFO("Db::toNativeQuery: proba query is empty\n" );
xq = sq;
} else {
xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, sq);
@@ -267,7 +263,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
for (vector<string>::iterator it = m_filetypes.begin();
it != m_filetypes.end(); it++) {
string term = wrap_prefix(mimetype_prefix) + *it;
LOGDEB0(("Adding file type term: [%s]\n", term.c_str()));
LOGDEB0("Adding file type term: [" << (term) << "]\n" );
tq = tq.empty() ? Xapian::Query(term) :
Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
}
@@ -282,7 +278,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
for (vector<string>::iterator it = m_nfiletypes.begin();
it != m_nfiletypes.end(); it++) {
string term = wrap_prefix(mimetype_prefix) + *it;
LOGDEB0(("Adding negative file type term: [%s]\n", term.c_str()));
LOGDEB0("Adding negative file type term: [" << (term) << "]\n" );
tq = tq.empty() ? Xapian::Query(term) :
Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
}
@@ -337,8 +333,7 @@ public:
if (m_lastpos < pos)
m_lastpos = pos;
bool noexpand = be ? m_ts->nostemexp() : true;
LOGDEB1(("TermProcQ::takeword: pushing [%s] pos %d noexp %d\n",
term.c_str(), pos, noexpand));
LOGDEB1("TermProcQ::takeword: pushing [" << (term) << "] pos " << (pos) << " noexp " << (noexpand) << "\n" );
if (m_terms[pos].size() < term.size()) {
m_terms[pos] = term;
m_nste[pos] = noexpand;
@@ -401,8 +396,7 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
vector<string>* multiwords
)
{
LOGDEB0(("expandTerm: mods 0x%x fld [%s] trm [%s] lang [%s]\n",
mods, m_field.c_str(), term.c_str(), getStemLang().c_str()));
LOGDEB0("expandTerm: mods 0x" << (mods) << " fld [" << (m_field) << "] trm [" << (term) << "] lang [" << (getStemLang()) << "]\n" );
sterm.clear();
oexp.clear();
if (term.empty())
@@ -426,7 +420,7 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
// No stem expansion if there are wildcards or if prevented by caller
bool nostemexp = (mods & SDCM_NOSTEMMING) != 0;
if (haswild || getStemLang().empty()) {
LOGDEB2(("expandTerm: found wildcards or stemlang empty: no exp\n"));
LOGDEB2("expandTerm: found wildcards or stemlang empty: no exp\n" );
nostemexp = true;
}
@@ -448,7 +442,7 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
// performed (conversion+comparison) will automatically ignore
// accented characters which are actually a separate letter
if (getAutoDiac() && unachasaccents(term)) {
LOGDEB0(("expandTerm: term has accents -> diac-sensitive\n"));
LOGDEB0("expandTerm: term has accents -> diac-sensitive\n" );
diac_sensitive = true;
}
@@ -459,14 +453,13 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
Utf8Iter it(term);
it++;
if (getAutoCase() && unachasuppercase(term.substr(it.getBpos()))) {
LOGDEB0(("expandTerm: term has uppercase -> case-sensitive\n"));
LOGDEB0("expandTerm: term has uppercase -> case-sensitive\n" );
case_sensitive = true;
}
// If we are sensitive to case or diacritics turn stemming off
if (diac_sensitive || case_sensitive) {
LOGDEB0(("expandTerm: diac or case sens set -> "
"stemexpand and synonyms off\n"));
LOGDEB0("expandTerm: diac or case sens set -> stemexpand and synonyms off\n" );
nostemexp = true;
synonyms = false;
}
@@ -479,8 +472,7 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
if (noexpansion) {
oexp.push_back(prefix + term);
m_hldata.terms[term] = term;
LOGDEB(("ExpandTerm: noexpansion: final: %s\n",
stringsToString(oexp).c_str()));
LOGDEB("ExpandTerm: noexpansion: final: " << (stringsToString(oexp)) << "\n" );
return true;
}
@@ -521,7 +513,7 @@ bool SearchDataClauseSimple::expandTerm(Rcl::Db &db,
it != oexp.end(); it++) {
m_hldata.terms[strip_prefix(*it)] = term;
}
LOGDEB(("ExpandTerm: final: %s\n", stringsToString(oexp).c_str()));
LOGDEB("ExpandTerm: final: " << (stringsToString(oexp)) << "\n" );
return true;
}
@@ -572,8 +564,7 @@ processSimpleSpan(Rcl::Db &db, string& ermsg,
int mods, void * pq)
{
vector<Xapian::Query>& pqueries(*(vector<Xapian::Query>*)pq);
LOGDEB0(("StringToXapianQ::processSimpleSpan: [%s] mods 0x%x\n",
span.c_str(), (unsigned int)mods));
LOGDEB0("StringToXapianQ::processSimpleSpan: [" << (span) << "] mods 0x" << ((unsigned int)mods) << "\n" );
vector<string> exp;
string sterm; // dumb version of user term
@@ -669,7 +660,7 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
vector<bool>::const_iterator nxit = splitData->nostemexps().begin();
for (vector<string>::const_iterator it = splitData->terms().begin();
it != splitData->terms().end(); it++, nxit++) {
LOGDEB0(("ProcessPhrase: processing [%s]\n", it->c_str()));
LOGDEB0("ProcessPhrase: processing [" << *it << "]\n" );
// Adjust when we do stem expansion. Not if disabled by
// caller, not inside phrases, and some versions of xapian
// will accept only one OR clause inside NEAR.
@@ -685,8 +676,7 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
vector<string> exp;
if (!expandTerm(db, ermsg, lmods, *it, exp, sterm, prefix))
return;
LOGDEB0(("ProcessPhraseOrNear: exp size %d, exp: %s\n", exp.size(),
stringsToString(exp).c_str()));
LOGDEB0("ProcessPhraseOrNear: exp size " << (exp.size()) << ", exp: " << (stringsToString(exp)) << "\n" );
// groups is used for highlighting, we don't want prefixes in there.
vector<string> noprefs;
for (vector<string>::const_iterator it = exp.begin();
@@ -712,8 +702,7 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
// Generate an appropriate PHRASE/NEAR query with adjusted slack
// For phrases, give a relevance boost like we do for original terms
LOGDEB2(("PHRASE/NEAR: alltermcount %d lastpos %d\n",
splitData->alltermcount(), splitData->lastpos()));
LOGDEB2("PHRASE/NEAR: alltermcount " << (splitData->alltermcount()) << " lastpos " << (splitData->lastpos()) << "\n" );
Xapian::Query xq(op, orqueries.begin(), orqueries.end(),
splitData->lastpos() + 1 + slack);
if (op == Xapian::Query::OP_PHRASE)
@@ -783,9 +772,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
vector<Xapian::Query> &pqueries(*(vector<Xapian::Query>*)pq);
int mods = m_modifiers;
LOGDEB(("StringToXapianQ:pUS:: qstr [%s] fld [%s] mods 0x%x "
"slack %d near %d\n",
iq.c_str(), m_field.c_str(), mods, slack, useNear));
LOGDEB("StringToXapianQ:pUS:: qstr [" << (iq) << "] fld [" << (m_field) << "] mods 0x" << (mods) << " slack " << (slack) << " near " << (useNear) << "\n" );
ermsg.erase();
m_curcl = 0;
const StopList stops = db.getStopList();
@@ -805,7 +792,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
try {
for (vector<string>::iterator it = phrases.begin();
it != phrases.end(); it++) {
LOGDEB0(("strToXapianQ: phrase/word: [%s]\n", it->c_str()));
LOGDEB0("strToXapianQ: phrase/word: [" << *it << "]\n" );
// Anchoring modifiers
int amods = stringToMods(*it);
int terminc = amods != 0 ? 1 : 0;
@@ -843,7 +830,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
slack += tpq.lastpos() - int(tpq.terms().size()) + 1;
LOGDEB0(("strToXapianQ: termcount: %d\n", tpq.terms().size()));
LOGDEB0("strToXapianQ: termcount: " << (tpq.terms().size()) << "\n" );
switch (tpq.terms().size() + terminc) {
case 0:
continue;// ??
@@ -878,7 +865,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
ermsg = "Caught unknown exception";
}
if (!ermsg.empty()) {
LOGERR(("stringToXapianQueries: %s\n", ermsg.c_str()));
LOGERR("stringToXapianQueries: " << (ermsg) << "\n" );
return false;
}
return true;
@@ -887,9 +874,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
// Translate a simple OR or AND search clause.
bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB(("SearchDataClauseSimple::toNativeQuery: fld [%s] val [%s] "
"stemlang [%s]\n", m_field.c_str(), m_text.c_str(),
getStemLang().c_str()));
LOGDEB("SearchDataClauseSimple::toNativeQuery: fld [" << (m_field) << "] val [" << (m_text) << "] stemlang [" << (getStemLang()) << "]\n" );
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
@@ -899,7 +884,7 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
case SCLT_AND: op = Xapian::Query::OP_AND; break;
case SCLT_OR: op = Xapian::Query::OP_OR; break;
default:
LOGERR(("SearchDataClauseSimple: bad m_tp %d\n", m_tp));
LOGERR("SearchDataClauseSimple: bad m_tp " << (m_tp) << "\n" );
m_reason = "Internal error";
return false;
}
@@ -908,7 +893,7 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
if (!processUserString(db, m_text, m_reason, &pqueries))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseSimple: resolved to null query\n"));
LOGERR("SearchDataClauseSimple: resolved to null query\n" );
m_reason = string("Resolved to null query. Term too long ? : [" +
m_text + string("]"));
return false;
@@ -952,12 +937,12 @@ bool SearchDataClauseFilename::toNativeQuery(Rcl::Db &db, void *p)
// Translate a dir: path filtering clause. See comments in .h
bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB(("SearchDataClausePath::toNativeQuery: [%s]\n", m_text.c_str()));
LOGDEB("SearchDataClausePath::toNativeQuery: [" << (m_text) << "]\n" );
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
if (m_text.empty()) {
LOGERR(("SearchDataClausePath: empty path??\n"));
LOGERR("SearchDataClausePath: empty path??\n" );
m_reason = "Empty path ?";
return false;
}
@@ -982,8 +967,7 @@ bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
*pit, exp, sterm, wrap_prefix(pathelt_prefix))) {
return false;
}
LOGDEB0(("SDataPath::toNative: exp size %d. Exp: %s\n", exp.size(),
stringsToString(exp).c_str()));
LOGDEB0("SDataPath::toNative: exp size " << (exp.size()) << ". Exp: " << (stringsToString(exp)) << "\n" );
if (exp.size() == 1)
orqueries.push_back(Xapian::Query(exp[0]));
else
@@ -1006,7 +990,7 @@ bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
// Translate NEAR or PHRASE clause.
bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB(("SearchDataClauseDist::toNativeQuery\n"));
LOGDEB("SearchDataClauseDist::toNativeQuery\n" );
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
@@ -1025,7 +1009,7 @@ bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
if (!processUserString(db, s, m_reason, &pqueries, m_slack, useNear))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseDist: resolved to null query\n"));
LOGERR("SearchDataClauseDist: resolved to null query\n" );
m_reason = string("Resolved to null query. Term too long ? : [" +
m_text + string("]"));
return false;
@@ -1039,3 +1023,4 @@ bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
}
} // Namespace Rcl
src/rcldb/searchdataxml.cpp
+6 -4
View File
@@ -25,12 +25,13 @@
#include <string>
#include <vector>
#include <sstream>
using namespace std;
#include "searchdata.h"
#include "debuglog.h"
#include "log.h"
#include "base64.h"
using namespace std;
namespace Rcl {
static string tpToString(SClType tp)
@@ -48,7 +49,7 @@ static string tpToString(SClType tp)
string SearchData::asXML()
{
LOGDEB(("SearchData::asXML\n"));
LOGDEB("SearchData::asXML\n" );
ostringstream os;
// Searchdata
@@ -64,7 +65,7 @@ string SearchData::asXML()
for (unsigned int i = 0; i < m_query.size(); i++) {
SearchDataClause *c = m_query[i];
if (c->getTp() == SCLT_SUB) {
LOGERR(("SearchData::asXML: can't do subclauses !\n"));
LOGERR("SearchData::asXML: can't do subclauses !\n" );
continue;
}
if (c->getTp() == SCLT_PATH) {
@@ -159,3 +160,4 @@ string SearchData::asXML()
}
src/rcldb/stemdb.cpp
+3 -3
View File
@@ -33,7 +33,7 @@ using namespace std;
#include <xapian.h>
#include "stemdb.h"
#include "debuglog.h"
#include "log.h"
#include "smallut.h"
#include "synfamily.h"
#include "unacpp.h"
@@ -85,10 +85,10 @@ bool StemDb::stemExpand(const std::string& langs, const std::string& _term,
sort(result.begin(), result.end());
vector<string>::iterator uit = unique(result.begin(), result.end());
result.resize(uit - result.begin());
LOGDEB1(("stemExpand:%s: %s -> %s\n", langs.c_str(), term.c_str(),
stringsToString(result).c_str()));
LOGDEB1("stemExpand:" << (langs) << ": " << (term) << " -> " << (stringsToString(result)) << "\n" );
return true;
}
}
src/rcldb/stemdb.h
+2 -2
View File
@@ -73,8 +73,7 @@ public:
virtual std::string operator()(const std::string& in)
{
string out = m_stemmer(in);
LOGDEB2(("SynTermTransStem(%s): in [%s] out [%s]\n", m_lang.c_str(),
in.c_str(), out.c_str()));
LOGDEB2("SynTermTransStem(" << (m_lang) << "): in [" << (in) << "] out [" << (out) << "]\n" );
return out;
}
Xapian::Stem m_stemmer;
@@ -102,3 +101,4 @@ public:
}
#endif /* _STEMDB_H_INCLUDED_ */
src/rcldb/stoplist.cpp
+3 -3
View File
@@ -16,7 +16,7 @@
*/
#ifndef TEST_STOPLIST
#include "debuglog.h"
#include "log.h"
#include "readfile.h"
#include "unacpp.h"
#include "smallut.h"
@@ -32,8 +32,7 @@ bool StopList::setFile(const string &filename)
m_stops.clear();
string stoptext, reason;
if (!file_to_string(filename, stoptext, &reason)) {
LOGDEB0(("StopList::StopList: file_to_string(%s) failed: %s\n",
filename.c_str(), reason.c_str()));
LOGDEB0("StopList::StopList: file_to_string(" << (filename) << ") failed: " << (reason) << "\n" );
return false;
}
set<string> stops;
@@ -116,3 +115,4 @@ int main(int argc, char **argv)
}
#endif // TEST_STOPLIST
src/rcldb/synfamily.cpp
+23 -35
View File
@@ -22,7 +22,7 @@
#include <algorithm>
#include MEMORY_INCLUDE
#include "debuglog.h"
#include "log.h"
#include "cstr.h"
#include "xmacros.h"
#include "synfamily.h"
@@ -39,7 +39,7 @@ bool XapWritableSynFamily::createMember(const string& membername)
m_wdb.add_synonym(memberskey(), membername);
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapSynFamily::createMember: error: %s\n", ermsg.c_str()));
LOGERR("XapSynFamily::createMember: error: " << (ermsg) << "\n" );
return false;
}
return true;
@@ -68,7 +68,7 @@ bool XapSynFamily::getMembers(vector<string>& members)
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapSynFamily::getMembers: xapian error %s\n", ermsg.c_str()));
LOGERR("XapSynFamily::getMembers: xapian error " << (ermsg) << "\n" );
return false;
}
return true;
@@ -90,7 +90,7 @@ bool XapSynFamily::listMap(const string& membername)
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapSynFamily::listMap: xapian error %s\n", ermsg.c_str()));
LOGERR("XapSynFamily::listMap: xapian error " << (ermsg) << "\n" );
return false;
}
vector<string>members;
@@ -107,21 +107,19 @@ bool XapSynFamily::listMap(const string& membername)
bool XapSynFamily::synExpand(const string& member, const string& term,
vector<string>& result)
{
LOGDEB(("XapSynFamily::synExpand:(%s) %s for %s\n",
m_prefix1.c_str(), term.c_str(), member.c_str()));
LOGDEB("XapSynFamily::synExpand:(" << (m_prefix1) << ") " << (term) << " for " << (member) << "\n" );
string key = entryprefix(member) + term;
string ermsg;
try {
for (Xapian::TermIterator xit = m_rdb.synonyms_begin(key);
xit != m_rdb.synonyms_end(key); xit++) {
LOGDEB2((" Pushing %s\n", (*xit).c_str()));
LOGDEB2(" Pushing " << ((*xit)) << "\n" );
result.push_back(*xit);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("synFamily::synExpand: error for member [%s] term [%s]\n",
member.c_str(), term.c_str()));
LOGERR("synFamily::synExpand: error for member [" << (member) << "] term [" << (term) << "]\n" );
result.push_back(term);
return false;
}
@@ -144,43 +142,37 @@ bool XapComputableSynFamMember::synExpand(const string& term,
string key = m_prefix + root;
LOGDEB(("XapCompSynFamMbr::synExpand([%s]): term [%s] root [%s] "
"m_trans: %s filter: %s\n",
m_prefix.c_str(), term.c_str(), root.c_str(),
m_trans->name().c_str(),
filtertrans ? filtertrans->name().c_str() : "none"));
LOGDEB("XapCompSynFamMbr::synExpand([" << (m_prefix) << "]): term [" << (term) << "] root [" << (root) << "] m_trans: " << (m_trans->name()) << " filter: " << (filtertrans ? filtertrans->name() : "none") << "\n" );
string ermsg;
try {
for (Xapian::TermIterator xit = m_family.getdb().synonyms_begin(key);
xit != m_family.getdb().synonyms_end(key); xit++) {
if (!filtertrans || (*filtertrans)(*xit) == filter_root) {
LOGDEB2((" Pushing %s\n", (*xit).c_str()));
LOGDEB2(" Pushing " << ((*xit)) << "\n" );
result.push_back(*xit);
}
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapSynDb::synExpand: error for term [%s] (key %s)\n",
term.c_str(), key.c_str()));
LOGERR("XapSynDb::synExpand: error for term [" << (term) << "] (key " << (key) << ")\n" );
result.push_back(term);
return false;
}
// If the input term and root are not in the list, add them
if (find(result.begin(), result.end(), term) == result.end()) {
LOGDEB2((" Pushing %s\n", term.c_str()));
LOGDEB2(" Pushing " << (term) << "\n" );
result.push_back(term);
}
if (root != term &&
find(result.begin(), result.end(), root) == result.end()) {
if (!filtertrans || (*filtertrans)(root) == filter_root) {
LOGDEB2((" Pushing %s\n", root.c_str()));
LOGDEB2(" Pushing " << (root) << "\n" );
result.push_back(root);
}
}
LOGDEB(("XapCompSynFamMbr::synExpand([%s]): term [%s] -> [%s]\n",
m_prefix.c_str(), term.c_str(), stringsToString(result).c_str()));
LOGDEB("XapCompSynFamMbr::synExpand([" << (m_prefix) << "]): term [" << (term) << "] -> [" << (stringsToString(result)) << "]\n" );
return true;
}
@@ -188,7 +180,7 @@ bool XapComputableSynFamMember::synKeyExpand(StrMatcher* inexp,
vector<string>& result,
SynTermTrans *filtertrans)
{
LOGDEB(("XapCompSynFam::synKeyExpand: [%s]\n", inexp->exp().c_str()));
LOGDEB("XapCompSynFam::synKeyExpand: [" << (inexp->exp()) << "]\n" );
// If set, compute filtering term (e.g.: only case-folded)
STD_SHARED_PTR<StrMatcher> filter_exp;
@@ -204,14 +196,13 @@ bool XapComputableSynFamMember::synKeyExpand(StrMatcher* inexp,
string::size_type es = inexp->baseprefixlen();
string is = inexp->exp().substr(0, es);
string::size_type preflen = m_prefix.size();
LOGDEB2(("XapCompSynFam::synKeyExpand: init section: [%s]\n", is.c_str()));
LOGDEB2("XapCompSynFam::synKeyExpand: init section: [" << (is) << "]\n" );
string ermsg;
try {
for (Xapian::TermIterator xit = m_family.getdb().synonym_keys_begin(is);
xit != m_family.getdb().synonym_keys_end(is); xit++) {
LOGDEB2((" Checking1 [%s] against [%s]\n", (*xit).c_str(),
inexp->exp().c_str()));
LOGDEB2(" Checking1 [" << ((*xit)) << "] against [" << (inexp->exp()) << "]\n" );
if (!inexp->match(*xit))
continue;
@@ -222,36 +213,32 @@ bool XapComputableSynFamMember::synKeyExpand(StrMatcher* inexp,
string term = *xit1;
if (filter_exp) {
string term1 = (*filtertrans)(term);
LOGDEB2((" Testing [%s] against [%s]\n",
term1.c_str(), filter_exp->exp().c_str()));
LOGDEB2(" Testing [" << (term1) << "] against [" << (filter_exp->exp()) << "]\n" );
if (!filter_exp->match(term1)) {
continue;
}
}
LOGDEB2(("XapCompSynFam::keyWildExpand: [%s]\n",
(*xit1).c_str()));
LOGDEB2("XapCompSynFam::keyWildExpand: [" << ((*xit1)) << "]\n" );
result.push_back(*xit1);
}
// Same with key itself
string term = (*xit).substr(preflen);
if (filter_exp) {
string term1 = (*filtertrans)(term);
LOGDEB2((" Testing [%s] against [%s]\n",
term1.c_str(), filter_exp->exp().c_str()));
LOGDEB2(" Testing [" << (term1) << "] against [" << (filter_exp->exp()) << "]\n" );
if (!filter_exp->match(term1)) {
continue;
}
}
LOGDEB2(("XapCompSynFam::keyWildExpand: [%s]\n", term.c_str()));
LOGDEB2("XapCompSynFam::keyWildExpand: [" << (term) << "]\n" );
result.push_back(term);
}
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapCompSynFam::synKeyExpand: xapian: [%s]\n", ermsg.c_str()));
LOGERR("XapCompSynFam::synKeyExpand: xapian: [" << (ermsg) << "]\n" );
return false;
}
LOGDEB1(("XapCompSynFam::synKeyExpand: final: [%s]\n",
stringsToString(result).c_str()));
LOGDEB1("XapCompSynFam::synKeyExpand: final: [" << (stringsToString(result)) << "]\n" );
return true;
}
@@ -401,3 +388,4 @@ int main(int argc, char **argv)
}
#endif // TEST_SYNFAMILY
src/rcldb/synfamily.h
+5 -6
View File
@@ -37,7 +37,7 @@
#include <xapian.h>
#include "debuglog.h"
#include "log.h"
#include "xmacros.h"
#include "strmatcher.h"
@@ -168,10 +168,9 @@ public:
virtual bool addSynonym(const std::string& term)
{
LOGDEB2(("addSynonym:me %p term [%s] m_trans %p\n", this,
term.c_str(), m_trans));
LOGDEB2("addSynonym:me " << (this) << " term [" << (term) << "] m_trans " << (m_trans) << "\n" );
std::string transformed = (*m_trans)(term);
LOGDEB2(("addSynonym: transformed [%s]\n", transformed.c_str()));
LOGDEB2("addSynonym: transformed [" << (transformed) << "]\n" );
if (transformed == term)
return true;
@@ -180,8 +179,7 @@ public:
m_family.getdb().add_synonym(m_prefix + transformed, term);
} XCATCHERROR(ermsg);
if (!ermsg.empty()) {
LOGERR(("XapWritableComputableSynFamMember::addSynonym: "
"xapian error %s\n", ermsg.c_str()));
LOGERR("XapWritableComputableSynFamMember::addSynonym: xapian error " << (ermsg) << "\n" );
return false;
}
return true;
@@ -224,3 +222,4 @@ static const std::string synFamDiCa("DCa");
} // end namespace Rcl
#endif /* _SYNFAMILY_H_INCLUDED_ */