tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implemented range queries, based on storing fields in xapian values

Browse Source
This commit is contained in:
Jean-Francois Dockes
2018-01-24 09:43:20 +01:00
parent 26d15dbe4a
commit 595e419d93
20 changed files with 875 additions and 428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+33 -21
View File
@@ -50,6 +50,7 @@ using namespace std;
#include "searchdata.h"
#include "rclquery.h"
#include "rclquery_p.h"
#include "rclvalues.h"
#include "md5ut.h"
#include "rclversion.h"
#include "cancelcheck.h"
@@ -1489,7 +1490,8 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi, Doc &doc)
for (vector<string>::iterator it = vpath.begin();
it != vpath.end(); it++){
if (it->length() > 230) {
// Just truncate it. May still be useful because of wildcards
// Just truncate it. May still be useful because
// of wildcards
*it = it->substr(0, 230);
}
newdocument.add_posting(wrap_prefix(pathelt_prefix) + *it,
@@ -1504,26 +1506,36 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi, Doc &doc)
//
// The order has no importance, and we set a position gap of 100
// between fields to avoid false proximity matches.
map<string, string>::iterator meta_it;
for (meta_it = doc.meta.begin(); meta_it != doc.meta.end(); meta_it++) {
if (!meta_it->second.empty()) {
const FieldTraits *ftp;
// We don't test for an empty prefix here. Some fields are part
// of the internal conf with an empty prefix (ie: abstract).
if (!fieldToTraits(meta_it->first, &ftp)) {
LOGDEB0("Db::add: no prefix for field [" <<
meta_it->first << "], no indexing\n");
continue;
}
LOGDEB0("Db::add: field [" << meta_it->first << "] pfx [" <<
for (const auto& entry: doc.meta) {
if (entry.second.empty()) {
continue;
}
const FieldTraits *ftp{nullptr};
fieldToTraits(entry.first, &ftp);
if (ftp && ftp->valueslot) {
LOGDEB("Adding value: for field " << entry.first << " slot "
<< ftp->valueslot << endl);
add_field_value(newdocument, *ftp, entry.second);
}
// There was an old comment here about not testing for
// empty prefix, and we indeed did not test. I don't think
// that it makes sense any more (and was in disagreement
// with the LOG message. Really now: no prefix: no
// indexing.
if (ftp && !ftp->pfx.empty()) {
LOGDEB0("Db::add: field [" << entry.first << "] pfx [" <<
ftp->pfx << "] inc " << ftp->wdfinc << ": [" <<
meta_it->second << "]\n");
entry.second << "]\n");
splitter.setTraits(*ftp);
if (!splitter.text_to_words(meta_it->second)) {
LOGDEB("Db::addOrUpdate: split failed for " <<
meta_it->first << "\n");
if (!splitter.text_to_words(entry.second)) {
LOGDEB("Db::addOrUpdate: split failed for " <<
entry.first << "\n");
}
}
} else {
LOGDEB0("Db::add: no prefix for field [" <<
entry.first << "], no indexing\n");
}
}
// Reset to no prefix and default params
@@ -1578,8 +1590,8 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi, Doc &doc)
if (doc.getmeta(Doc::keyfn, &utf8fn) && !utf8fn.empty()) {
string fn;
if (unacmaybefold(utf8fn, fn, "UTF-8", UNACOP_UNACFOLD)) {
// We should truncate after extracting the extension, but this is
// a pathological case anyway
// We should truncate after extracting the extension,
// but this is a pathological case anyway
if (fn.size() > 230)
utf8truncate(fn, 230);
string::size_type pos = fn.rfind('.');
@@ -1587,7 +1599,7 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi, Doc &doc)
newdocument.add_boolean_term(wrap_prefix(fileext_prefix) +
fn.substr(pos + 1));
}
newdocument.add_term(wrap_prefix(unsplitfilename_prefix) + fn, 0);
newdocument.add_term(wrap_prefix(unsplitfilename_prefix) + fn,0);
}
}
src/rcldb/rcldb_p.h
+1 -1
View File
@@ -197,7 +197,7 @@ class Db::Native {
std::string rawtextMetaKey(Xapian::docid did) {
// Xapian's Olly Betts avises to use a key which will
// sort the same as the docid (which we do), and to
// use Xapian's pack_uint_preserving_sort() which is
// use Xapian's pack.h:pack_uint_preserving_sort() which is
// efficient but hard to read. I'd wager that this
// does not make much of a difference. 10 ascii bytes
// gives us 10 billion docs, which is enough (says I).
src/rcldb/rclvalues.cpp
+87
View File
@@ -0,0 +1,87 @@
/* Copyright (C) 2004-2018 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include "autoconfig.h"
#include <string>
#include "xapian.h"
#include "rclconfig.h"
#include "smallut.h"
#include "log.h"
using namespace std;
namespace Rcl {
void add_field_value(Xapian::Document& xdoc, const FieldTraits& ft,
const string& data)
{
string ndata{data};
switch (ft.valuetype) {
case FieldTraits::STR:
break;
case FieldTraits::INT:
{
int len = ft.valuelen ? ft.valuelen : 10;
leftzeropad(ndata, len);
}
}
LOGDEB0("Rcl::add_field_value: slot " << ft.valueslot << " [" <<
ndata << "]\n");
xdoc.add_value(ft.valueslot, ndata);
}
string convert_field_value(const FieldTraits& ft,
const string& data)
{
string ndata(data);
switch (ft.valuetype) {
case FieldTraits::STR:
break;
case FieldTraits::INT:
{
if (ndata.empty())
break;
// Apply suffixes
char c = ndata.back();
string zeroes;
switch(c) {
case 'k':case 'K': zeroes = "000";break;
case 'm':case 'M': zeroes = "000000";break;
case 'g':case 'G': zeroes = "000000000";break;
case 't':case 'T': zeroes = "000000000000";break;
default: break;
}
if (!zeroes.empty()) {
ndata.pop_back();
ndata += zeroes;
}
int len = ft.valuelen ? ft.valuelen : 10;
leftzeropad(ndata, len);
}
}
return ndata;
}
}
src/rcldb/rclvalues.h
+29
View File
@@ -0,0 +1,29 @@
#ifndef _RCLVALUES_H_INCLUDED_
#define _RCLVALUES_H_INCLUDED_
/* Copyright (C) 2004-2018 J.F.Dockes
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the
* Free Software Foundation, Inc.,
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include <string>
namespace Rcl {
extern void add_field_value(Xapian::Document& xdoc, const FieldTraits& ft,
const std::string& data);
extern std::string convert_field_value(const FieldTraits& ft,
const std::string& data);
}
#endif /* _RCLVALUES_H_INCLUDED_ */
src/rcldb/searchdata.cpp
+8
View File
@@ -348,6 +348,14 @@ void SearchDataClausePath::dump(ostream& o) const
o << "[" << m_text << "]";
}
void SearchDataClauseRange::dump(ostream& o) const
{
o << "ClauseRange: ";
if (m_exclude)
o << " - ";
o << "[" << gettext() << "]";
}
void SearchDataClauseDist::dump(ostream& o) const
{
if (m_tp == SCLT_NEAR)
src/rcldb/searchdata.h
+117 -139
View File
@@ -41,9 +41,8 @@ namespace Rcl {
/** Search clause types */
enum SClType {
SCLT_AND,
SCLT_OR, SCLT_FILENAME, SCLT_PHRASE, SCLT_NEAR, SCLT_PATH,
SCLT_SUB
SCLT_AND, SCLT_OR, SCLT_FILENAME, SCLT_PHRASE, SCLT_NEAR,
SCLT_PATH, SCLT_RANGE, SCLT_SUB,
};
class SearchDataClause;
@@ -79,16 +78,14 @@ class SearchDataClauseDist;
class SearchData {
public:
SearchData(SClType tp, const string& stemlang)
: m_tp(tp), m_stemlang(stemlang)
{
if (m_tp != SCLT_OR && m_tp != SCLT_AND)
m_tp = SCLT_OR;
commoninit();
: m_tp(tp), m_stemlang(stemlang) {
if (m_tp != SCLT_OR && m_tp != SCLT_AND)
m_tp = SCLT_OR;
commoninit();
}
SearchData()
: m_tp(SCLT_AND)
{
commoninit();
: m_tp(SCLT_AND) {
commoninit();
}
~SearchData();
@@ -110,7 +107,7 @@ public:
* user terms in order will have higher relevance. This must be called
* before toNativeQuery().
* @param threshold: don't use terms more frequent than the value
* (proportion of docs where they occur)
* (proportion of docs where they occur)
*/
bool maybeAddAutoPhrase(Rcl::Db &db, double threshold);
@@ -142,21 +139,19 @@ public:
void setDescription(const std::string& d) {m_description = d;}
/** Return an XML version of the contents, for storage in search history
by the GUI */
by the GUI */
string asXML();
void setTp(SClType tp)
{
m_tp = tp;
void setTp(SClType tp) {
m_tp = tp;
}
SClType getTp() {
return m_tp;
}
void setMaxExpand(int max)
{
m_softmaxexpand = max;
void setMaxExpand(int max) {
m_softmaxexpand = max;
}
bool getAutoDiac() {return m_autodiacsens;}
bool getAutoCase() {return m_autocasesens;}
@@ -217,8 +212,8 @@ private:
bool expandFileTypes(Rcl::Db &db, std::vector<std::string>& exptps);
bool clausesToQuery(Rcl::Db &db, SClType tp,
std::vector<SearchDataClause*>& query,
string& reason, void *d);
std::vector<SearchDataClause*>& query,
string& reason, void *d);
void commoninit();
/* Copyconst and assignment private and forbidden */
@@ -229,9 +224,9 @@ private:
class SearchDataClause {
public:
enum Modifier {SDCM_NONE=0, SDCM_NOSTEMMING=0x1, SDCM_ANCHORSTART=0x2,
SDCM_ANCHOREND=0x4, SDCM_CASESENS=0x8, SDCM_DIACSENS=0x10,
SDCM_NOTERMS=0x20, // Don't include terms for highlighting
SDCM_NOSYNS = 0x40, // Don't perform synonym expansion
SDCM_ANCHOREND=0x4, SDCM_CASESENS=0x8, SDCM_DIACSENS=0x10,
SDCM_NOTERMS=0x20, // Don't include terms for highlighting
SDCM_NOSYNS = 0x40, // Don't perform synonym expansion
// Aargh special case. pathelts are case/diac-sensitive
// even in a stripped index
SDCM_PATHELT = 0x80,
@@ -239,70 +234,57 @@ public:
enum Relation {REL_CONTAINS, REL_EQUALS, REL_LT, REL_LTE, REL_GT, REL_GTE};
SearchDataClause(SClType tp)
: m_tp(tp), m_parentSearch(0), m_haveWildCards(0),
m_modifiers(SDCM_NONE), m_weight(1.0), m_exclude(false),
m_rel(REL_CONTAINS)
{}
: m_tp(tp), m_parentSearch(0), m_haveWildCards(0),
m_modifiers(SDCM_NONE), m_weight(1.0), m_exclude(false),
m_rel(REL_CONTAINS) {}
virtual ~SearchDataClause() {}
virtual bool toNativeQuery(Rcl::Db &db, void *) = 0;
bool isFileName() const {return m_tp == SCLT_FILENAME ? true: false;}
virtual std::string getReason() const {return m_reason;}
virtual void getTerms(HighlightData&) const {}
SClType getTp() const
{
return m_tp;
SClType getTp() const {
return m_tp;
}
void setTp(SClType tp) {
m_tp = tp;
}
void setParent(SearchData *p)
{
m_parentSearch = p;
void setParent(SearchData *p) {
m_parentSearch = p;
}
string getStemLang()
{
return (m_modifiers & SDCM_NOSTEMMING) || m_parentSearch == 0 ?
cstr_null : m_parentSearch->getStemLang();
string getStemLang() {
return (m_modifiers & SDCM_NOSTEMMING) || m_parentSearch == 0 ?
cstr_null : m_parentSearch->getStemLang();
}
bool getAutoDiac()
{
return m_parentSearch ? m_parentSearch->getAutoDiac() : false;
bool getAutoDiac() {
return m_parentSearch ? m_parentSearch->getAutoDiac() : false;
}
bool getAutoCase()
{
return m_parentSearch ? m_parentSearch->getAutoCase() : true;
bool getAutoCase() {
return m_parentSearch ? m_parentSearch->getAutoCase() : true;
}
int getMaxExp()
{
return m_parentSearch ? m_parentSearch->getMaxExp() : 10000;
int getMaxExp() {
return m_parentSearch ? m_parentSearch->getMaxExp() : 10000;
}
size_t getMaxCl()
{
return m_parentSearch ? m_parentSearch->getMaxCl() : 100000;
size_t getMaxCl() {
return m_parentSearch ? m_parentSearch->getMaxCl() : 100000;
}
int getSoftMaxExp()
{
return m_parentSearch ? m_parentSearch->getSoftMaxExp() : -1;
int getSoftMaxExp() {
return m_parentSearch ? m_parentSearch->getSoftMaxExp() : -1;
}
virtual void addModifier(Modifier mod)
{
m_modifiers = m_modifiers | mod;
virtual void addModifier(Modifier mod) {
m_modifiers = m_modifiers | mod;
}
virtual unsigned int getmodifiers() {
return m_modifiers;
return m_modifiers;
}
virtual void setWeight(float w)
{
m_weight = w;
virtual void setWeight(float w) {
m_weight = w;
}
virtual bool getexclude() const
{
return m_exclude;
virtual bool getexclude() const {
return m_exclude;
}
virtual void setexclude(bool onoff)
{
m_exclude = onoff;
virtual void setexclude(bool onoff) {
m_exclude = onoff;
}
virtual void setrel(Relation rel) {
m_rel = rel;
@@ -322,15 +304,6 @@ protected:
float m_weight;
bool m_exclude;
Relation m_rel;
private:
SearchDataClause(const SearchDataClause&)
{
}
SearchDataClause& operator=(const SearchDataClause&)
{
return *this;
}
};
/**
@@ -341,37 +314,30 @@ class TermProcQ;
class SearchDataClauseSimple : public SearchDataClause {
public:
SearchDataClauseSimple(SClType tp, const std::string& txt,
const std::string& fld = std::string())
: SearchDataClause(tp), m_text(txt), m_field(fld), m_curcl(0)
{
m_haveWildCards =
(txt.find_first_of(cstr_minwilds) != std::string::npos);
const std::string& fld = std::string())
: SearchDataClause(tp), m_text(txt), m_field(fld), m_curcl(0) {
m_haveWildCards =
(txt.find_first_of(cstr_minwilds) != std::string::npos);
}
SearchDataClauseSimple(const std::string& txt, SClType tp)
: SearchDataClause(tp), m_text(txt), m_curcl(0)
{
m_haveWildCards =
(txt.find_first_of(cstr_minwilds) != std::string::npos);
: SearchDataClause(tp), m_text(txt), m_curcl(0) {
m_haveWildCards =
(txt.find_first_of(cstr_minwilds) != std::string::npos);
}
virtual ~SearchDataClauseSimple()
{
}
virtual ~SearchDataClauseSimple() {}
/** Translate to Xapian query */
virtual bool toNativeQuery(Rcl::Db &, void *);
virtual void getTerms(HighlightData& hldata) const
{
hldata.append(m_hldata);
virtual void getTerms(HighlightData& hldata) const {
hldata.append(m_hldata);
}
virtual const std::string& gettext()
{
return m_text;
virtual const std::string& gettext() const {
return m_text;
}
virtual const std::string& getfield()
{
return m_field;
virtual const std::string& getfield() const {
return m_field;
}
virtual void setfield(const string& field) {
m_field = field;
@@ -384,22 +350,49 @@ protected:
HighlightData m_hldata;
// Current count of Xapian clauses, to check against expansion limit
size_t m_curcl;
bool processUserString(Rcl::Db &db, const string &iq,
std::string &ermsg,
void* pq, int slack = 0, bool useNear = false);
std::string &ermsg,
void* pq, int slack = 0, bool useNear = false);
bool expandTerm(Rcl::Db &db, std::string& ermsg, int mods,
const std::string& term,
std::vector<std::string>& exp,
const std::string& term,
std::vector<std::string>& exp,
std::string& sterm, const std::string& prefix,
std::vector<std::string>* multiwords = 0);
std::vector<std::string>* multiwords = 0);
// After splitting entry on whitespace: process non-phrase element
void processSimpleSpan(Rcl::Db &db, string& ermsg, const string& span,
int mods, void *pq);
int mods, void *pq);
// Process phrase/near element
void processPhraseOrNear(Rcl::Db &db, string& ermsg, TermProcQ *splitData,
int mods, void *pq, bool useNear, int slack);
int mods, void *pq, bool useNear, int slack);
};
class SearchDataClauseRange : public SearchDataClauseSimple {
public:
SearchDataClauseRange(const std::string& t1, const std::string& t2,
const std::string& fld = std::string())
: SearchDataClauseSimple(SCLT_RANGE, t1, fld), m_t2(t2) {}
// This is for 'upgrading' a clauseSimple with eq/gt/lt... rel to
// a range. Either of t1 or t2 or both can be set to the original
// text, which is why they are passed as separate parameters
SearchDataClauseRange(const SearchDataClauseSimple& cl,
const std::string& t1, const std::string& t2)
: SearchDataClauseSimple(cl) {
m_text = t1;
m_t2 = t2;
}
virtual ~SearchDataClauseRange() {}
virtual void dump(ostream& o) const;
virtual const std::string& gettext2() const {
return m_t2;
}
virtual bool toNativeQuery(Rcl::Db &db, void *);
protected:
std::string m_t2;
};
/**
* Filename search clause. This is special because term expansion is only
@@ -412,15 +405,12 @@ protected:
class SearchDataClauseFilename : public SearchDataClauseSimple {
public:
SearchDataClauseFilename(const std::string& txt)
: SearchDataClauseSimple(txt, SCLT_FILENAME)
{
// File name searches don't count when looking for wild cards.
m_haveWildCards = false;
: SearchDataClauseSimple(txt, SCLT_FILENAME) {
// File name searches don't count when looking for wild cards.
m_haveWildCards = false;
}
virtual ~SearchDataClauseFilename()
{
}
virtual ~SearchDataClauseFilename() {}
virtual bool toNativeQuery(Rcl::Db &, void *);
virtual void dump(ostream& o) const;
@@ -450,15 +440,12 @@ public:
class SearchDataClausePath : public SearchDataClauseSimple {
public:
SearchDataClausePath(const std::string& txt, bool excl = false)
: SearchDataClauseSimple(SCLT_PATH, txt, "dir")
{
m_exclude = excl;
m_haveWildCards = false;
: SearchDataClauseSimple(SCLT_PATH, txt, "dir") {
m_exclude = excl;
m_haveWildCards = false;
}
virtual ~SearchDataClausePath()
{
}
virtual ~SearchDataClausePath() {}
virtual bool toNativeQuery(Rcl::Db &, void *);
virtual void dump(ostream& o) const;
@@ -471,19 +458,14 @@ public:
class SearchDataClauseDist : public SearchDataClauseSimple {
public:
SearchDataClauseDist(SClType tp, const std::string& txt, int slack,
const std::string& fld = std::string())
: SearchDataClauseSimple(tp, txt, fld), m_slack(slack)
{
}
const std::string& fld = std::string())
: SearchDataClauseSimple(tp, txt, fld), m_slack(slack) {}
virtual ~SearchDataClauseDist()
{
}
virtual ~SearchDataClauseDist() {}
virtual bool toNativeQuery(Rcl::Db &, void *);
virtual int getslack() const
{
return m_slack;
virtual int getslack() const {
return m_slack;
}
virtual void setslack(int slack) {
m_slack = slack;
@@ -497,20 +479,16 @@ private:
class SearchDataClauseSub : public SearchDataClause {
public:
SearchDataClauseSub(std::shared_ptr<SearchData> sub)
: SearchDataClause(SCLT_SUB), m_sub(sub)
{
}
virtual bool toNativeQuery(Rcl::Db &db, void *p)
{
bool ret = m_sub->toNativeQuery(db, p);
if (!ret)
m_reason = m_sub->getReason();
return ret;
: SearchDataClause(SCLT_SUB), m_sub(sub) {}
virtual bool toNativeQuery(Rcl::Db &db, void *p) {
bool ret = m_sub->toNativeQuery(db, p);
if (!ret)
m_reason = m_sub->getReason();
return ret;
}
virtual void getTerms(HighlightData& hldata) const
{
m_sub.get()->getTerms(hldata);
virtual void getTerms(HighlightData& hldata) const {
m_sub.get()->getTerms(hldata);
}
virtual std::shared_ptr<SearchData> getSub() {
return m_sub;
src/rcldb/searchdatatox.cpp
+124 -38
View File
@@ -47,11 +47,10 @@ using namespace std;
#include "expansiondbs.h"
#include "base64.h"
#include "daterange.h"
#include "rclvalues.h"
namespace Rcl {
typedef vector<SearchDataClause *>::iterator qlist_it_t;
static const int original_term_wqf_booster = 10;
// Expand doc categories and mime type wild card expressions
@@ -62,7 +61,7 @@ bool SearchData::expandFileTypes(Db &db, vector<string>& tps)
{
const RclConfig *cfg = db.getConf();
if (!cfg) {
LOGFATAL("Db::expandFileTypes: null configuration!!\n" );
LOGFATAL("Db::expandFileTypes: null configuration!!\n");
return false;
}
vector<string> exptps;
@@ -110,15 +109,16 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
string& reason, void *d)
{
Xapian::Query xq;
for (qlist_it_t it = query.begin(); it != query.end(); it++) {
for (auto& clausep : query) {
Xapian::Query nq;
if (!(*it)->toNativeQuery(db, &nq)) {
LOGERR("SearchData::clausesToQuery: toNativeQuery failed: " << ((*it)->getReason()) << "\n" );
reason += (*it)->getReason() + " ";
if (!clausep->toNativeQuery(db, &nq)) {
LOGERR("SearchData::clausesToQuery: toNativeQuery failed: "
<< clausep->getReason() << "\n");
reason += clausep->getReason() + " ";
return false;
}
if (nq.empty()) {
LOGDEB("SearchData::clausesToQuery: skipping empty clause\n" );
LOGDEB("SearchData::clausesToQuery: skipping empty clause\n");
continue;
}
// If this structure is an AND list, must use AND_NOT for excl clauses.
@@ -126,7 +126,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
// addClause())
Xapian::Query::op op;
if (tp == SCLT_AND) {
if ((*it)->getexclude()) {
if (clausep->getexclude()) {
op = Xapian::Query::OP_AND_NOT;
} else {
op = Xapian::Query::OP_AND;
@@ -143,7 +143,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
xq = Xapian::Query(op, xq, nq);
}
if (int(xq.get_length()) >= getMaxCl()) {
LOGERR("" << (maxXapClauseMsg) << "\n" );
LOGERR("" << maxXapClauseMsg << "\n");
m_reason += maxXapClauseMsg;
if (!o_index_stripchars)
m_reason += maxXapClauseCaseDiacMsg;
@@ -151,7 +151,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
}
}
LOGDEB0("SearchData::clausesToQuery: got " << (xq.get_length()) << " clauses\n" );
LOGDEB0("SearchData::clausesToQuery: got " << xq.get_length()<<" clauses\n");
if (xq.empty())
xq = Xapian::Query::MatchAll;
@@ -162,7 +162,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
{
LOGDEB("SearchData::toNativeQuery: stemlang [" << (m_stemlang) << "]\n" );
LOGDEB("SearchData::toNativeQuery: stemlang [" << m_stemlang << "]\n");
m_reason.erase();
db.getConf()->getConfParam("maxTermExpand", &m_maxexp);
@@ -174,7 +174,8 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
// Xapian query tree
Xapian::Query xq;
if (!clausesToQuery(db, m_tp, m_query, m_reason, &xq)) {
LOGERR("SearchData::toNativeQuery: clausesToQuery failed. reason: " << (m_reason) << "\n" );
LOGERR("SearchData::toNativeQuery: clausesToQuery failed. reason: "
<< m_reason << "\n");
return false;
}
@@ -183,7 +184,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
if (m_dates.y1 == 0 || m_dates.y2 == 0) {
int minyear = 1970, maxyear = 2100;
if (!db.maxYearSpan(&minyear, &maxyear)) {
LOGERR("Can't retrieve index min/max dates\n" );
LOGERR("Can't retrieve index min/max dates\n");
//whatever, go on.
}
@@ -198,16 +199,18 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
m_dates.d2 = 31;
}
}
LOGDEB("Db::toNativeQuery: date interval: " << (m_dates.y1) << "-" << (m_dates.m1) << "-" << (m_dates.d1) << "/" << (m_dates.y2) << "-" << (m_dates.m2) << "-" << (m_dates.d2) << "\n" );
LOGDEB("Db::toNativeQuery: date interval: " << m_dates.y1 <<
"-" << m_dates.m1 << "-" << m_dates.d1 << "/" <<
m_dates.y2 << "-" << m_dates.m2 << "-" << m_dates.d2 << "\n");
Xapian::Query dq = date_range_filter(m_dates.y1, m_dates.m1, m_dates.d1,
m_dates.y2, m_dates.m2, m_dates.d2);
if (dq.empty()) {
LOGINFO("Db::toNativeQuery: date filter is empty\n" );
LOGINFO("Db::toNativeQuery: date filter is empty\n");
}
// If no probabilistic query is provided then promote the daterange
// filter to be THE query instead of filtering an empty query.
if (xq.empty()) {
LOGINFO("Db::toNativeQuery: proba query is empty\n" );
LOGINFO("Db::toNativeQuery: proba query is empty\n");
xq = dq;
} else {
xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, dq);
@@ -239,7 +242,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
// If no probabilistic query is provided then promote the
// filter to be THE query instead of filtering an empty query.
if (xq.empty()) {
LOGINFO("Db::toNativeQuery: proba query is empty\n" );
LOGINFO("Db::toNativeQuery: proba query is empty\n");
xq = sq;
} else {
xq = Xapian::Query(Xapian::Query::OP_FILTER, xq, sq);
@@ -263,7 +266,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
for (vector<string>::iterator it = m_filetypes.begin();
it != m_filetypes.end(); it++) {
string term = wrap_prefix(mimetype_prefix) + *it;
LOGDEB0("Adding file type term: [" << (term) << "]\n" );
LOGDEB0("Adding file type term: [" << term << "]\n");
tq = tq.empty() ? Xapian::Query(term) :
Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
}
@@ -278,7 +281,7 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
for (vector<string>::iterator it = m_nfiletypes.begin();
it != m_nfiletypes.end(); it++) {
string term = wrap_prefix(mimetype_prefix) + *it;
LOGDEB0("Adding negative file type term: [" << (term) << "]\n" );
LOGDEB0("Adding negative file type term: [" << term << "]\n");
tq = tq.empty() ? Xapian::Query(term) :
Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term));
}
@@ -333,7 +336,8 @@ public:
if (m_lastpos < pos)
m_lastpos = pos;
bool noexpand = be ? m_ts->nostemexp() : true;
LOGDEB1("TermProcQ::takeword: pushing [" << (term) << "] pos " << (pos) << " noexp " << (noexpand) << "\n" );
LOGDEB1("TermProcQ::takeword: pushing [" << term << "] pos " <<
pos << " noexp " << noexpand << "\n");
if (m_terms[pos].size() < term.size()) {
m_terms[pos] = term;
m_nste[pos] = noexpand;
@@ -577,13 +581,12 @@ static void prefix_vector(vector<string>& v, const string& prefix)
}
}
void SearchDataClauseSimple::
processSimpleSpan(Rcl::Db &db, string& ermsg,
const string& span,
int mods, void * pq)
void SearchDataClauseSimple::processSimpleSpan(
Rcl::Db &db, string& ermsg, const string& span, int mods, void *pq)
{
vector<Xapian::Query>& pqueries(*(vector<Xapian::Query>*)pq);
LOGDEB0("StringToXapianQ::processSimpleSpan: [" << (span) << "] mods 0x" << ((unsigned int)mods) << "\n" );
LOGDEB0("StringToXapianQ::processSimpleSpan: [" << span << "] mods 0x"
<< (unsigned int)mods << "\n");
vector<string> exp;
string sterm; // dumb version of user term
@@ -679,7 +682,7 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
vector<bool>::const_iterator nxit = splitData->nostemexps().begin();
for (vector<string>::const_iterator it = splitData->terms().begin();
it != splitData->terms().end(); it++, nxit++) {
LOGDEB0("ProcessPhrase: processing [" << *it << "]\n" );
LOGDEB0("ProcessPhrase: processing [" << *it << "]\n");
// Adjust when we do stem expansion. Not if disabled by
// caller, not inside phrases, and some versions of xapian
// will accept only one OR clause inside NEAR.
@@ -695,7 +698,8 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
vector<string> exp;
if (!expandTerm(db, ermsg, lmods, *it, exp, sterm, prefix))
return;
LOGDEB0("ProcessPhraseOrNear: exp size " << (exp.size()) << ", exp: " << (stringsToString(exp)) << "\n" );
LOGDEB0("ProcessPhraseOrNear: exp size " << exp.size() << ", exp: " <<
stringsToString(exp) << "\n");
// groups is used for highlighting, we don't want prefixes in there.
vector<string> noprefs;
for (vector<string>::const_iterator it = exp.begin();
@@ -721,7 +725,8 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
// Generate an appropriate PHRASE/NEAR query with adjusted slack
// For phrases, give a relevance boost like we do for original terms
LOGDEB2("PHRASE/NEAR: alltermcount " << (splitData->alltermcount()) << " lastpos " << (splitData->lastpos()) << "\n" );
LOGDEB2("PHRASE/NEAR: alltermcount " << splitData->alltermcount() <<
" lastpos " << splitData->lastpos() << "\n");
Xapian::Query xq(op, orqueries.begin(), orqueries.end(),
splitData->lastpos() + 1 + slack);
if (op == Xapian::Query::OP_PHRASE)
@@ -791,7 +796,8 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
vector<Xapian::Query> &pqueries(*(vector<Xapian::Query>*)pq);
int mods = m_modifiers;
LOGDEB("StringToXapianQ:pUS:: qstr [" << (iq) << "] fld [" << (m_field) << "] mods 0x" << (mods) << " slack " << (slack) << " near " << (useNear) << "\n" );
LOGDEB("StringToXapianQ:pUS:: qstr [" << iq << "] fld [" << m_field <<
"] mods 0x"<<mods<<" slack " << slack << " near " << useNear <<"\n");
ermsg.erase();
m_curcl = 0;
const StopList stops = db.getStopList();
@@ -811,7 +817,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
try {
for (vector<string>::iterator it = phrases.begin();
it != phrases.end(); it++) {
LOGDEB0("strToXapianQ: phrase/word: [" << *it << "]\n" );
LOGDEB0("strToXapianQ: phrase/word: [" << *it << "]\n");
// Anchoring modifiers
int amods = stringToMods(*it);
int terminc = amods != 0 ? 1 : 0;
@@ -849,7 +855,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
slack += tpq.lastpos() - int(tpq.terms().size()) + 1;
LOGDEB0("strToXapianQ: termcount: " << (tpq.terms().size()) << "\n" );
LOGDEB0("strToXapianQ: termcount: " << tpq.terms().size() << "\n");
switch (tpq.terms().size() + terminc) {
case 0:
continue;// ??
@@ -884,7 +890,7 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
ermsg = "Caught unknown exception";
}
if (!ermsg.empty()) {
LOGERR("stringToXapianQueries: " << (ermsg) << "\n" );
LOGERR("stringToXapianQueries: " << ermsg << "\n");
return false;
}
return true;
@@ -893,8 +899,36 @@ bool SearchDataClauseSimple::processUserString(Rcl::Db &db, const string &iq,
// Translate a simple OR or AND search clause.
bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB("SearchDataClauseSimple::toNativeQuery: fld [" << (m_field) << "] val [" << (m_text) << "] stemlang [" << (getStemLang()) << "]\n" );
LOGDEB("SearchDataClauseSimple::toNativeQuery: fld [" << m_field <<
"] val [" << m_text << "] stemlang [" << getStemLang() << "]\n");
// Transform (in)equalities into a range query
switch (getrel()) {
case REL_EQUALS:
{
SearchDataClauseRange cl(*this, gettext(), gettext());
bool ret = cl.toNativeQuery(db, p);
m_reason = cl.getReason();
return ret;
}
case REL_LT: case REL_LTE:
{
SearchDataClauseRange cl(*this, "", gettext());
bool ret = cl.toNativeQuery(db, p);
m_reason = cl.getReason();
return ret;
}
case REL_GT: case REL_GTE:
{
SearchDataClauseRange cl(*this, gettext(), "");
bool ret = cl.toNativeQuery(db, p);
m_reason = cl.getReason();
return ret;
}
default:
break;
}
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
@@ -903,7 +937,7 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
case SCLT_AND: op = Xapian::Query::OP_AND; break;
case SCLT_OR: op = Xapian::Query::OP_OR; break;
default:
LOGERR("SearchDataClauseSimple: bad m_tp " << (m_tp) << "\n" );
LOGERR("SearchDataClauseSimple: bad m_tp " << m_tp << "\n");
m_reason = "Internal error";
return false;
}
@@ -912,7 +946,7 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
if (!processUserString(db, m_text, m_reason, &pqueries))
return false;
if (pqueries.empty()) {
LOGERR("SearchDataClauseSimple: resolved to null query\n" );
LOGERR("SearchDataClauseSimple: resolved to null query\n");
m_reason = string("Resolved to null query. Term too long ? : [" +
m_text + string("]"));
return false;
@@ -925,6 +959,58 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p)
return true;
}
// Translate a range clause. This only works if a Xapian value slot
// was attributed to the field.
bool SearchDataClauseRange::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB("SearchDataClauseRange::toNativeQuery: " << m_field <<
" :[" << m_text << ".." << m_t2 << "]\n");
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
if (m_field.empty() || (m_text.empty() && m_t2.empty())) {
m_reason = "Range clause needs a field and a value";
return false;
}
// Get the value number for the field from the configuration
const FieldTraits *ftp;
if (!db.fieldToTraits(m_field, &ftp, true)) {
m_reason = string("field ") + m_field + " not found in configuration";
return false;
}
if (ftp->valueslot == 0) {
m_reason = string("No value slot specified in configuration for field ")
+ m_field;
return false;
}
LOGDEB("SearchDataClauseRange: value slot " << ftp->valueslot << endl);
// Build Xapian VALUE query.
string errstr;
try {
if (m_text.empty()) {
*qp = Xapian::Query(Xapian::Query::OP_VALUE_LE,
ftp->valueslot, convert_field_value(*ftp, m_t2));
} else if (m_t2.empty()) {
*qp = Xapian::Query(Xapian::Query::OP_VALUE_GE, ftp->valueslot,
convert_field_value(*ftp, m_text));
} else {
*qp = Xapian::Query(Xapian::Query::OP_VALUE_RANGE, ftp->valueslot,
convert_field_value(*ftp, m_text),
convert_field_value(*ftp, m_t2));
}
}
XCATCHERROR(errstr);
if (!errstr.empty()) {
LOGERR("SearchDataClauseRange: range query creation failed for slot "<<
ftp->valueslot << endl);
m_reason = "Range query creation failed\n";
*qp = Xapian::Query();
return false;
}
return true;
}
// Translate a FILENAME search clause. This always comes
// from a "filename" search from the gui or recollq. A query language
// "filename:"-prefixed field will not go through here, but through
@@ -1018,7 +1104,7 @@ bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
// Translate NEAR or PHRASE clause.
bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB("SearchDataClauseDist::toNativeQuery\n" );
LOGDEB("SearchDataClauseDist::toNativeQuery\n");
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
@@ -1037,7 +1123,7 @@ bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
if (!processUserString(db, s, m_reason, &pqueries, m_slack, useNear))
return false;
if (pqueries.empty()) {
LOGERR("SearchDataClauseDist: resolved to null query\n" );
LOGERR("SearchDataClauseDist: resolved to null query\n");
m_reason = string("Resolved to null query. Term too long ? : [" +
m_text + string("]"));
return false;
src/rcldb/searchdataxml.cpp
+10
View File
@@ -42,6 +42,7 @@ static string tpToString(SClType tp)
case SCLT_FILENAME: return "FN";
case SCLT_PHRASE: return "PH";
case SCLT_NEAR: return "NE";
case SCLT_RANGE: return "RG";
case SCLT_SUB: return "SU"; // Unsupported actually
default: return "UN";
}
@@ -101,6 +102,15 @@ string SearchData::asXML()
endl;
}
os << "<T>" << base64_encode(cl->gettext()) << "</T>" << endl;
if (cl->getTp() == SCLT_RANGE) {
SearchDataClauseRange *clr =
dynamic_cast<SearchDataClauseRange*>(cl);
const string& t = clr->gettext2();
if (!t.empty()) {
os << "<T2>" << base64_encode(clr->gettext2()) <<
"</T2>" << endl;
}
}
if (cl->getTp() == SCLT_NEAR || cl->getTp() == SCLT_PHRASE) {
SearchDataClauseDist *cld =
dynamic_cast<SearchDataClauseDist*>(cl);