tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

define new searchdataclausepath to replace the old dir: filtering mechanism. ORing dirs now works

Browse Source
This commit is contained in:
Jean-Francois Dockes
2013-01-05 16:21:30 +01:00
parent 9e88426d13
commit cbc269abb1
10 changed files with 200 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/searchdata.cpp
+31 -33
View File
@@ -128,7 +128,7 @@ bool SearchData::clausesToQuery(Rcl::Db &db, SClType tp,
// addClause())
Xapian::Query::op op;
if (tp == SCLT_AND) {
if ((*it)->m_tp == SCLT_EXCL) {
if ((*it)->m_tp == SCLT_EXCL || (*it)->getexclude()) {
op = Xapian::Query::OP_AND_NOT;
} else {
op = Xapian::Query::OP_AND;
@@ -274,36 +274,6 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_AND_NOT, xq, tq);
}
// Add the directory filtering clauses. Each is a phrase of terms
// prefixed with the pathelt prefix XP
for (vector<DirSpec>::const_iterator dit = m_dirspecs.begin();
dit != m_dirspecs.end(); dit++) {
vector<string> vpath;
stringToTokens(dit->dir, vpath, "/");
vector<string> pvpath;
if (dit->dir[0] == '/')
pvpath.push_back(wrap_prefix(pathelt_prefix));
for (vector<string>::const_iterator pit = vpath.begin();
pit != vpath.end(); pit++){
pvpath.push_back(wrap_prefix(pathelt_prefix) + *pit);
}
Xapian::Query::op tdop;
if (dit->weight == 1.0) {
tdop = dit->exclude ?
Xapian::Query::OP_AND_NOT : Xapian::Query::OP_FILTER;
} else {
tdop = dit->exclude ?
Xapian::Query::OP_AND_NOT : Xapian::Query::OP_AND_MAYBE;
}
Xapian::Query tdq = Xapian::Query(Xapian::Query::OP_PHRASE,
pvpath.begin(), pvpath.end());
if (dit->weight != 1.0)
tdq = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT,
tdq, dit->weight);
xq = Xapian::Query(tdop, xq, tdq);
}
*((Xapian::Query *)d) = xq;
return true;
}
@@ -418,7 +388,7 @@ bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
// Add clause to current list. OR lists cant have EXCL clauses.
bool SearchData::addClause(SearchDataClause* cl)
{
if (m_tp == SCLT_OR && (cl->m_tp == SCLT_EXCL)) {
if (m_tp == SCLT_OR && (cl->m_tp == SCLT_EXCL || cl->getexclude())) {
LOGERR(("SearchData::addClause: cant add EXCL to OR list\n"));
m_reason = "No Negative (AND_NOT) clauses allowed in OR queries";
return false;
@@ -438,7 +408,6 @@ void SearchData::erase()
delete *it;
m_query.clear();
m_filetypes.clear();
m_dirspecs.clear();
m_description.erase();
m_reason.erase();
m_haveDates = false;
@@ -1169,6 +1138,35 @@ bool SearchDataClauseFilename::toNativeQuery(Rcl::Db &db, void *p)
return true;
}
// Translate a dir: path filtering clause. See comments in .h
bool SearchDataClausePath::toNativeQuery(Rcl::Db &db, void *p)
{
LOGDEB(("SearchDataClausePath::toNativeQuery: [%s]\n", m_text.c_str()));
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
if (m_text.empty()) {
LOGERR(("SearchDataClausePath: empty path??\n"));
return false;
}
vector<string> vpath;
stringToTokens(m_text, vpath, "/");
vector<string> pvpath;
if (m_text[0] == '/')
pvpath.push_back(wrap_prefix(pathelt_prefix));
for (vector<string>::const_iterator pit = vpath.begin();
pit != vpath.end(); pit++){
pvpath.push_back(wrap_prefix(pathelt_prefix) + *pit);
}
*qp = Xapian::Query(Xapian::Query::OP_PHRASE,
pvpath.begin(), pvpath.end());
if (m_weight != 1.0) {
*qp = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, *qp, m_weight);
}
return true;
}
// Translate NEAR or PHRASE clause.
bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p)
{
src/rcldb/searchdata.h
+57 -24
View File
@@ -41,7 +41,7 @@ namespace Rcl {
/** Search clause types */
enum SClType {
SCLT_AND,
SCLT_OR, SCLT_EXCL, SCLT_FILENAME, SCLT_PHRASE, SCLT_NEAR,
SCLT_OR, SCLT_EXCL, SCLT_FILENAME, SCLT_PHRASE, SCLT_NEAR, SCLT_PATH,
SCLT_SUB
};
@@ -84,7 +84,7 @@ public:
commoninit();
}
SearchData()
: m_tp(SCLT_AND), m_stemlang("english")
: m_tp(SCLT_AND)
{
commoninit();
}
@@ -118,12 +118,6 @@ public:
*/
bool maybeAddAutoPhrase(Rcl::Db &db, double threshold);
/** Set/get top subdirectory for filtering results */
void addDirSpec(const std::string& t, bool excl = false, float w = 1.0)
{
m_dirspecs.push_back(DirSpec(t, excl, w));
}
const std::string& getStemLang() {return m_stemlang;}
void setMinSize(size_t size) {m_minSize = size;}
@@ -182,20 +176,6 @@ private:
// Excluded set of file types if not empty
std::vector<std::string> m_nfiletypes;
// Restrict to subtree or exclude one
class DirSpec {
public:
std::string dir;
bool exclude;
// For positive spec: affect weight instead of filter
float weight;
DirSpec(const std::string&d, bool x, float w)
: dir(d), exclude(x), weight(w)
{
}
};
std::vector<DirSpec> m_dirspecs;
bool m_haveDates;
DateInterval m_dates; // Restrict to date interval
size_t m_maxSize;
@@ -240,7 +220,7 @@ public:
SearchDataClause(SClType tp)
: m_tp(tp), m_parentSearch(0), m_haveWildCards(0),
m_modifiers(SDCM_NONE), m_weight(1.0)
m_modifiers(SDCM_NONE), m_weight(1.0), m_exclude(false)
{}
virtual ~SearchDataClause() {}
virtual bool toNativeQuery(Rcl::Db &db, void *) = 0;
@@ -299,8 +279,12 @@ public:
{
m_weight = w;
}
friend class SearchData;
virtual bool getexclude() const
{
return m_exclude;
}
friend class SearchData;
protected:
std::string m_reason;
SClType m_tp;
@@ -308,6 +292,7 @@ protected:
bool m_haveWildCards;
Modifier m_modifiers;
float m_weight;
bool m_exclude;
private:
SearchDataClause(const SearchDataClause&)
{
@@ -404,6 +389,54 @@ protected:
std::string m_text;
};
/**
* Pathname filtering clause. This is special because of history:
* - Pathname filtering used to be performed as a post-processing step
* done with the url fields of doc data records.
* - Then it was done as special phrase searchs on path elements prefixed
* with XP.
* Up to this point dir filtering data was stored as part of the searchdata
* object, not in the SearchDataClause tree. Only one, then a list,
* of clauses where stored, and they were always ANDed together.
*
* In order to allow for OR searching, dir clauses are now stored in a
* specific SearchDataClause, but this is still special because the field has
* non-standard phrase-like processing, reflected in index storage by
* an empty element representing / (as "XP").
*
* A future version should use a standard phrase with an anchor to the
* start if the path starts with /. As this implies an index format
* change but is no important enough to warrant it, this has to wait for
* the next format change.
*/
class SearchDataClausePath : public SearchDataClause {
public:
SearchDataClausePath(const std::string& txt, bool excl = false)
: SearchDataClause(SCLT_PATH), m_text(txt)
{
m_exclude = excl;
m_haveWildCards = false;
}
virtual ~SearchDataClausePath()
{
}
virtual void getTerms(HighlightData&) const
{
}
virtual bool toNativeQuery(Rcl::Db &, void *);
virtual const std::string& gettext() const
{
return m_text;
}
protected:
std::string m_text;
};
/**
* A clause coming from a NEAR or PHRASE entry field. There is only one
* std::string group, and a specified distance, which applies to it.
src/rcldb/searchdataxml.cpp
+16 -10
View File
@@ -15,7 +15,8 @@
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
// Handle translation from rcl's SearchData structures to Xapian Queries
// Handle translation from rcl's SearchData structures to XML. Used for
// complex search history storage in the GUI
#include "autoconfig.h"
@@ -64,6 +65,20 @@ string SearchData::asXML()
LOGERR(("SearchData::asXML: can't do subclauses !\n"));
continue;
}
//if (c->getexclude())
// os << "<NEG/>" << endl;
if (c->getTp() == SCLT_PATH) {
// Keep these apart, for compat with the older history format
SearchDataClausePath *cl =
dynamic_cast<SearchDataClausePath*>(c);
if (cl->getexclude()) {
os << "<ND>" << base64_encode(cl->gettext()) << "</ND>" << endl;
} else {
os << "<YD>" << base64_encode(cl->gettext()) << "</YD>" << endl;
}
continue;
}
SearchDataClauseSimple *cl =
dynamic_cast<SearchDataClauseSimple*>(c);
os << "<C>" << endl;
@@ -100,7 +115,6 @@ string SearchData::asXML()
}
}
if (m_minSize != size_t(-1)) {
os << "<MIS>" << m_minSize << "</MIS>" << endl;
}
@@ -126,14 +140,6 @@ string SearchData::asXML()
os << "</IT>" << endl;
}
for (vector<DirSpec>::const_iterator dit = m_dirspecs.begin();
dit != m_dirspecs.end(); dit++) {
if (dit->exclude) {
os << "<ND>" << base64_encode(dit->dir) << "</ND>" << endl;
} else {
os << "<YD>" << base64_encode(dit->dir) << "</YD>" << endl;
}
}
os << "</SD>";
return os.str();
}
src/rcldb/stemdb.cpp
+9 -9
View File
@@ -57,18 +57,18 @@ bool StemDb::stemExpand(const std::string& langs, const std::string& term,
}
#ifndef RCL_INDEX_STRIPCHARS
// Expand the unaccented stem
if (!o_index_stripchars) {
string unac;
unacmaybefold(term, unac, "UTF-8", UNACOP_UNAC);
if (term != unac) {
for (vector<string>::const_iterator it = llangs.begin();
it != llangs.end(); it++) {
SynTermTransStem stemmer(*it);
XapComputableSynFamMember expander(getdb(), synFamStemUnac,
*it, &stemmer);
(void)expander.synExpand(unac, result);
}
// Expand the unaccented stem, using the unaccented stem
// db. Because it's a different db, We need to do it even if
// the input has no accent (unac == term)
for (vector<string>::const_iterator it = llangs.begin();
it != llangs.end(); it++) {
SynTermTransStem stemmer(*it);
XapComputableSynFamMember expander(getdb(), synFamStemUnac,
*it, &stemmer);
(void)expander.synExpand(unac, result);
}
}
#endif