tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added dynamic clauses to adv search. Still needs work

Browse Source
This commit is contained in:
dockes
2006-11-14 13:55:43 +00:00
parent e4abb291fa
commit 7cc20a8f78
11 changed files with 751 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+7 -3
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.93 2006-11-13 14:51:58 dockes Exp $ (C) 2004 J.F.Dockes";
static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.94 2006-11-14 13:55:43 dockes Exp $ (C) 2004 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -1250,7 +1250,7 @@ bool Db::setQuery(RefCntr<SearchData> sdata, int opts,
LOGERR(("Db::setQuery: no db!\n"));
return false;
}
m_reason.erase();
LOGDEB(("Db::setQuery:\n"));
m_filterTopDir = sdata->m_topdir;
@@ -1259,7 +1259,11 @@ bool Db::setQuery(RefCntr<SearchData> sdata, int opts,
m_ndb->m_termfreqs.clear();
Xapian::Query xq;
sdata->toNativeQuery(*this, &xq, (opts & Db::QO_STEM) ? stemlang : "");
if (!sdata->toNativeQuery(*this, &xq,
(opts & Db::QO_STEM) ? stemlang : "")) {
m_reason += sdata->getReason();
return false;
}
m_ndb->query = xq;
delete m_ndb->enquire;
src/rcldb/rcldb.h
+3 -2
View File
@@ -16,7 +16,7 @@
*/
#ifndef _DB_H_INCLUDED_
#define _DB_H_INCLUDED_
/* @(#$Id: rcldb.h,v 1.41 2006-11-13 08:49:44 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: rcldb.h,v 1.42 2006-11-14 13:55:43 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
@@ -217,13 +217,14 @@ class Db {
/** Filename wildcard expansion */
bool filenameWildExp(const string& exp, list<string>& names);
string getReason(){return m_reason;}
private:
string m_filterTopDir; // Current query filter on subtree top directory
vector<int> m_dbindices; // In case there is a postq filter: sequence of
// db indices that match
string m_reason; // Error explanation
// Things we don't want to have here.
friend class Native;
Native *m_ndb; // Pointer to private data. We don't want db(ie
src/rcldb/searchdata.cpp
+121 -79
View File
@@ -1,5 +1,5 @@
#ifndef lint
static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.1 2006-11-13 08:49:44 dockes Exp $ (C) 2006 J.F.Dockes";
static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.2 2006-11-14 13:55:43 dockes Exp $ (C) 2006 J.F.Dockes";
#endif
/*
* This program is free software; you can redistribute it and/or modify
@@ -43,16 +43,21 @@ typedef list<SearchDataClause *>::iterator qlist_it_t;
bool SearchData::toNativeQuery(Rcl::Db &db, void *d, const string& stemlang)
{
Xapian::Query xq;
m_reason.erase();
// Walk the clause list translating each in turn and building the
// Xapian query tree
for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) {
Xapian::Query nq;
(*it)->toNativeQuery(db, &nq, stemlang);
Xapian::Query::op op;
if (!(*it)->toNativeQuery(db, &nq, stemlang)) {
LOGERR(("SearchData::toNativeQuery: failed\n"));
m_reason = (*it)->getReason();
return false;
}
// If this structure is an AND list, must use AND_NOT for excl clauses.
// Else this is an OR list, and there can't be excl clauses
Xapian::Query::op op;
if (m_tp == SCLT_AND) {
op = (*it)->m_tp == SCLT_EXCL ?
Xapian::Query::OP_AND_NOT: Xapian::Query::OP_AND;
@@ -137,97 +142,133 @@ class wsQData : public TextSplitCB {
}
};
/** Possibly expand term into its stem siblings, make them dumb strings */
static void maybeStemExp(Db& db, const string& stemlang, const string& term,
list<string>& exp)
{
string term1;
dumb_string(term, term1);
if (!stemlang.empty()) {
bool nostemexp = false;
// Check if the first letter is a majuscule in which
// case we do not want to do stem expansion. Note that
// the test is convoluted and possibly problematic
if (term.length() > 0) {
string noacterm,noaclowterm;
if (unacmaybefold(term, noacterm, "UTF-8", false) &&
unacmaybefold(noacterm, noaclowterm, "UTF-8", true)) {
Utf8Iter it1(noacterm);
Utf8Iter it2(noaclowterm);
if (*it1 != *it2)
nostemexp = true;
}
}
LOGDEB1(("Term: %s stem expansion: %s\n",
term.c_str(), nostemexp?"no":"yes"));
if (!nostemexp) {
exp = db.stemExpand(stemlang, term1);
return;
}
}
// Turn string into list of xapian queries. There is little
// interpretation done on the string (no +term -term or filename:term
// stuff). We just separate words and phrases, and interpret
// capitalized terms as wanting no stem expansion.
// The final list contains one query for each term or phrase
// - Elements corresponding to a stem-expanded part are an OP_OR
// composition of the stem-expanded terms (or a single term query).
// - Elements corresponding to a phrase are an OP_PHRASE composition of the
// phrase terms (no stem expansion in this case)
static void stringToXapianQueries(const string &iq,
exp.push_back(term1);
}
/** Turn string into list of xapian queries. There is little
* interpretation done on the string (no +term -term or filename:term
* stuff). We just separate words and phrases, and interpret
* capitalized terms as wanting no stem expansion.
* The final list contains one query for each term or phrase
* - Elements corresponding to a stem-expanded part are an OP_OR
* composition of the stem-expanded terms (or a single term query).
* - Elements corresponding to a phrase are an OP_PHRASE composition of the
* phrase terms (no stem expansion in this case)
* @return the subquery count (either or'd stem-expanded terms or phrase word
* count)
*/
static bool stringToXapianQueries(const string &iq,
const string& stemlang,
Db& db,
list<Xapian::Query> &pqueries)
string &ermsg,
list<Xapian::Query> &pqueries,
int slack = 0, bool useNear = false)
{
string qstring = iq;
bool opt_stemexp = !stemlang.empty();
ermsg.erase();
// Split into (possibly single word) phrases ("this is a phrase"):
// Split into words and phrases (word1 word2 "this is a phrase"):
list<string> phrases;
stringToStrings(qstring, phrases);
// Then process each phrase: split into terms and transform into
// appropriate Xapian Query
try {
for (list<string>::iterator it = phrases.begin();
it != phrases.end(); it++) {
LOGDEB(("strToXapianQ: phrase or word: [%s]\n", it->c_str()));
for (list<string>::iterator it=phrases.begin(); it !=phrases.end(); it++) {
LOGDEB(("strToXapianQ: phrase or word: [%s]\n", it->c_str()));
// If there are both spans and single words in this element,
// we need to use a word split, else a phrase query including
// a span would fail if we didn't adjust the proximity to
// account for the additional span term which is complicated.
wsQData splitDataS, splitDataW;
TextSplit splitterS(&splitDataS, TextSplit::TXTS_ONLYSPANS);
splitterS.text_to_words(*it);
TextSplit splitterW(&splitDataW, TextSplit::TXTS_NOSPANS);
splitterW.text_to_words(*it);
wsQData& splitData = splitDataS;
if (splitDataS.terms.size() > 1 && splitDataS.terms.size() !=
splitDataW.terms.size())
splitData = splitDataW;
// If there are both spans and single words in this element,
// we need to use a word split, else a phrase query including
// a span would fail if we didn't adjust the proximity to
// account for the additional span term which is complicated.
wsQData splitDataS, splitDataW;
TextSplit splitterS(&splitDataS, TextSplit::TXTS_ONLYSPANS);
splitterS.text_to_words(*it);
TextSplit splitterW(&splitDataW, TextSplit::TXTS_NOSPANS);
splitterW.text_to_words(*it);
wsQData& splitData = splitDataS;
if (splitDataS.terms.size() > 1 && splitDataS.terms.size() !=
splitDataW.terms.size())
splitData = splitDataW;
LOGDEB1(("strToXapianQ: splitter term count: %d\n",
splitData.terms.size()));
switch(splitData.terms.size()) {
case 0: continue;// ??
case 1: // Not a real phrase: one term
{
string term = splitData.terms.front();
bool nostemexp = false;
// Check if the first letter is a majuscule in which
// case we do not want to do stem expansion. Note that
// the test is convoluted and possibly problematic
if (term.length() > 0) {
string noacterm,noaclowterm;
if (unacmaybefold(term, noacterm, "UTF-8", false) &&
unacmaybefold(noacterm, noaclowterm, "UTF-8", true)) {
Utf8Iter it1(noacterm);
Utf8Iter it2(noaclowterm);
if (*it1 != *it2)
nostemexp = true;
}
LOGDEB1(("strToXapianQ: splitter term count: %d\n",
splitData.terms.size()));
switch(splitData.terms.size()) {
case 0: continue;// ??
case 1: // Not a real phrase: one term
{
string term = splitData.terms.front();
list<string> exp;
maybeStemExp(db, stemlang, term, exp);
// Push either term or OR of stem-expanded set
pqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
exp.begin(), exp.end()));
}
LOGDEB1(("Term: %s stem expansion: %s\n",
term.c_str(), nostemexp?"no":"yes"));
break;
list<string> exp;
string term1;
dumb_string(term, term1);
// Possibly perform stem compression/expansion
if (!nostemexp && opt_stemexp) {
exp = db.stemExpand(stemlang, term1);
} else {
exp.push_back(term1);
default:
// Phrase/near
Xapian::Query::op op = useNear ? Xapian::Query::OP_NEAR :
Xapian::Query::OP_PHRASE;
list<Xapian::Query> orqueries;
for (vector<string>::iterator it = splitData.terms.begin();
it != splitData.terms.end(); it++) {
list<string>exp;
maybeStemExp(db, stemlang, *it, exp);
orqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
exp.begin(), exp.end()));
}
// Push either term or OR of stem-expanded set
pqueries.push_back(Xapian::Query(Xapian::Query::OP_OR,
exp.begin(), exp.end()));
pqueries.push_back(Xapian::Query(op,
orqueries.begin(),
orqueries.end(),
splitData.terms.size() + slack));
}
break;
default:
// Phrase: no stem expansion
splitData.dumball();
LOGDEB(("Pushing phrase: [%s]\n", splitData.catterms().c_str()));
pqueries.push_back(Xapian::Query(Xapian::Query::OP_PHRASE,
splitData.terms.begin(),
splitData.terms.end()));
}
} catch (const Xapian::Error &e) {
ermsg = e.get_msg();
} catch (const string &s) {
ermsg = s;
} catch (const char *s) {
ermsg = s;
} catch (...) {
ermsg = "Caught unknown exception";
}
if (!ermsg.empty()) {
LOGERR(("stringToXapianQueries: %s\n", ermsg.c_str()));
return false;
}
return true;
}
// Translate a simple OR, AND, or EXCL search clause.
@@ -247,7 +288,8 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p,
return false;
}
list<Xapian::Query> pqueries;
stringToXapianQueries(m_text, stemlang, db, pqueries);
if (!stringToXapianQueries(m_text, stemlang, db, m_reason, pqueries))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseSimple: resolved to null query\n"));
return true;
@@ -277,17 +319,17 @@ bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p,
{
Xapian::Query *qp = (Xapian::Query *)p;
*qp = Xapian::Query();
Xapian::Query::op op = m_tp == SCLT_PHRASE ? Xapian::Query::OP_PHRASE :
Xapian::Query::OP_NEAR;
list<Xapian::Query> pqueries;
Xapian::Query nq;
string s = string("\"") + m_text + string("\"");
bool useNear = m_tp == SCLT_NEAR;
// Use stringToXapianQueries anyway to lowercase and simplify the
// phrase terms etc. The result should be a single element list
stringToXapianQueries(s, stemlang, db, pqueries);
if (!stringToXapianQueries(s, stemlang, db, m_reason, pqueries,
m_slack, useNear))
return false;
if (pqueries.empty()) {
LOGERR(("SearchDataClauseDist: resolved to null query\n"));
return true;
src/rcldb/searchdata.h
+11 -5
View File
@@ -1,6 +1,6 @@
#ifndef _SEARCHDATA_H_INCLUDED_
#define _SEARCHDATA_H_INCLUDED_
/* @(#$Id: searchdata.h,v 1.3 2006-11-13 08:49:45 dockes Exp $ (C) 2004 J.F.Dockes */
/* @(#$Id: searchdata.h,v 1.4 2006-11-14 13:55:43 dockes Exp $ (C) 2004 J.F.Dockes */
#include <string>
#include <list>
@@ -36,6 +36,7 @@ class SearchData {
// Printable expanded version of the complete query, obtained from Xapian
// valid after setQuery() call
string m_description;
string m_reason;
SearchData(SClType tp) : m_tp(tp) {}
~SearchData() {erase();}
@@ -52,6 +53,8 @@ class SearchData {
/** We become the owner of cl and will delete it */
bool addClause(SearchDataClause *cl);
string getReason() {return m_reason;}
private:
/* Copyconst and assignment private and forbidden */
SearchData(const SearchData &) {}
@@ -66,6 +69,9 @@ class SearchDataClause {
virtual ~SearchDataClause() {}
virtual bool toNativeQuery(Rcl::Db &db, void *, const string&) = 0;
virtual bool isFileName() {return m_tp == SCLT_FILENAME ? true : false;}
string getReason() {return m_reason;}
protected:
string m_reason;
};
class SearchDataClauseSimple : public SearchDataClause {
@@ -81,20 +87,20 @@ protected:
class SearchDataClauseFilename : public SearchDataClauseSimple {
public:
SearchDataClauseFilename(string txt)
: SearchDataClauseSimple(SCLT_FILENAME, m_text) {}
: SearchDataClauseSimple(SCLT_FILENAME, txt) {}
virtual ~SearchDataClauseFilename() {}
virtual bool toNativeQuery(Rcl::Db &db, void *, const string& stemlang);
};
class SearchDataClauseDist : public SearchDataClauseSimple {
public:
SearchDataClauseDist(SClType tp, string txt, int dist)
: SearchDataClauseSimple(tp, txt), m_distance(dist) {}
SearchDataClauseDist(SClType tp, string txt, int slack)
: SearchDataClauseSimple(tp, txt), m_slack(slack) {}
virtual ~SearchDataClauseDist() {}
virtual bool toNativeQuery(Rcl::Db &db, void *, const string& stemlang);
protected:
int m_distance;
int m_slack;
};
class SearchDataClauseSub : public SearchDataClause {