#ifndef lint static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.32 2008-12-19 09:55:36 dockes Exp $ (C) 2006 J.F.Dockes"; #endif /* * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ // Handle translation from rcl's SearchData structures to Xapian Queries #include #include #include #include "xapian.h" #include "rcldb.h" #include "searchdata.h" #include "debuglog.h" #include "smallut.h" #include "textsplit.h" #include "unacpp.h" #include "utf8iter.h" #include "stoplist.h" #include "rclconfig.h" #ifndef NO_NAMESPACES using namespace std; namespace Rcl { #endif typedef vector::iterator qlist_it_t; typedef vector::const_iterator qlist_cit_t; static const int original_term_wqf_booster = 10; bool SearchData::toNativeQuery(Rcl::Db &db, void *d) { Xapian::Query xq; m_reason.erase(); if (m_query.size() < 1) { m_reason = "empty query"; return false; } // It's not allowed to have a pure negative query and also it // seems that Xapian doesn't like the first element to be AND_NOT qlist_it_t itnotneg = m_query.end(); for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) { if ((*it)->m_tp != SCLT_EXCL) { itnotneg = it; break; } } if (itnotneg == m_query.end()) { LOGERR(("SearchData::toNativeQuery: can't have all negative clauses")); m_reason = "Can't have only negative clauses"; return false; } if ((*m_query.begin())->m_tp == SCLT_EXCL) iter_swap(m_query.begin(), itnotneg); // Walk the clause list translating each in turn and building the // Xapian query tree for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) { Xapian::Query nq; if (!(*it)->toNativeQuery(db, &nq, m_stemlang)) { LOGERR(("SearchData::toNativeQuery: failed\n")); m_reason = (*it)->getReason(); return false; } if (nq.empty()) { LOGDEB(("SearchData::toNativeQuery: skipping empty clause\n")); continue; } // If this structure is an AND list, must use AND_NOT for excl clauses. // Else this is an OR list, and there can't be excl clauses (checked by // addClause()) Xapian::Query::op op; if (m_tp == SCLT_AND) { op = (*it)->m_tp == SCLT_EXCL ? Xapian::Query::OP_AND_NOT: Xapian::Query::OP_AND; } else { op = Xapian::Query::OP_OR; } xq = xq.empty() ? nq : Xapian::Query(op, xq, nq); } // Add the file type filtering clause if any if (!m_filetypes.empty()) { vector exptps; exptps.reserve(m_filetypes.size()); // Expand categories RclConfig *cfg = db.getConf(); for (vector::iterator it = m_filetypes.begin(); it != m_filetypes.end(); it++) { if (cfg && cfg->isMimeCategory(*it)) { listtps; cfg->getMimeCatTypes(*it, tps); exptps.insert(exptps.end(), tps.begin(), tps.end()); } else { exptps.push_back(*it); } } list pqueries; Xapian::Query tq; for (vector::iterator it = exptps.begin(); it != exptps.end(); it++) { string term = "T" + *it; LOGDEB(("Adding file type term: [%s]\n", term.c_str())); tq = tq.empty() ? Xapian::Query(term) : Xapian::Query(Xapian::Query::OP_OR, tq, Xapian::Query(term)); } xq = xq.empty() ? tq : Xapian::Query(Xapian::Query::OP_FILTER, xq, tq); } *((Xapian::Query *)d) = xq; return true; } // Add clause to current list. OR lists cant have EXCL clauses. bool SearchData::addClause(SearchDataClause* cl) { if (m_tp == SCLT_OR && (cl->m_tp == SCLT_EXCL)) { LOGERR(("SearchData::addClause: cant add EXCL to OR list\n")); m_reason = "No Negative (AND_NOT) clauses allowed in OR queries"; return false; } cl->setParent(this); m_haveWildCards = m_haveWildCards || cl->m_haveWildCards; m_query.push_back(cl); return true; } // Make me all new void SearchData::erase() { LOGDEB(("SearchData::erase\n")); m_tp = SCLT_AND; for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) delete *it; m_query.clear(); m_filetypes.clear(); m_topdir.erase(); m_description.erase(); m_reason.erase(); } // Am I a file name only search ? This is to turn off term highlighting bool SearchData::fileNameOnly() { for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) if (!(*it)->isFileName()) return false; return true; } // Extract all terms and term groups bool SearchData::getTerms(vector& terms, vector >& groups, vector& gslks) const { for (qlist_cit_t it = m_query.begin(); it != m_query.end(); it++) (*it)->getTerms(terms, groups, gslks); return true; } // Extract user terms void SearchData::getUTerms(vector& terms) const { for (qlist_cit_t it = m_query.begin(); it != m_query.end(); it++) (*it)->getUTerms(terms); sort(terms.begin(), terms.end()); vector::iterator it = unique(terms.begin(), terms.end()); terms.erase(it, terms.end()); } // Splitter callback for breaking a user string into simple terms and // phrases. This is for parts of the user entry which would appear as // a single word because there is no white space inside, but are // actually multiple terms to rcldb (ie term1,term2) class TextSplitQ : public TextSplit { public: TextSplitQ(Flags flags, const StopList &_stops) : TextSplit(flags), stops(_stops), alltermcount(0), lastpos(0) {} bool takeword(const std::string &interm, int pos, int, int) { alltermcount++; lastpos = pos LOGDEB1(("TextSplitQ::takeword: %s\n", interm.c_str())); // Check if the first letter is a majuscule in which // case we do not want to do stem expansion. Note that // the test is convoluted and possibly problematic string noacterm, noaclowterm; if (!unacmaybefold(interm, noacterm, "UTF-8", false)) { LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n", interm.c_str())); return true; } if (!unacmaybefold(noacterm, noaclowterm, "UTF-8", true)) { LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n", noacterm.c_str())); return true; } bool nostemexp = false; Utf8Iter it1(noacterm); Utf8Iter it2(noaclowterm); if (*it1 != *it2) nostemexp = true; if (stops.hasStops() && stops.isStop(noaclowterm)) { LOGDEB1(("TextSplitQ::takeword [%s] in stop list\n", noaclowterm.c_str())); return true; } terms.push_back(noaclowterm); nostemexps.push_back(nostemexp); return true; } vector terms; vector nostemexps; const StopList &stops; // Count of terms including stopwords: this is for adjusting // phrase/near slack int alltermcount; int lastpos; }; // A class used to translate a user compound string (*not* a query // language string) as may be entered in any_terms/all_terms search // entry fields, ex: [term1 "a phrase" term3] into a xapian query // tree. // The object keeps track of the query terms and term groups while // translating. class StringToXapianQ { public: StringToXapianQ(Db& db, const string& field, const string &stmlng, bool boostUser) : m_db(db), m_field(field), m_stemlang(stmlng), m_doBoostUserTerms(boostUser) { } bool processUserString(const string &iq, string &ermsg, list &pqueries, const StopList &stops, int slack = 0, bool useNear = false); // After processing the string: return search terms and term // groups (ie: for highlighting) bool getTerms(vector& terms, vector >& groups) { terms.insert(terms.end(), m_terms.begin(), m_terms.end()); groups.insert(groups.end(), m_groups.begin(), m_groups.end()); return true; } bool getUTerms(vector& terms) { terms.insert(terms.end(), m_uterms.begin(), m_uterms.end()); return true; } private: void expandTerm(bool dont, const string& term, list& exp, string& sterm, string *prefix = 0); // After splitting entry on whitespace: process non-phrase element void processSimpleSpan(const string& span, bool nostemexp, list &pqueries); // Process phrase/near element void processPhraseOrNear(TextSplitQ *splitData, list &pqueries, bool useNear, int slack); Db& m_db; const string& m_field; const string& m_stemlang; bool m_doBoostUserTerms; // Single terms and phrases resulting from breaking up text; vector m_uterms; vector m_terms; vector > m_groups; }; #if 0 static void listVector(const string& what, const vector&l) { string a; for (vector::const_iterator it = l.begin(); it != l.end(); it++) { a = a + *it + " "; } LOGDEB(("%s: %s\n", what.c_str(), a.c_str())); } #endif /** Expand stem and wildcards * * @param nostemexp don't perform stem expansion. This is mainly used to * prevent stem expansion inside phrases (because the user probably * does not expect it). This does NOT prevent wild card expansion. * Other factors than nostemexp can prevent stem expansion: * a null stemlang, resulting from a global user preference, a * capitalized term, or wildcard(s) * @param term input single word * @param exp output expansion list * @param sterm output original input term if there were no wildcards */ void StringToXapianQ::expandTerm(bool nostemexp, const string& term, list& exp, string &sterm, string *prefix) { LOGDEB2(("expandTerm: field [%s] term [%s] stemlang [%s] nostemexp %d\n", m_field.c_str(), term.c_str(), m_stemlang.c_str(), nostemexp)); sterm.erase(); exp.clear(); if (term.empty()) { return; } bool haswild = term.find_first_of("*?[") != string::npos; // No stemming if there are wildcards or prevented globally. if (haswild || m_stemlang.empty()) nostemexp = true; if (nostemexp && !haswild) { // Neither stemming nor wildcard expansion: just the word string pfx; if (!m_field.empty()) m_db.fieldToPrefix(m_field, pfx); sterm = term; m_uterms.push_back(sterm); exp.push_front(pfx+term); exp.resize(1); if (prefix) *prefix = pfx; } else { list l; if (haswild) { m_db.termMatch(Rcl::Db::ET_WILD, m_stemlang, term, l, -1, m_field, prefix); } else { sterm = term; m_uterms.push_back(sterm); m_db.termMatch(Rcl::Db::ET_STEM, m_stemlang, term, l, -1, m_field, prefix); } for (list::const_iterator it = l.begin(); it != l.end(); it++) { exp.push_back(it->term); } } //listVector("ExpandTerm:uterms now: ", m_uterms); } // Do distribution of string vectors: a,b c,d -> a,c a,d b,c b,d void multiply_groups(vector >::const_iterator vvit, vector >::const_iterator vvend, vector& comb, vector >&allcombs) { // Remember my string vector and compute next, for recursive calls. vector >::const_iterator myvit = vvit++; // Walk the string vector I'm called upon and, for each string, // add it to current result, an call myself recursively on the // next string vector. The last call (last element of the vector of // vectors), adds the elementary result to the output // Walk my string vector for (vector::const_iterator strit = (*myvit).begin(); strit != (*myvit).end(); strit++) { // Add my current value to the string vector we're building comb.push_back(*strit); if (vvit == vvend) { // Last call: store current result allcombs.push_back(comb); } else { // Call recursively on next string vector multiply_groups(vvit, vvend, comb, allcombs); } // Pop the value I just added (make room for the next element in my // vector) comb.pop_back(); } } void StringToXapianQ::processSimpleSpan(const string& span, bool nostemexp, list &pqueries) { list exp; string sterm; // dumb version of user term string prefix; expandTerm(nostemexp, span, exp, sterm, &prefix); m_terms.insert(m_terms.end(), exp.begin(), exp.end()); // Push either term or OR of stem-expanded set Xapian::Query xq(Xapian::Query::OP_OR, exp.begin(), exp.end()); // If sterm (simplified original user term) is not null, give it a // relevance boost. We do this even if no expansion occurred (else // the non-expanded terms in a term list would end-up with even // less wqf). This does not happen if there are wildcards anywhere // in the search. if (m_doBoostUserTerms && !sterm.empty()) { xq = Xapian::Query(Xapian::Query::OP_OR, xq, Xapian::Query(prefix+sterm, original_term_wqf_booster)); } pqueries.push_back(xq); } // User entry element had several terms: transform into a PHRASE or // NEAR xapian query, the elements of which can themselves be OR // queries if the terms get expanded by stemming or wildcards (we // don't do stemming for PHRASE though) void StringToXapianQ::processPhraseOrNear(TextSplitQ *splitData, list &pqueries, bool useNear, int slack) { Xapian::Query::op op = useNear ? Xapian::Query::OP_NEAR : Xapian::Query::OP_PHRASE; list orqueries; bool hadmultiple = false; vector >groups; // Go through the list and perform stem/wildcard expansion for each element vector::iterator nxit = splitData->nostemexps.begin(); for (vector::iterator it = splitData->terms.begin(); it != splitData->terms.end(); it++, nxit++) { // Adjust when we do stem expansion. Not inside phrases, and // some versions of xapian will accept only one OR clause // inside NEAR, all others must be leafs. bool nostemexp = *nxit || (op == Xapian::Query::OP_PHRASE) || hadmultiple; string sterm; listexp; expandTerm(nostemexp, *it, exp, sterm); groups.push_back(vector(exp.begin(), exp.end())); orqueries.push_back(Xapian::Query(Xapian::Query::OP_OR, exp.begin(), exp.end())); #ifdef XAPIAN_NEAR_EXPAND_SINGLE_BUF if (exp.size() > 1) hadmultiple = true; #endif } // Generate an appropriate PHRASE/NEAR query with adjusted slack // For phrases, give a relevance boost like we do for original terms LOGDEB2(("PHRASE/NEAR: alltermcount %d lastpos %d\n", splitData->alltermcount, splitData->lastpos)); Xapian::Query xq(op, orqueries.begin(), orqueries.end(), splitData->lastpos + 1 + slack); if (op == Xapian::Query::OP_PHRASE) xq = Xapian::Query(Xapian::Query::OP_SCALE_WEIGHT, xq, original_term_wqf_booster); pqueries.push_back(xq); // Add all combinations of NEAR/PHRASE groups to the highlighting data. vector > allcombs; vector comb; multiply_groups(groups.begin(), groups.end(), comb, allcombs); m_groups.insert(m_groups.end(), allcombs.begin(), allcombs.end()); } /** * Turn user entry string (NOT query language) into a list of xapian queries. * We just separate words and phrases, and do wildcard and stem expansion, * * This is used to process data entered into an OR/AND/NEAR/PHRASE field of * the GUI. * * The final list contains one query for each term or phrase * - Elements corresponding to a stem-expanded part are an OP_OR * composition of the stem-expanded terms (or a single term query). * - Elements corresponding to phrase/near are an OP_PHRASE/NEAR * composition of the phrase terms (no stem expansion in this case) * @return the subquery count (either or'd stem-expanded terms or phrase word * count) */ bool StringToXapianQ::processUserString(const string &iq, string &ermsg, list &pqueries, const StopList& stops, int slack, bool useNear ) { LOGDEB(("StringToXapianQ:: query string: [%s]\n", iq.c_str())); ermsg.erase(); m_uterms.clear(); m_terms.clear(); m_groups.clear(); // Simple whitespace-split input into user-level words and // double-quoted phrases: word1 word2 "this is a phrase". // // The text splitter may further still decide that the resulting // "words" are really phrases, this depends on separators: // [paul@dom.net] would still be a word (span), but [about:me] // will probably be handled as a phrase. list phrases; TextSplit::stringToStrings(iq, phrases); // Process each element: textsplit into terms, handle stem/wildcard // expansion and transform into an appropriate Xapian::Query try { for (list::iterator it = phrases.begin(); it != phrases.end(); it++) { LOGDEB0(("strToXapianQ: phrase/word: [%s]\n", it->c_str())); // If there are multiple spans in this element, including // at least one composite, we have to increase the slack // else a phrase query including a span would fail. // Ex: "term0@term1 term2" is onlyspans-split as: // 0 term0@term1 0 12 // 2 term2 13 18 // The position of term2 is 2, not 1, so a phrase search // would fail. // We used to do word split, searching for // "term0 term1 term2" instead, which may have worse // performance, but will succeed. // We now adjust the phrase/near slack by the term count // difference (this is mainly better for cjk where this is a very // common occurrence because of the ngrams thing. TextSplitQ splitterS(TextSplit::Flags(TextSplit::TXTS_ONLYSPANS | TextSplit::TXTS_KEEPWILD), stops); splitterS.text_to_words(*it); TextSplitQ splitterW(TextSplit::Flags(TextSplit::TXTS_NOSPANS | TextSplit::TXTS_KEEPWILD), stops); splitterW.text_to_words(*it); TextSplitQ *splitter = &splitterS; if (splitterS.terms.size() > 1 && splitterS.terms.size() != splitterW.terms.size()) { slack += splitterW.terms.size() - splitterS.terms.size(); // used to: splitData = &splitDataW; } LOGDEB0(("strToXapianQ: termcount: %d\n", splitter->terms.size())); switch (splitter->terms.size()) { case 0: continue;// ?? case 1: processSimpleSpan(splitter->terms.front(), splitter->nostemexps.front(), pqueries); break; default: processPhraseOrNear(splitter, pqueries, useNear, slack); } } } catch (const Xapian::Error &e) { ermsg = e.get_msg(); } catch (const string &s) { ermsg = s; } catch (const char *s) { ermsg = s; } catch (...) { ermsg = "Caught unknown exception"; } if (!ermsg.empty()) { LOGERR(("stringToXapianQueries: %s\n", ermsg.c_str())); return false; } return true; } static const string nullstemlang; // Translate a simple OR, AND, or EXCL search clause. bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p, const string& stemlang) { const string& l_stemlang = (m_modifiers&SDCM_NOSTEMMING)? nullstemlang: stemlang; m_terms.clear(); m_groups.clear(); Xapian::Query *qp = (Xapian::Query *)p; *qp = Xapian::Query(); Xapian::Query::op op; switch (m_tp) { case SCLT_AND: op = Xapian::Query::OP_AND; break; // EXCL will be set with AND_NOT in the list. So it's an OR list here case SCLT_OR: case SCLT_EXCL: op = Xapian::Query::OP_OR; break; default: LOGERR(("SearchDataClauseSimple: bad m_tp %d\n", m_tp)); return false; } list pqueries; // We normally boost the original term in the stem expansion list. Don't // do it if there are wildcards anywhere, this would skew the results. bool doBoostUserTerm = (m_parentSearch && !m_parentSearch->haveWildCards()) || (m_parentSearch == 0 && !m_haveWildCards); StringToXapianQ tr(db, m_field, l_stemlang, doBoostUserTerm); if (!tr.processUserString(m_text, m_reason, pqueries, db.getStopList())) return false; if (pqueries.empty()) { LOGERR(("SearchDataClauseSimple: resolved to null query\n")); return true; } tr.getTerms(m_terms, m_groups); tr.getUTerms(m_uterms); //listVector("SearchDataClauseSimple: Uterms: ", m_uterms); *qp = Xapian::Query(op, pqueries.begin(), pqueries.end()); return true; } // Translate a FILENAME search clause. Actually this is now mostly // a "filename" field search. bool SearchDataClauseFilename::toNativeQuery(Rcl::Db &db, void *p, const string&) { Xapian::Query *qp = (Xapian::Query *)p; *qp = Xapian::Query(); list patterns; TextSplit::stringToStrings(m_text, patterns); list names; for (list::iterator it = patterns.begin(); it != patterns.end(); it++) { list more; db.filenameWildExp(*it, more); names.splice(names.end(), more); } // Build a query out of the matching file name terms. *qp = Xapian::Query(Xapian::Query::OP_OR, names.begin(), names.end()); return true; } // Translate NEAR or PHRASE clause. bool SearchDataClauseDist::toNativeQuery(Rcl::Db &db, void *p, const string& stemlang) { const string& l_stemlang = (m_modifiers&SDCM_NOSTEMMING)? nullstemlang: stemlang; LOGDEB(("SearchDataClauseDist::toNativeQuery\n")); m_terms.clear(); m_groups.clear(); Xapian::Query *qp = (Xapian::Query *)p; *qp = Xapian::Query(); list pqueries; Xapian::Query nq; // We normally boost the original term in the stem expansion list. Don't // do it if there are wildcards anywhere, this would skew the results. bool doBoostUserTerm = (m_parentSearch && !m_parentSearch->haveWildCards()) || (m_parentSearch == 0 && !m_haveWildCards); // We produce a single phrase out of the user entry (there should be // no dquotes in there), then use stringToXapianQueries() to // lowercase and simplify the phrase terms etc. This will result // into a single (complex) Xapian::Query. if (m_text.find_first_of("\"") != string::npos) { LOGDEB(("Double quotes inside phrase/near field\n")); return false; } string s = string("\"") + m_text + string("\""); bool useNear = (m_tp == SCLT_NEAR); StringToXapianQ tr(db, m_field, l_stemlang, doBoostUserTerm); if (!tr.processUserString(s, m_reason, pqueries, db.getStopList(), m_slack, useNear)) return false; if (pqueries.empty()) { LOGERR(("SearchDataClauseDist: resolved to null query\n")); return true; } tr.getTerms(m_terms, m_groups); tr.getUTerms(m_uterms); *qp = *pqueries.begin(); return true; } // Translate subquery bool SearchDataClauseSub::toNativeQuery(Rcl::Db &db, void *p, const string&) { return m_sub->toNativeQuery(db, p); } bool SearchDataClauseSub::getTerms(vector& terms, vector >& groups, vector& gslks) const { return m_sub.getconstptr()->getTerms(terms, groups, gslks); } void SearchDataClauseSub::getUTerms(vector& terms) const { m_sub.getconstptr()->getUTerms(terms); } } // Namespace Rcl