Add frequency threshold to avoid adding common term to the automatic phrase search extension. Use autophrase by default with simple search, with a default freq threshold at 2%

src/rcldb/searchdata.cpp

@@ -168,6 +168,8 @@ bool SearchData::expandFileTypes(RclConfig *cfg, vector<string>& tps)
 
 bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
 {
+    LOGDEB2(("SearchData::toNativeQuery: stemlang [%s]\n", 
+	    m_stemlang.c_str()));
     Xapian::Query xq;
     m_reason.erase();
 
@@ -309,8 +311,11 @@ bool SearchData::toNativeQuery(Rcl::Db &db, void *d)
     return true;
 }
 
-
-bool SearchData::maybeAddAutoPhrase()
+// This is called by the GUI simple search if the option is set: add
+// (OR) phrase to a query (if it is simple enough) so that results
+// where the search terms are close and in order will come up on top.
+// We remove very common terms from the query to avoid performance issues.
+bool SearchData::maybeAddAutoPhrase(Rcl::Db& db, double freqThreshold)
 {
     LOGDEB0(("SearchData::maybeAddAutoPhrase()\n"));
     if (!m_query.size()) {
@@ -319,13 +324,13 @@ bool SearchData::maybeAddAutoPhrase()
     }
 
     string field;
-    string words;
+    list<string> words;
     // Walk the clause list. If we find any non simple clause or different
     // field names, bail out.
     for (qlist_it_t it = m_query.begin(); it != m_query.end(); it++) {
 	SClType tp = (*it)->m_tp;
 	if (tp != SCLT_AND && tp != SCLT_OR) {
-	    LOGDEB2(("SearchData::maybeAddAutoPhrase: complex clause\n"));
+	    LOGDEB2(("SearchData::maybeAddAutoPhrase: rejected clause\n"));
 	    return false;
 	}
 	SearchDataClauseSimple *clp = 
@@ -338,25 +343,57 @@ bool SearchData::maybeAddAutoPhrase()
 	    field = clp->getfield();
 	} else {
 	    if (clp->getfield().compare(field)) {
-		LOGDEB2(("SearchData::maybeAddAutoPhrase: different fields\n"));
+		LOGDEB2(("SearchData::maybeAddAutoPhrase: diff. fields\n"));
 		return false;
 	    }
 	}
-	if (!words.empty())
-	    words += " ";
-	words +=  clp->gettext();
+
+	// If there are wildcards or quotes in there, bail out
+	if (clp->gettext().find_first_of("\"*[?") != string::npos) { 
+	    LOGDEB2(("SearchData::maybeAddAutoPhrase: wildcards\n"));
+	    return false;
+	}
+        // Do a simple word-split here, don't bother with the full-blown
+	// textsplit. The autophrase thing is just "best effort", it's
+	// normal that it won't work in strange cases.
+	vector<string> wl;
+	stringToStrings(clp->gettext(), wl);
+	words.insert(words.end(), wl.begin(), wl.end());
     }
 
-    // If there are wildcards or quotes in there, or this is a single word,
-    // bail out
-    if (words.find_first_of("\"*[?") != string::npos &&
-	TextSplit::countWords(words) <= 1) { 
-	LOGDEB2(("SearchData::maybeAddAutoPhrase: wildcards or single word\n"));
+
+    // Trim the word list by eliminating very frequent terms
+    // (increasing the slack as we do it):
+    int slack = 0;
+    int doccnt = db.docCnt();
+    if (!doccnt)
+	doccnt = 1;
+    string swords;
+    for (list<string>::iterator it = words.begin(); 
+	 it != words.end(); it++) {
+	double freq = double(db.termDocCnt(*it)) / doccnt;
+	if (freq < freqThreshold) {
+	    if (!swords.empty())
+		swords.append(1, ' ');
+	    swords += *it;
+	} else {
+	    LOGDEB0(("Autophrase: [%s] too frequent (%.2f %%)\n", 
+		    it->c_str(), 100 * freq));
+	    slack++;
+	}
+    }
+    
+    // We can't make a phrase with a single word :)
+    if (TextSplit::countWords(swords) <= 1) {
+	LOGDEB2(("SearchData::maybeAddAutoPhrase: ended with 1 word\n"));
 	return false;
     }
-
+	
     SearchDataClauseDist *nclp = 
-	new SearchDataClauseDist(SCLT_PHRASE, words, 0, field);
+	new SearchDataClauseDist(SCLT_PHRASE, swords, slack, field);
+
+    // If the toplevel conjunction is an OR, just OR the phrase, else 
+    // deepen the tree.
     if (m_tp == SCLT_OR) {
 	addClause(nclp);
     } else {
@@ -365,6 +402,7 @@ bool SearchData::maybeAddAutoPhrase()
 	// phrase.
 	SearchData *sd = new SearchData(m_tp);
 	sd->m_query = m_query;
+	sd->m_stemlang = m_stemlang;
 	m_tp = SCLT_OR;
 	m_query.clear();
 	SearchDataClauseSub *oq = 
@@ -556,8 +594,8 @@ void StringToXapianQ::expandTerm(bool nostemexp,
                                  list<string>& exp,
                                  string &sterm, const string& prefix)
 {
-    LOGDEB2(("expandTerm: field [%s] term [%s] stemlang [%s] nostemexp %d\n", 
-             m_field.c_str(), term.c_str(), m_stemlang.c_str(), nostemexp));
+    LOGDEB2(("expandTerm: field [%s] term [%s] stemlang [%s] nostemexp %d\n",
+	     m_field.c_str(), term.c_str(), m_stemlang.c_str(), nostemexp));
     sterm.erase();
     exp.clear();
     if (term.empty()) {
@@ -567,8 +605,10 @@ void StringToXapianQ::expandTerm(bool nostemexp,
     bool haswild = term.find_first_of(cstr_minwilds) != string::npos;
 
     // No stemming if there are wildcards or prevented globally.
-    if (haswild || m_stemlang.empty())
+    if (haswild || m_stemlang.empty()) {
+	LOGDEB2(("expandTerm: found wildcards or stemlang empty: no exp\n"));
 	nostemexp = true;
+    }
 
     if (nostemexp && !haswild) {
 	sterm = term;
@@ -631,6 +671,8 @@ void multiply_groups(vector<vector<string> >::const_iterator vvit,
 void StringToXapianQ::processSimpleSpan(const string& span, bool nostemexp,
 					list<Xapian::Query> &pqueries)
 {
+    LOGDEB2(("StringToXapianQ::processSimpleSpan: [%s] nostemexp %d\n",
+	     span.c_str(), int(nostemexp)));
     list<string> exp;  
     string sterm; // dumb version of user term
 
@@ -866,6 +908,8 @@ bool SearchDataClauseSimple::toNativeQuery(Rcl::Db &db, void *p,
 {
     const string& l_stemlang = (m_modifiers&SDCM_NOSTEMMING)? cstr_null:
 	stemlang;
+    LOGDEB2(("SearchDataClauseSimple::toNativeQuery: stemlang [%s]\n",
+	     stemlang.c_str()));
 
     m_terms.clear();
     m_groups.clear();

src/rcldb/searchdata.h

@@ -98,11 +98,13 @@ public:
     bool addClause(SearchDataClause *cl);
 
     /** If this is a simple query (one field only, no distance clauses),
-	add phrase made of query terms to query, so that docs containing the
-	user terms in order will have higher relevance. This must be called 
-	before toNativeQuery().
-    */
-    bool maybeAddAutoPhrase();
+     * add phrase made of query terms to query, so that docs containing the
+     * user terms in order will have higher relevance. This must be called 
+     * before toNativeQuery().
+     * @param threshold: don't use terms more frequent than the value 
+     *     (proportion of docs where they occur) 	
+     */
+    bool maybeAddAutoPhrase(Rcl::Db &db, double threshold);
 
     /** Set/get top subdirectory for filtering results */
     void setTopdir(const string& t, bool excl = false, float w = 1.0)