Add frequency threshold to avoid adding common term to the automatic phrase search extension. Use autophrase by default with simple search, with a default freq threshold at 2%

src/qtgui/guiutils.cpp

@@ -113,7 +113,9 @@ void rwSettings(bool writing)
 	       Bool, false);
     SETTING_RW(prefs.catgToolBar, "/Recoll/prefs/catgToolBar", Bool, false);
     SETTING_RW(prefs.ssearchAutoPhrase, 
-	       "/Recoll/prefs/ssearchAutoPhrase", Bool, false);
+	       "/Recoll/prefs/ssearchAutoPhrase", Bool, true);
+    SETTING_RW(prefs.ssearchAutoPhraseThreshPC, 
+	       "/Recoll/prefs/ssearchAutoPhraseThreshPC", Double, 2.0);
     SETTING_RW(prefs.respagesize, "/Recoll/prefs/reslist/pagelen", Int, 8);
     SETTING_RW(prefs.collapseDuplicates, 
 	       "/Recoll/prefs/reslist/collapseDuplicates", Bool, false);

src/qtgui/guiutils.h

@@ -85,6 +85,7 @@ class PrefsPack {
     QStringList ssearchHistory;
     // Make phrase out of search terms and add to search in simple search
     bool ssearchAutoPhrase;
+    double ssearchAutoPhraseThreshPC;
     // Ignored file types in adv search (startup default)
     QStringList asearchIgnFilTyps;
     bool        fileTypesByCats;

src/qtgui/ssearch_w.cpp

@@ -132,7 +132,8 @@ void SSearch::startSimpleSearch()
     if (tp == SST_LANG) {
 	string reason;
         if (prefs.autoSuffsEnable)
-            sdata = wasaStringToRcl(theconfig, u8, reason, (const char *)prefs.autoSuffs.toUtf8());
+            sdata = wasaStringToRcl(theconfig, u8, reason, 
+				    (const char *)prefs.autoSuffs.toUtf8());
         else
             sdata = wasaStringToRcl(theconfig, u8, reason);
 	if (sdata == 0) {
@@ -140,51 +141,40 @@ void SSearch::startSimpleSearch()
 				 QString::fromAscii(reason.c_str()));
 	    return;
 	}
-	if (prefs.ssearchAutoPhrase) {
-	    sdata->maybeAddAutoPhrase();
-	}
     } else {
 	sdata = new Rcl::SearchData(Rcl::SCLT_OR);
 	if (sdata == 0) {
 	    QMessageBox::warning(0, "Recoll", tr("Out of memory"));
 	    return;
 	}
-
-	// If there is no white space inside the query, then the user
-	// certainly means it as a phrase.
-	bool isreallyaphrase = false;
-	if (!TextSplit::hasVisibleWhite(u8))
-	    isreallyaphrase = true;
-
-	// Maybe add automatic phrase ? For ALL and ANY, and not if
-	// there is already a phrase or wildcard terms.
-	if (!isreallyaphrase && 
-	    prefs.ssearchAutoPhrase && (tp == SST_ANY || tp == SST_ALL) &&
-	    u8.find_first_of("\"*[]?") == string::npos && 
-	    TextSplit::countWords(u8) > 1) {
-	    sdata->addClause(new Rcl::SearchDataClauseDist(Rcl::SCLT_PHRASE, 
-							   u8, 0));
-	}
 	Rcl::SearchDataClause *clp = 0;
-	switch (tp) {
-	case SST_ANY:
-	default:
-	    clp = isreallyaphrase ? 
-		new Rcl::SearchDataClauseDist(Rcl::SCLT_PHRASE, u8, 0) :
-		new Rcl::SearchDataClauseSimple(Rcl::SCLT_OR, u8);
-	    break;
-	case SST_ALL:
-	    clp = isreallyaphrase ? 
-		new Rcl::SearchDataClauseDist(Rcl::SCLT_PHRASE, u8, 0) :
-		new Rcl::SearchDataClauseSimple(Rcl::SCLT_AND, u8);
-	    break;
-	case SST_FNM:
+	if (tp == SST_FNM) {
 	    clp = new Rcl::SearchDataClauseFilename(u8);
-	    break;
+	} else if (!TextSplit::hasVisibleWhite(u8)) {
+	    // If there is no white space inside the query, then the user
+	    // certainly means it as a phrase.
+	    clp = new Rcl::SearchDataClauseDist(Rcl::SCLT_PHRASE, u8, 0);
+	} else {
+	    // ANY or ALL, several words.
+	    if (tp == SST_ANY) {
+		clp = new Rcl::SearchDataClauseSimple(Rcl::SCLT_OR, u8);
+	    } else {
+		clp = new Rcl::SearchDataClauseSimple(Rcl::SCLT_AND, u8);
+	    }
 	}
 	sdata->addClause(clp);
     }
 
+    if (prefs.ssearchAutoPhrase && rcldb) {
+	string stemLang = (const char *)prefs.queryStemLang.toAscii();
+	if (stemLang == "ALL") {
+	    theconfig->getConfParam("indexstemminglanguages", stemLang);
+	}
+	sdata->setStemlang(stemLang);
+	sdata->maybeAddAutoPhrase(*rcldb, 
+				  prefs.ssearchAutoPhraseThreshPC / 100.0);
+    }
+
     // Search terms history
 
     // We want to have the new text at the top and any older identical

src/qtgui/uiprefs.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>527</width>
+    <width>530</width>
     <height>559</height>
    </rect>
   </property>
@@ -21,6 +21,9 @@
     <layout class="QVBoxLayout">
      <item>
       <widget class="QTabWidget" name="tabWidget">
+       <property name="currentIndex">
+        <number>1</number>
+       </property>
        <widget class="QWidget" name="tab">
         <attribute name="title">
          <string>User interface</string>
@@ -300,12 +303,12 @@
            </item>
            <item>
             <widget class="QCheckBox" name="previewPlainPreCB">
+             <property name="toolTip">
+              <string>Lines in PRE text are not folded. Using BR loses indentation.</string>
+             </property>
              <property name="text">
               <string>Use &lt;PRE&gt; tags instead of &lt;BR&gt;to display plain text as html.</string>
              </property>
-               <property name="toolTip">
-                <string>Lines in PRE text are not folded. Using BR loses indentation.</string>
-               </property>
              <property name="checked">
               <bool>false</bool>
              </property>
@@ -350,6 +353,42 @@ This should give higher precedence to the results where the search terms appear
              </property>
             </widget>
            </item>
+           <item>
+            <layout class="QHBoxLayout">
+             <item>
+              <widget class="QLabel" name="textLabel33">
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+                 <horstretch>1</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Frequency percentage threshold over which we do not use terms inside autophrase. 
+Frequent terms are a major performance issue with phrases. 
+Skipped terms augment the phrase slack, and reduce the autophrase efficiency.
+The default value is 2 (percent). </string>
+               </property>
+               <property name="text">
+                <string>Autophrase term frequency threshold percentage</string>
+               </property>
+               <property name="wordWrap">
+                <bool>false</bool>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QDoubleSpinBox" name="autoPThreshSB">
+               <property name="singleStep">
+                <double>0.200000000000000</double>
+               </property>
+               <property name="value">
+                <double>2.000000000000000</double>
+               </property>
+              </widget>
+             </item>
+            </layout>
+           </item>
            <item>
             <widget class="Line" name="line2">
              <property name="frameShape">

src/qtgui/uiprefs_w.cpp

@@ -140,6 +140,7 @@ void UIPrefsDialog::setFromPrefs()
     stemLangCMB->setCurrentIndex(cur);
 
     autoPhraseCB->setChecked(prefs.ssearchAutoPhrase);
+    autoPThreshSB->setValue(prefs.ssearchAutoPhraseThreshPC);
 
     buildAbsCB->setChecked(prefs.queryBuildAbstract);
     replAbsCB->setEnabled(prefs.queryBuildAbstract);
@@ -199,6 +200,7 @@ void UIPrefsDialog::accept()
 	prefs.queryStemLang = stemLangCMB->currentText();
     }
     prefs.ssearchAutoPhrase = autoPhraseCB->isChecked();
+    prefs.ssearchAutoPhraseThreshPC = autoPThreshSB->value();
     prefs.queryBuildAbstract = buildAbsCB->isChecked();
     prefs.queryReplaceAbstract = buildAbsCB->isChecked() && 
 	replAbsCB->isChecked();