Highlighting and snippets extraction: reworked to handle phrases properly. Use a compound position list instead of multiplying the OR groups inside a near clause

src/rcldb/rclabsfromtext.cpp

@@ -119,10 +119,12 @@ public:
 
         // Take note of the group (phrase/near) terms because we need
         // to compute the position lists for them.
-        for (const auto& group : hdata.groups) {
-            if (group.size() > 1) {
-                for (const auto& term: group) {
-                    m_gterms.insert(term);
+        for (const auto& tg : hdata.index_term_groups) {
+            if (tg.kind != HighlightData::TermGroup::TGK_TERM) {
+                for (const auto& group : tg.orgroups) {
+                    for (const auto& term: group) {
+                        m_gterms.insert(term);
+                    }
                 }
             }
         }
@@ -134,7 +136,9 @@ public:
         LOGDEB2("takeword: " << term << endl);
         // Limit time taken with monster documents. The resulting
         // abstract will be incorrect or inexistant, but this is
-        // better than taking forever (the default cutoff is 10E6)
+        // better than taking forever (the default cutoff value comes
+        // from the snippetMaxPosWalk configuration parameter, and is
+        // 10E6)
         if (maxtermcount && termcount++ > maxtermcount) {
             LOGINF("Rclabsfromtext: stopping because maxtermcount reached: "<<
                    maxtermcount << endl);
@@ -276,8 +280,9 @@ public:
         // Look for matches to PHRASE and NEAR term groups and finalize
         // the matched regions list (sort it by increasing start then
         // decreasing length). We process all groups as NEAR (ignore order).
-        for (unsigned int i = 0; i < m_hdata.groups.size(); i++) {
-            if (m_hdata.groups[i].size() > 1) {
+        for (unsigned int i = 0; i < m_hdata.index_term_groups.size(); i++) {
+            if (m_hdata.index_term_groups[i].kind !=
+                HighlightData::TermGroup::TGK_TERM) {
                 matchGroup(m_hdata, i, m_plists, m_gpostobytes, tboffs);
             }
         }

src/rcldb/rclabstract.cpp

@@ -163,7 +163,7 @@ double Query::Native::qualityTerms(Xapian::docid docid,
     // expanded from (by stemming)
     map<string, vector<string> > byRoot;
     for (const auto& term: terms) {
-        map<string, string>::const_iterator eit = hld.terms.find(term);
+        const auto eit = hld.terms.find(term);
         if (eit != hld.terms.end()) {
             byRoot[eit->second].push_back(term);
         } else {
@@ -174,9 +174,7 @@ double Query::Native::qualityTerms(Xapian::docid docid,
 
 #ifdef DEBUGABSTRACT
     {
-        string deb;
-        hld.toString(deb);
-        LOGABS("qualityTerms: hld: " << deb << "\n");
+        LOGABS("qualityTerms: hld: " << hld.toString() << "\n");
         string byRootstr;
         for (const auto& entry : byRoot) {
             byRootstr.append("[").append(entry.first).append("]->");

src/rcldb/searchdatatox.cpp

@@ -603,11 +603,11 @@ void SearchDataClauseSimple::processSimpleSpan(
 	return;
     
     // Set up the highlight data. No prefix should go in there
-    for (vector<string>::const_iterator it = exp.begin(); 
-	 it != exp.end(); it++) {
-	m_hldata.groups.push_back(vector<string>(1, it->substr(prefix.size())));
-	m_hldata.slacks.push_back(0);
-	m_hldata.grpsugidx.push_back(m_hldata.ugroups.size() - 1);
+    for (const auto& term : exp) {
+        HighlightData::TermGroup tg;
+        tg.term = term.substr(prefix.size());
+        tg.grpsugidx =  m_hldata.ugroups.size() - 1;
+        m_hldata.index_term_groups.push_back(tg);
     }
 
     // Push either term or OR of stem-expanded set
@@ -735,18 +735,16 @@ void SearchDataClauseSimple::processPhraseOrNear(Rcl::Db &db, string& ermsg,
 			   original_term_wqf_booster);
     pqueries.push_back(xq);
 
-    // Add all combinations of NEAR/PHRASE groups to the highlighting data. 
-    vector<vector<string> > allcombs;
-    vector<string> comb;
-    multiply_groups(groups.begin(), groups.end(), comb, allcombs);
-    
     // Insert the search groups and slacks in the highlight data, with
     // a reference to the user entry that generated them:
-    m_hldata.groups.insert(m_hldata.groups.end(), 
-			   allcombs.begin(), allcombs.end());
-    m_hldata.slacks.insert(m_hldata.slacks.end(), allcombs.size(), slack);
-    m_hldata.grpsugidx.insert(m_hldata.grpsugidx.end(), allcombs.size(), 
-			      m_hldata.ugroups.size() - 1);
+    HighlightData::TermGroup tg;
+    tg.orgroups = groups;
+    tg.slack = slack;
+    tg.grpsugidx =  m_hldata.ugroups.size() - 1;
+    tg.kind = (op == Xapian::Query::OP_PHRASE) ?
+        HighlightData::TermGroup::TGK_PHRASE :
+        HighlightData::TermGroup::TGK_NEAR;
+    m_hldata.index_term_groups.push_back(tg);
 }
 
 // Trim string beginning with ^ or ending with $ and convert to flags