Make Recoll optionally sensitive to case and diacritics

--HG--
branch : CASEDIACSENS

src/query/docseqdb.cpp

@@ -120,7 +120,8 @@ bool DocSequenceDb::setFiltSpec(const DocSeqFiltSpec &fs)
     LOGDEB(("DocSequenceDb::setFiltSpec\n"));
     if (fs.isNotNull()) {
 	// We build a search spec by adding a filtering layer to the base one.
-	m_fsdata = RefCntr<Rcl::SearchData>(new Rcl::SearchData(Rcl::SCLT_AND));
+	m_fsdata = RefCntr<Rcl::SearchData>(
+	    new Rcl::SearchData(Rcl::SCLT_AND, m_sdata->getStemLang()));
 	Rcl::SearchDataClauseSub *cl = 
 	    new Rcl::SearchDataClauseSub(Rcl::SCLT_SUB, m_sdata);
 	m_fsdata->addClause(cl);
@@ -138,6 +139,7 @@ bool DocSequenceDb::setFiltSpec(const DocSeqFiltSpec &fs)
 		string reason;
 		Rcl::SearchData *sd = 
 		    wasaStringToRcl(m_q->whatDb()->getConf(), 
+				    m_sdata->getStemLang(),
 				    fs.values[i], reason);
 		if (sd)  {
 		    Rcl::SearchDataClauseSub *cl1 = 

src/query/plaintorich.cpp

@@ -50,7 +50,10 @@ static string vecStringToString(const vector<string>& t)
 }
 
 struct MatchEntry {
+    // Start/End byte offsets in the document text
     pair<int, int> offs;
+    // Index of the search group this comes from: this is to relate a 
+    // match to the original user input.
     unsigned int grpidx;
     MatchEntry(int sta, int sto, unsigned int idx) 
 	: offs(sta, sto), grpidx(idx)
@@ -76,11 +79,23 @@ class TextSplitPTR : public TextSplit {
 	for (vector<vector<string> >::const_iterator vit = hdata.groups.begin();
 	     vit != hdata.groups.end(); vit++) {
 	    if (vit->size() == 1) {
+#ifdef RCL_INDEX_STRIPCHARS
 		m_terms[vit->front()] = vit - hdata.groups.begin();
+#else
+		string dumb = vit->front();
+		unacmaybefold(vit->front(), dumb, "UTF-8", UNACOP_UNACFOLD);
+		m_terms[dumb] = vit - hdata.groups.begin();
+#endif
 	    } else if (vit->size() > 1) {
 		for (vector<string>::const_iterator it = vit->begin(); 
 		     it != vit->end(); it++) {
+#ifdef RCL_INDEX_STRIPCHARS
 		    m_gterms.insert(*it);
+#else
+		    string dumb = *it;
+		    unacmaybefold(*it, dumb, "UTF-8", UNACOP_UNACFOLD);
+		    m_gterms.insert(dumb);
+#endif
 		}
 	    }
 	}

src/query/recollq.cpp

@@ -286,7 +286,7 @@ int recollq(RclConfig **cfp, int argc, char **argv)
     Rcl::SearchData *sd = 0;
 
     if (op_flags & (OPT_a|OPT_o|OPT_f)) {
-	sd = new Rcl::SearchData(Rcl::SCLT_OR);
+	sd = new Rcl::SearchData(Rcl::SCLT_OR, stemlang);
 	Rcl::SearchDataClause *clp = 0;
 	if (op_flags & OPT_f) {
 	    clp = new Rcl::SearchDataClauseFilename(qs);
@@ -305,14 +305,13 @@ int recollq(RclConfig **cfp, int argc, char **argv)
 	if (sd)
 	    sd->addClause(clp);
     } else {
-	sd = wasaStringToRcl(rclconfig, qs, reason);
+	sd = wasaStringToRcl(rclconfig, stemlang, qs, reason);
     }
 
     if (!sd) {
 	cerr << "Query string interpretation failed: " << reason << endl;
 	return 1;
     }
-    sd->setStemlang(stemlang);
 
     RefCntr<Rcl::SearchData> rq(sd);
     Rcl::Query query(&rcldb);

src/query/wasatorcl.cpp

@@ -32,7 +32,9 @@ using std::list;
 #include "refcntr.h"
 #include "textsplit.h"
 
-static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa, 
+static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, 
+				       const string& stemlang,
+				       WasaQuery *wasa, 
 				       const string& autosuffs, string& reason)
 {
     if (wasa == 0) {
@@ -47,7 +49,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
 
     Rcl::SearchData *sdata = new 
 	Rcl::SearchData(wasa->m_op == WasaQuery::OP_AND ? Rcl::SCLT_AND : 
-			Rcl::SCLT_OR);
+			Rcl::SCLT_OR, stemlang);
     LOGDEB2(("wasaQueryToRcl: %s chain\n", wasa->m_op == WasaQuery::OP_AND ? 
 	     "AND" : "OR"));
 
@@ -250,7 +252,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
 		     (*it)->m_fieldspec.c_str(), (*it)->m_value.c_str()));
 	    // Create a subquery.
 	    Rcl::SearchData *sub = 
-		wasaQueryToRcl(config, *it, autosuffs, reason);
+		wasaQueryToRcl(config, stemlang, *it, autosuffs, reason);
 	    if (sub == 0) {
 		continue;
 	    }
@@ -278,7 +280,7 @@ static Rcl::SearchData *wasaQueryToRcl(RclConfig *config, WasaQuery *wasa,
     return sdata;
 }
 
-Rcl::SearchData *wasaStringToRcl(RclConfig *config, 
+Rcl::SearchData *wasaStringToRcl(RclConfig *config, const string& stemlang,
 				 const string &qs, string &reason, 
                                  const string& autosuffs)
 {
@@ -286,5 +288,5 @@ Rcl::SearchData *wasaStringToRcl(RclConfig *config,
     WasaQuery *wq = parser.stringToQuery(qs, reason);
     if (wq == 0) 
 	return 0;
-    return wasaQueryToRcl(config, wq, autosuffs, reason);
+    return wasaQueryToRcl(config, stemlang, wq, autosuffs, reason);
 }

src/query/wasatorcl.h

@@ -25,7 +25,7 @@ using std::string;
 
 class RclConfig;
 
-extern Rcl::SearchData *wasaStringToRcl(RclConfig *, 
+extern Rcl::SearchData *wasaStringToRcl(RclConfig *, const string& stemlang,
 					const string& query, string &reason,
 					const string& autosuffs = string());
 #endif /* _WASATORCL_H_INCLUDED_ */

src/query/xadump.cpp

@@ -14,6 +14,9 @@
  *   Free Software Foundation, Inc.,
  *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  */
+
+#include "autoconfig.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <signal.h>
@@ -36,7 +39,8 @@ using namespace std;
 static string thisprog;
 
 static string usage =
-    " -d <dbdir> -e <output encoding>\n"
+    " -d <dbdir> \n"
+    "-e <output encoding>\n"
     " -i docid -D : get document data for docid\n"
     " -i docid -X : delete document docid\n"
     " -i docid -b : 'rebuild' document from term positions\n"
@@ -112,6 +116,15 @@ static void sigcleanup(int sig)
     exit(1);
 }
 
+inline bool has_prefix(const string& trm)
+{
+#ifdef RCL_INDEX_STRIPCHARS
+    return trm.size() && 'A' <= trm[0] && trm[0] <= 'Z';
+#else
+    return trm.size() > 0 && trm[0] == ':';
+#endif
+}
+
 int main(int argc, char **argv)
 {
     string dbdir = path_cat(path_home(), ".recoll/xapiandb");
@@ -201,8 +214,7 @@ int main(int argc, char **argv)
 		for (term = db->termlist_begin(docid); 
 		     term != db->termlist_end(docid);term++) {
 		    const string& s = *term;
-		    if ((op_flags&OPT_l) && 
-			!s.empty() && s[0] >= 'A' && s[0] <= 'Z')
+		    if ((op_flags&OPT_l) && has_prefix(s))
 			continue;
 		    cout << op << detailstring(s) << cl << endl;
 		}
@@ -210,8 +222,7 @@ int main(int argc, char **argv)
 		for (term = db->allterms_begin(); 
 		     term != db->allterms_end();term++) {
 		    const string& s = *term;
-		    if ((op_flags&OPT_l) && 
-			!s.empty() && s[0] >= 'A' && s[0] <= 'Z')
+		    if ((op_flags&OPT_l) && has_prefix(s))
 			continue;
 		    if (op_flags & OPT_f)
 			cout <<  db->get_collection_freq(*term) << " "