merge stemExpand into termExpand. return term frequencies from there and display in spellW

src/rcldb/rcldb.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.100 2006-12-07 13:24:19 dockes Exp $ (C) 2004 J.F.Dockes";
+static char rcsid[] = "@(#$Id: rcldb.cpp,v 1.101 2006-12-19 12:11:21 dockes Exp $ (C) 2004 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -1285,6 +1285,42 @@ bool Db::setQuery(RefCntr<SearchData> sdata, int opts,
     return true;
 }
 
+class TermMatchCmpByWcf {
+public:
+    int operator()(const TermMatchEntry& l, const TermMatchEntry& r) {
+	return r.wcf - l.wcf < 0;
+    }
+};
+class TermMatchCmpByTerm {
+public:
+    int operator()(const TermMatchEntry& l, const TermMatchEntry& r) {
+	return l.term.compare(r.term) > 0;
+    }
+};
+class TermMatchTermEqual {
+public:
+    int operator()(const TermMatchEntry& l, const TermMatchEntry& r) {
+	return !l.term.compare(r.term);
+    }
+};
+
+bool Db::stemExpand(const string &lang, const string &term, 
+		    list<TermMatchEntry>& result, int max)
+{
+    list<string> dirs = m_extraDbs;
+    dirs.push_front(m_basedir);
+    for (list<string>::iterator it = dirs.begin();
+	 it != dirs.end(); it++) {
+	list<string> more;
+	StemDb::stemExpand(*it, lang, term, more);
+	LOGDEB1(("Db::stemExpand: Got %d from %s\n", 
+		 more.size(), it->c_str()));
+	result.insert(result.end(), more.begin(), more.end());
+    }
+    LOGDEB1(("Db:::stemExpand: final count %d \n", result.size()));
+    return true;
+}
+
 // Characters that can begin a wildcard or regexp expression. We use skipto
 // to begin the allterms search with terms that begin with the portion of
 // the input string prior to these chars.
@@ -1292,85 +1328,97 @@ const string wildSpecChars = "*?[";
 const string regSpecChars = "(.[{";
 
 // Find all index terms that match a wildcard or regular expression
-bool Db::termMatch(MatchType typ, const string &root, list<string>& res,
-		     const string &lang, int max)
+bool Db::termMatch(MatchType typ, const string &lang,
+		   const string &root, 
+		   list<TermMatchEntry>& res,
+		   int max)
 {
     if (!m_ndb || !m_ndb->m_isopen)
 	return false;
+
     Xapian::Database db = m_ndb->m_iswritable ? m_ndb->wdb: m_ndb->db;
+
     res.clear();
+
     // Get rid of capitals and accents
     string droot;
     dumb_string(root, droot);
     string nochars = typ == ET_WILD ? wildSpecChars : regSpecChars;
 
-    regex_t reg;
-    int errcode;
-    // Compile regexp. We anchor the input by enclosing it in ^ and $
-    if (typ == ET_REGEXP) {
-	string mroot = droot;
-	if (mroot.at(0) != '^')
-	    mroot = string("^") + mroot;
-	if (mroot.at(mroot.length()-1) != '$')
-	    mroot += "$";
-	if ((errcode = regcomp(&reg, mroot.c_str(), REG_EXTENDED|REG_NOSUB))) {
-	    char errbuf[200];
-	    regerror(errcode, &reg, errbuf, 199);
-	    LOGERR(("termMatch: regcomp failed: %s\n", errbuf));
-	    res.push_back(errbuf);
-	    regfree(&reg);
+    if (typ == ET_STEM) {
+	if (!stemExpand(lang, root, res, max))
 	    return false;
+	for (list<TermMatchEntry>::iterator it = res.begin(); 
+	     it != res.end(); it++) {
+	    it->wcf = db.get_collection_freq(it->term);
+	    LOGDEB(("termMatch: %d [%s]\n", it->wcf, it->term.c_str()));
 	}
-    }
-
-    // Find the initial section before any special char
-    string::size_type es = droot.find_first_of(nochars);
-    string is;
-    switch (es) {
-    case string::npos: is = droot;break;
-    case 0: break;
-    default: is = droot.substr(0, es);break;
-    }
-    LOGDEB(("termMatch: initsec: [%s]\n", is.c_str()));
-
-    Xapian::TermIterator it = db.allterms_begin(); 
-    if (!is.empty())
-	it.skip_to(is.c_str());
-    for (int n = 0;it != db.allterms_end(); it++) {
-        // If we're beyond the terms matching the initial string, end
-	if (!is.empty() && (*it).find(is) != 0)
-	    break;
-	// Don't match special internal terms beginning with uppercase ascii
-	if ((*it).at(0) >= 'A' && (*it).at(0) <= 'Z')
-	    continue;
-	if (typ == ET_WILD) {
-	    if (fnmatch(droot.c_str(), (*it).c_str(), 0) == FNM_NOMATCH)
-		continue;
-	} else {
-	    if (regexec(&reg, (*it).c_str(), 0, 0, 0))
-		continue;
-	}
-	// Do we want stem expansion here? We don't do it for now
-	if (1 || lang.empty()) {
-	    res.push_back(*it);
-	    ++n;
-	} else {
-	    list<string> stemexps = stemExpand(lang, *it);
-	    unsigned int cnt = 
-		(int)stemexps.size() > max - n ? max - n : stemexps.size();
-	    list<string>::iterator sit = stemexps.begin();
-	    while (cnt--) {
-		res.push_back(*sit++);
-		n++;
+    } else {
+	regex_t reg;
+	int errcode;
+	if (typ == ET_REGEXP) {
+	    // Compile regexp. We anchor the input by enclosing it in ^ and $
+	    string mroot = droot;
+	    if (mroot.at(0) != '^')
+		mroot = string("^") + mroot;
+	    if (mroot.at(mroot.length()-1) != '$')
+		mroot += "$";
+	    if ((errcode = regcomp(&reg, mroot.c_str(), 
+				   REG_EXTENDED|REG_NOSUB))) {
+		char errbuf[200];
+		regerror(errcode, &reg, errbuf, 199);
+		LOGERR(("termMatch: regcomp failed: %s\n", errbuf));
+		res.push_back(string(errbuf));
+		regfree(&reg);
+		return false;
 	    }
 	}
-	if (n >= max)
-	    break;
+
+	// Find the initial section before any special char
+	string::size_type es = droot.find_first_of(nochars);
+	string is;
+	switch (es) {
+	case string::npos: is = droot;break;
+	case 0: break;
+	default: is = droot.substr(0, es);break;
+	}
+	LOGDEB(("termMatch: initsec: [%s]\n", is.c_str()));
+
+	Xapian::TermIterator it = db.allterms_begin(); 
+	if (!is.empty())
+	    it.skip_to(is.c_str());
+	for (int n = 0;it != db.allterms_end(); it++) {
+	    // If we're beyond the terms matching the initial string, end
+	    if (!is.empty() && (*it).find(is) != 0)
+		break;
+	    // Don't match special internal terms beginning with uppercase ascii
+	    if ((*it).at(0) >= 'A' && (*it).at(0) <= 'Z')
+		continue;
+	    if (typ == ET_WILD) {
+		if (fnmatch(droot.c_str(), (*it).c_str(), 0) == FNM_NOMATCH)
+		    continue;
+	    } else {
+		if (regexec(&reg, (*it).c_str(), 0, 0, 0))
+		    continue;
+	    }
+	    // Do we want stem expansion here? We don't do it for now
+	    res.push_back(TermMatchEntry(*it, it.get_termfreq()));
+	    ++n;
+	}
+	if (typ == ET_REGEXP) {
+	    regfree(&reg);
+	}
+
     }
-    res.sort();
-    res.unique();
-    if (typ == ET_REGEXP) {
-	regfree(&reg);
+
+    TermMatchCmpByTerm tcmp;
+    res.sort(tcmp);
+    TermMatchTermEqual teq;
+    res.unique(teq);
+    TermMatchCmpByWcf wcmp;
+    res.sort(wcmp);
+    if (max > 0) {
+	res.resize(MIN(res.size(), (unsigned int)max));
     }
     return true;
 }
@@ -1417,23 +1465,6 @@ bool Db::termExists(const string& word)
     return true;
 }
 
-list<string> Db::stemExpand(const string& lang, const string& term) 
-{
-    list<string> dirs = m_extraDbs;
-    dirs.push_front(m_basedir);
-    list<string> exp;
-    for (list<string>::iterator it = dirs.begin();
-	 it != dirs.end(); it++) {
-	list<string> more = StemDb::stemExpand(*it, lang, term);
-	LOGDEB1(("Db::stemExpand: Got %d from %s\n", 
-		 more.size(), it->c_str()));
-	exp.splice(exp.end(), more);
-    }
-    exp.sort();
-    exp.unique();
-    LOGDEB1(("Db:::stemExpand: final count %d \n", exp.size()));
-    return exp;
-}
 
 bool Db::stemDiffers(const string& lang, const string& word, 
 		     const string& base)

src/rcldb/rcldb.h

@@ -16,7 +16,7 @@
  */
 #ifndef _DB_H_INCLUDED_
 #define _DB_H_INCLUDED_
-/* @(#$Id: rcldb.h,v 1.44 2006-12-14 14:54:13 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: rcldb.h,v 1.45 2006-12-19 12:11:21 dockes Exp $  (C) 2004 J.F.Dockes */
 
 #include <string>
 #include <list>
@@ -53,7 +53,16 @@ namespace Rcl {
 class SearchData;
 class Native;
 class TermIter;
- 
+
+class TermMatchEntry {
+public:
+    TermMatchEntry() : wcf(0) {}
+    TermMatchEntry(const string&t, int f) : term(t), wcf(f) {}
+    TermMatchEntry(const string&t) : term(t), wcf(0) {}
+    string term;
+    int    wcf; // Within collection frequency
+};
+
 /**
  * Wrapper class for the native database.
  */
@@ -109,9 +118,9 @@ class Db {
     /** Return a list of index terms that match the input string
      * Expansion is performed either with either wildcard or regexp processing
      * Stem expansion is performed if lang is not empty */
-    enum MatchType {ET_WILD, ET_REGEXP};
-    bool termMatch(MatchType typ, const string &s, list<string>& result,
-		   const string &lang, int max=20);
+    enum MatchType {ET_WILD, ET_REGEXP, ET_STEM};
+    bool termMatch(MatchType typ, const string &lang, const string &s, 
+		   list<TermMatchEntry>& result, int max = -1);
 
     /** Add extra database for querying */
     bool addQueryDb(const string &dir);
@@ -159,12 +168,11 @@ class Db {
     bool stemDiffers(const string& lang, const string& term, 
 		     const string& base);
     
-    /** Perform stem expansion across all dbs configured for searching */
-    list<string> stemExpand(const string& lang, const string& term);
-
     /** Filename wildcard expansion */
     bool filenameWildExp(const string& exp, list<string>& names);
     string getReason(){return m_reason;}
+
+
 private:
 
     string m_filterTopDir; // Current query filter on subtree top directory 
@@ -201,6 +209,8 @@ private:
     vector<bool> updated;
 
     bool reOpen(); // Close/open, same mode/opts
+    bool stemExpand(const string &lang, const string &s, 
+		    list<TermMatchEntry>& result, int max = -1);
 
     /* Copyconst and assignemt private and forbidden */
     Db(const Db &) {}

src/rcldb/searchdata.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.6 2006-11-30 13:38:44 dockes Exp $ (C) 2006 J.F.Dockes";
+static char rcsid[] = "@(#$Id: searchdata.cpp,v 1.7 2006-12-19 12:11:21 dockes Exp $ (C) 2006 J.F.Dockes";
 #endif
 /*
  *   This program is free software; you can redistribute it and/or modify
@@ -214,7 +214,12 @@ void StringToXapianQ::maybeStemExp(bool nostemexp,
     if (nostemexp) {
 	exp = list<string>(1, term1);
     } else {
-	exp = m_db.stemExpand(m_stemlang, term1);
+	list<TermMatchEntry> l;
+	m_db.termMatch(Rcl::Db::ET_STEM, m_stemlang, term1, l);
+	for (list<TermMatchEntry>::const_iterator it = l.begin(); 
+	     it != l.end(); it++) {
+	    exp.push_back(it->term);
+	}
     }
 }
 

src/rcldb/stemdb.cpp

@@ -1,5 +1,5 @@
 #ifndef lint
-static char rcsid[] = "@(#$Id: stemdb.cpp,v 1.5 2006-10-09 16:37:08 dockes Exp $ (C) 2005 J.F.Dockes";
+static char rcsid[] = "@(#$Id: stemdb.cpp,v 1.6 2006-12-19 12:11:21 dockes Exp $ (C) 2005 J.F.Dockes";
 #endif
 
 /**
@@ -206,13 +206,24 @@ bool createDb(Xapian::Database& xdb, const string& dbdir, const string& lang)
     return true;
 }
 
+static string stringlistdisp(const list<string>& sl)
+{
+    string s;
+    for (list<string>::const_iterator it = sl.begin(); it!= sl.end(); it++)
+	s += "[" + *it + "] ";
+    if (!s.empty())
+	s.erase(s.length()-1);
+    return s;
+}
+
 /**
  * Expand term to list of all terms which stem to the same term.
  */
-list<string> stemExpand(const string& dbdir, const string& lang,
-			const string& term)
+bool stemExpand(const std::string& dbdir, 
+		const std::string& lang,
+		const std::string& term,
+		list<string>& result)
 {
-    list<string> explist;
     try {
 	Xapian::Stem stemmer(lang);
 	string stem = stemmer.stem_word(term);
@@ -224,14 +235,14 @@ list<string> stemExpand(const string& dbdir, const string& lang,
 		stemdbdir.c_str(), sdb.get_lastdocid()));
 	if (!sdb.term_exists(stem)) {
 	    LOGDEB1(("Db::stemExpand: no term for %s\n", stem.c_str()));
-	    explist.push_back(term);
-	    return explist;
+	    result.push_back(term);
+	    return true;
 	}
 	Xapian::PostingIterator did = sdb.postlist_begin(stem);
 	if (did == sdb.postlist_end(stem)) {
 	    LOGDEB1(("stemExpand: no term(1) for %s\n",stem.c_str()));
-	    explist.push_back(term);
-	    return explist;
+	    result.push_back(term);
+	    return true;
 	}
 	Xapian::Document doc = sdb.get_document(*did);
 	string data = doc.get_data();
@@ -242,24 +253,24 @@ list<string> stemExpand(const string& dbdir, const string& lang,
 	++pos;
 	string::size_type pos1 = data.find_last_of("\n");
 	if (pos == string::npos || pos1 == string::npos ||pos1 <= pos) { // ??
-	    explist.push_back(term);
-	    return explist;
+	    result.push_back(term);
+	    return true;
 	}
-	stringToStrings(data.substr(pos, pos1-pos), explist);
+	stringToStrings(data.substr(pos, pos1-pos), result);
 
 	// If the user term itself is not in the list, add it.
-	if (find(explist.begin(), explist.end(), term) == explist.end()) {
-	    explist.push_back(term);
+	if (find(result.begin(), result.end(), term) == result.end()) {
+	    result.push_back(term);
 	}
 	LOGDEB(("stemExpand: %s ->  %s\n", stem.c_str(),
-		stringlistdisp(explist).c_str()));
+		stringlistdisp(result).c_str()));
     } catch (...) {
 	LOGERR(("stemExpand: error accessing stem db. dbdir [%s] lang [%s]\n",
 		dbdir.c_str(), lang.c_str()));
-	explist.push_back(term);
-	return explist;
+	result.push_back(term);
+	return false;
     }
-    return explist;
+    return true;
 }
 
 }

src/rcldb/stemdb.h

@@ -1,6 +1,6 @@
 #ifndef _STEMDB_H_INCLUDED_
 #define _STEMDB_H_INCLUDED_
-/* @(#$Id: stemdb.h,v 1.2 2006-11-15 14:57:53 dockes Exp $  (C) 2004 J.F.Dockes */
+/* @(#$Id: stemdb.h,v 1.3 2006-12-19 12:11:21 dockes Exp $  (C) 2004 J.F.Dockes */
 /// Stem database code
 /// 
 /// Stem databases list stems and the set of index terms they expand to. They 
@@ -13,6 +13,7 @@
 #include <string>
 
 #include <xapian.h>
+
 #ifndef NO_NAMESPACES
 using std::string;
 using std::list;
@@ -28,10 +29,10 @@ extern bool deleteDb(const std::string& dbdir, const std::string& lang);
 extern bool createDb(Xapian::Database& xdb, 
 		     const std::string& dbdir, const std::string& lang);
 /// Expand term to stem siblings
-extern std::list<std::string> stemExpand(const std::string& dbdir, 
-					 const std::string& lang,
-					 const std::string& term);
-
+extern bool stemExpand(const std::string& dbdir, 
+		       const std::string& lang,
+		       const std::string& term,
+		       list<string>& result);
 #ifndef NO_NAMESPACES
 }
 }