fixed stemming, was completely broken in 1.13

src/index/indexer.cpp

@@ -195,6 +195,10 @@ bool ConfIndexer::createStemmingDatabases()
 {
     string slangs;
     if (m_config->getConfParam("indexstemminglanguages", slangs)) {
+        if (!m_db.open(Rcl::Db::DbRO)) {
+            LOGERR(("ConfIndexer::createStemmingDb: could not open db\n"))
+            return false;
+        }
 	list<string> langs;
 	stringToStrings(slangs, langs);
 
@@ -215,6 +219,7 @@ bool ConfIndexer::createStemmingDatabases()
 	    m_db.createStemDb(*it);
 	}
     }
+    m_db.close();
     return true;
 }
 
@@ -245,6 +250,7 @@ bool ConfIndexer::createAspellDict()
 	return true;
 
     if (!m_db.open(Rcl::Db::DbRO)) {
+        LOGERR(("ConfIndexer::createAspellDict: could not open db\n"));
 	return false;
     }
 

src/rcldb/stemdb.cpp

@@ -75,6 +75,30 @@ p_notlowerascii(unsigned int c)
     return false;
 }
 
+static bool addAssoc(Xapian::WritableDatabase &sdb, const string& stem,
+                     const list<string>& derivs)
+{
+    Xapian::Document newdocument;
+    newdocument.add_term(stem);
+    // The doc data is just parents=blank-separated-list
+    string record = "parents=";
+    for (list<string>::const_iterator it = derivs.begin(); 
+         it != derivs.end(); it++) {
+        record += *it + " ";
+    }
+    record += "\n";
+    LOGDEB2(("createStemDb: stmdoc data: [%s]\n", record.c_str()));
+    newdocument.set_data(record);
+    try {
+        sdb.replace_document(stem, newdocument);
+    } catch (...) {
+        LOGERR(("Db::createstemdb(addAssoc): replace failed\n"));
+        return false;
+    }
+    return true;
+}
+
+
 /**
  * Create database of stem to parents associations for a given language.
  * We walk the list of all terms, stem them, and create another Xapian db
@@ -102,8 +126,7 @@ bool createDb(Xapian::Database& xdb, const string& dbdir, const string& lang)
     try {
         Xapian::Stem stemmer(lang);
         Xapian::TermIterator it;
-	for (it = xdb.allterms_begin(); 
+        for (it = xdb.allterms_begin(); it != xdb.allterms_end(); it++) {
-	     it != xdb.allterms_end(); it++) {
             // Deciding if we try to stem the term. If it has any
             // non-lowercase 7bit char (that is, numbers, capitals and
             // punctuation) dont. We're still sending all multibyte
@@ -113,12 +136,13 @@ bool createDb(Xapian::Database& xdb, const string& dbdir, const string& lang)
             string::iterator sit = (*it).begin(), eit = sit + (*it).length();
             if ((sit = find_if(sit, eit, p_notlowerascii)) != eit) {
                 ++nostem;
-		// LOGDEB(("stemskipped: [%s], because of 0x%x\n", 
+                LOGDEB1(("stemskipped: [%s], because of 0x%x\n", 
-		// (*it).c_str(), *sit));
+                         (*it).c_str(), *sit));
                 continue;
             }
             string stem = stemmer(*it);
-	    //cerr << "word " << *it << " stem " << stem << endl;
+            LOGDEB2(("Db::createStemDb: word [%s], stem [%s]\n", (*it).c_str(),
+                     stem.c_str()));
             if (stem == *it) {
                 ++stemconst;
                 continue;
@@ -161,8 +185,7 @@ bool createDb(Xapian::Database& xdb, const string& dbdir, const string& lang)
         return false;
     }
 
-    // Enter pseud-docs in db. Walk the multimap, only enter
+    // Enter pseud-docs in db by walking the multimap.
-    // associations where there are several parent terms
     string stem;
     list<string> derivs;
     for (multimap<string,string>::const_iterator it = assocs.begin();
@@ -174,37 +197,35 @@ bool createDb(Xapian::Database& xdb, const string& dbdir, const string& lang)
         } else {
             // Changing stems 
             ++stemdiff;
+            LOGDEB2(("createStemDb: stem [%s]\n", stem.c_str()));
+
             // We need an entry even if there is only one derivative
             // so that it is possible to search by entering the stem
             // even if it doesnt exist as a term
-	    if (derivs.size() >= 1) {
+            if (!derivs.empty()) {
-		// Previous stem has multiple derivatives. Enter in db
+
+                if (derivs.size() > 1)
                     ++stemmultiple;
-		Xapian::Document newdocument;
+                    
-		newdocument.add_term(stem);
+                if (!addAssoc(sdb, stem, derivs)) {
-		// The doc data is just parents=blank-separated-list
-		string record = "parents=";
-		for (list<string>::const_iterator it = derivs.begin(); 
-		     it != derivs.end(); it++) {
-		    record += *it + " ";
-		}
-		record += "\n";
-		LOGDEB1(("stemdocument data: %s\n", record.c_str()));
-		newdocument.set_data(record);
-		try {
-		    sdb.replace_document(stem, newdocument);
-		    //sdb.add_document(newdocument);
-		} catch (...) {
-		    LOGERR(("Db::createstemdb: replace failed\n"));
                     return false;
                 }
-	    }
                 derivs.clear();
+            }
             stem = it->first;
             derivs.push_back(it->second);
             //	    cerr << "\n" << stem << " " << it->second;
         }
     }
+    if (!derivs.empty()) {
+        if (derivs.size() > 1)
+            ++stemmultiple;
+                    
+        if (!addAssoc(sdb, stem, derivs)) {
+            return false;
+        }
+    }
+
     LOGDEB1(("StemDb::createDb(%s): done: %.2f S\n", 
              lang.c_str(), cron.secs()));
     LOGDEB(("Stem map size: %d stems %d mult %d no %d const %d\n",