tris/recoll
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cosmetics: use derived class for actual splitter instead of callback

Browse Source
This commit is contained in:
Jean-Francois Dockes
2010-02-02 15:33:52 +01:00
parent 90a8280f21
commit 8b2b00bc72
9 changed files with 114 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rcldb/rcldb.cpp
+16 -17
View File
@@ -781,16 +781,15 @@ bool Db::fieldToPrefix(const string& fld, string &pfx)
}
// The text splitter callback class which receives words from the
// splitter and adds postings to the Xapian document.
class mySplitterCB : public TextSplitCB {
// The splitter breaks text into words and adds postings to the Xapian document.
class TextSplitDb : public TextSplit {
public:
Xapian::Document &doc; // Xapian document
Xapian::termpos basepos; // Base for document section
Xapian::termpos curpos; // Current position. Used to set basepos for the
// following section
StopList &stops;
mySplitterCB(Xapian::Document &d, StopList &_stops)
TextSplitDb(Xapian::Document &d, StopList &_stops)
: doc(d), basepos(1), curpos(0), stops(_stops)
{}
bool takeword(const std::string &term, int pos, int, int);
@@ -802,15 +801,16 @@ private:
string prefix;
};
// Callback for the document to word splitting class during indexation
bool mySplitterCB::takeword(const std::string &_term, int pos, int, int)
// Get one term from the doc, remove accents and lowercase, then add posting
bool TextSplitDb::takeword(const std::string &_term, int pos, int, int)
{
#if 0
LOGDEB(("mySplitterCB::takeword:splitCb: [%s]\n", _term.c_str()));
LOGDEB(("TextSplitDb::takeword: [%s]\n", _term.c_str()));
#endif
string term;
if (!unacmaybefold(_term, term, "UTF-8", true)) {
LOGINFO(("Db::splitter::takeword: unac failed for [%s]\n", _term.c_str()));
LOGINFO(("Db::splitter::takeword: unac failed for [%s]\n",
_term.c_str()));
term.clear();
// We don't generate a fatal error because of a bad term
return true;
@@ -892,14 +892,13 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
Doc doc = idoc;
Xapian::Document newdocument;
mySplitterCB splitData(newdocument, m_stops);
TextSplit splitter(&splitData);
TextSplitDb splitter(newdocument, m_stops);
// Split and index file name as document term(s)
LOGDEB2(("Db::add: split file name [%s]\n", fn.c_str()));
if (!splitter.text_to_words(doc.utf8fn))
LOGDEB(("Db::addOrUpdate: split failed for file name\n"));
splitData.basepos += splitData.curpos + 100;
splitter.basepos += splitter.curpos + 100;
// Index textual metadata. These are all indexed as text with
// positions, as we may want to do phrase searches with them (this
@@ -919,19 +918,19 @@ bool Db::addOrUpdate(const string &udi, const string &parent_udi,
LOGDEB0(("Db::add: field [%s] pfx [%s]: [%s]\n",
meta_it->first.c_str(), pfx.c_str(),
meta_it->second.c_str()));
splitData.setprefix(pfx); // Subject
splitter.setprefix(pfx); // Subject
if (!splitter.text_to_words(meta_it->second))
LOGDEB(("Db::addOrUpdate: split failed for %s\n",
meta_it->first.c_str()));
splitData.setprefix(string());
splitData.basepos += splitData.curpos + 100;
splitter.setprefix(string());
splitter.basepos += splitter.curpos + 100;
}
}
if (splitData.curpos < baseTextPosition)
splitData.basepos = baseTextPosition;
if (splitter.curpos < baseTextPosition)
splitter.basepos = baseTextPosition;
else
splitData.basepos += splitData.curpos + 100;
splitter.basepos += splitter.curpos + 100;
// Split and index body text
LOGDEB2(("Db::add: split body\n"));
src/rcldb/searchdata.cpp
+27 -24
View File
@@ -188,25 +188,27 @@ void SearchData::getUTerms(vector<string>& terms) const
// phrases. This is for parts of the user entry which would appear as
// a single word because there is no white space inside, but are
// actually multiple terms to rcldb (ie term1,term2)
class wsQData : public TextSplitCB {
class TextSplitQ : public TextSplit {
public:
wsQData(const StopList &_stops)
: stops(_stops), alltermcount(0)
TextSplitQ(Flags flags, const StopList &_stops)
: TextSplit(flags), stops(_stops), alltermcount(0)
{}
bool takeword(const std::string &interm, int , int, int) {
alltermcount++;
LOGDEB1(("wsQData::takeword: %s\n", interm.c_str()));
LOGDEB1(("TextSplitQ::takeword: %s\n", interm.c_str()));
// Check if the first letter is a majuscule in which
// case we do not want to do stem expansion. Note that
// the test is convoluted and possibly problematic
string noacterm, noaclowterm;
if (!unacmaybefold(interm, noacterm, "UTF-8", false)) {
LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n", interm.c_str()));
LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n",
interm.c_str()));
return true;
}
if (!unacmaybefold(noacterm, noaclowterm, "UTF-8", true)) {
LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n", noacterm.c_str()));
LOGINFO(("SearchData::splitter::takeword: unac failed for [%s]\n",
noacterm.c_str()));
return true;
}
bool nostemexp = false;
@@ -216,7 +218,8 @@ class wsQData : public TextSplitCB {
nostemexp = true;
if (stops.hasStops() && stops.isStop(noaclowterm)) {
LOGDEB1(("wsQData::takeword [%s] in stop list\n", noaclowterm.c_str()));
LOGDEB1(("TextSplitQ::takeword [%s] in stop list\n",
noaclowterm.c_str()));
return true;
}
terms.push_back(noaclowterm);
@@ -271,7 +274,7 @@ private:
// After splitting entry on whitespace: process non-phrase element
void processSimpleSpan(const string& span, bool nostemexp, list<Xapian::Query> &pqueries);
// Process phrase/near element
void processPhraseOrNear(wsQData *splitData,
void processPhraseOrNear(TextSplitQ *splitData,
list<Xapian::Query> &pqueries,
bool useNear, int slack);
@@ -420,7 +423,7 @@ void StringToXapianQ::processSimpleSpan(const string& span, bool nostemexp,
// NEAR xapian query, the elements of which can themselves be OR
// queries if the terms get expanded by stemming or wildcards (we
// don't do stemming for PHRASE though)
void StringToXapianQ::processPhraseOrNear(wsQData *splitData,
void StringToXapianQ::processPhraseOrNear(TextSplitQ *splitData,
list<Xapian::Query> &pqueries,
bool useNear, int slack)
{
@@ -527,31 +530,31 @@ bool StringToXapianQ::processUserString(const string &iq,
// We now adjust the phrase/near slack by the term count
// difference (this is mainly better for cjk where this is a very
// common occurrence because of the ngrams thing.
wsQData splitDataS(stops), splitDataW(stops);
TextSplit splitterS(&splitDataS,
TextSplit::Flags(TextSplit::TXTS_ONLYSPANS |
TextSplit::TXTS_KEEPWILD));
TextSplitQ splitterS(TextSplit::Flags(TextSplit::TXTS_ONLYSPANS |
TextSplit::TXTS_KEEPWILD),
stops);
splitterS.text_to_words(*it);
TextSplit splitterW(&splitDataW,
TextSplit::Flags(TextSplit::TXTS_NOSPANS |
TextSplit::TXTS_KEEPWILD));
TextSplitQ splitterW(TextSplit::Flags(TextSplit::TXTS_NOSPANS |
TextSplit::TXTS_KEEPWILD),
stops);
splitterW.text_to_words(*it);
wsQData *splitData = &splitDataS;
if (splitDataS.terms.size() > 1 &&
splitDataS.terms.size() != splitDataW.terms.size()) {
slack += splitDataW.terms.size() - splitDataS.terms.size();
TextSplitQ *splitter = &splitterS;
if (splitterS.terms.size() > 1 &&
splitterS.terms.size() != splitterW.terms.size()) {
slack += splitterW.terms.size() - splitterS.terms.size();
// used to: splitData = &splitDataW;
}
LOGDEB0(("strToXapianQ: termcount: %d\n", splitData->terms.size()));
switch (splitData->terms.size()) {
LOGDEB0(("strToXapianQ: termcount: %d\n", splitter->terms.size()));
switch (splitter->terms.size()) {
case 0:
continue;// ??
case 1:
processSimpleSpan(splitData->terms.front(), splitData->nostemexps.front(), pqueries);
processSimpleSpan(splitter->terms.front(),
splitter->nostemexps.front(), pqueries);
break;
default:
processPhraseOrNear(splitData, pqueries, useNear, slack);
processPhraseOrNear(splitter, pqueries, useNear, slack);
}
}
} catch (const Xapian::Error &e) {
src/rcldb/stoplist.cpp
+18 -11
View File
@@ -5,6 +5,7 @@ static char rcsid[] = "@(#$Id: stoplist.cpp,v 1.1 2007-06-02 08:30:42 dockes Exp
#include "debuglog.h"
#include "readfile.h"
#include "unacpp.h"
#include "textsplit.h"
#include "stoplist.h"
#ifndef NO_NAMESPACES
@@ -12,6 +13,21 @@ namespace Rcl
{
#endif
class TextSplitSW : public TextSplit {
public:
set<string>& stops;
TextSplitSW(Flags flags, set<string>& stps)
: TextSplit(flags), stops(stps)
{}
virtual bool takeword(const string& term, int, int, int)
{
string dterm;
unacmaybefold(term, dterm, "UTF-8", true);
stops.insert(dterm);
return true;
}
};
bool StopList::setFile(const string &filename)
{
m_hasStops = false;
@@ -22,18 +38,9 @@ bool StopList::setFile(const string &filename)
filename.c_str(), reason.c_str()));
return false;
}
TextSplit ts(this, TextSplit::TXTS_ONLYSPANS);
TextSplitSW ts(TextSplit::TXTS_ONLYSPANS, m_stops);
ts.text_to_words(stoptext);
return true;
}
bool StopList::takeword(const string& term, int, int, int)
{
string dterm;
unacmaybefold(term, dterm, "UTF-8", true);
LOGDEB2(("StopList::takeword: inserting [%s]\n", dterm.c_str()));
m_hasStops = true;
m_stops.insert(dterm);
m_hasStops = !m_stops.empty();
return true;
}
src/rcldb/stoplist.h
+1 -4
View File
@@ -5,8 +5,6 @@
#include <set>
#include <string>
#include "textsplit.h"
#ifndef NO_NAMESPACES
using std::set;
using std::string;
@@ -14,7 +12,7 @@ namespace Rcl
{
#endif
class StopList : public TextSplitCB {
class StopList {
public:
StopList() : m_hasStops(false) {}
StopList(const string &filename) {setFile(filename);}
@@ -23,7 +21,6 @@ public:
bool setFile(const string &filename);
bool isStop(const string &term) const;
bool hasStops() const {return m_hasStops;}
virtual bool takeword(const string& term, int pos, int bts, int bte);
private:
bool m_hasStops;