added code for unac to perform pure case-folding
This commit is contained in:
@@ -117,7 +117,7 @@ public:
|
||||
// approach to collating...)
|
||||
string sortterm;
|
||||
// We're not even sure the term is utf8 here (ie: url)
|
||||
if (!unacmaybefold(term, sortterm, "UTF-8", true)) {
|
||||
if (!unacmaybefold(term, sortterm, "UTF-8", UNACOP_UNACFOLD)) {
|
||||
sortterm = term;
|
||||
}
|
||||
// Also remove some common uninteresting starting characters
|
||||
|
||||
@@ -41,7 +41,7 @@ bool StopList::setFile(const string &filename)
|
||||
for (set<string>::iterator it = stops.begin();
|
||||
it != stops.end(); it++) {
|
||||
string dterm;
|
||||
unacmaybefold(*it, dterm, "UTF-8", true);
|
||||
unacmaybefold(*it, dterm, "UTF-8", UNACOP_UNACFOLD);
|
||||
m_stops.insert(dterm);
|
||||
}
|
||||
|
||||
|
||||
@@ -117,7 +117,7 @@ public:
|
||||
{
|
||||
m_totalterms++;
|
||||
string otrm;
|
||||
if (!unacmaybefold(itrm, otrm, "UTF-8", true)) {
|
||||
if (!unacmaybefold(itrm, otrm, "UTF-8", UNACOP_UNACFOLD)) {
|
||||
LOGDEB(("splitter::takeword: unac [%s] failed\n", itrm.c_str()));
|
||||
m_unacerrors++;
|
||||
// We don't generate a fatal error because of a bad term,
|
||||
|
||||
Reference in New Issue
Block a user