added code for unac to perform pure case-folding

This commit is contained in:
Jean-Francois Dockes
2012-08-27 12:40:57 +02:00
parent 9327527101
commit 913dffc597
12 changed files with 2133 additions and 1972 deletions
+1 -1
View File
@@ -117,7 +117,7 @@ public:
// approach to collating...)
string sortterm;
// We're not even sure the term is utf8 here (ie: url)
if (!unacmaybefold(term, sortterm, "UTF-8", true)) {
if (!unacmaybefold(term, sortterm, "UTF-8", UNACOP_UNACFOLD)) {
sortterm = term;
}
// Also remove some common uninteresting starting characters
+1 -1
View File
@@ -41,7 +41,7 @@ bool StopList::setFile(const string &filename)
for (set<string>::iterator it = stops.begin();
it != stops.end(); it++) {
string dterm;
unacmaybefold(*it, dterm, "UTF-8", true);
unacmaybefold(*it, dterm, "UTF-8", UNACOP_UNACFOLD);
m_stops.insert(dterm);
}
+1 -1
View File
@@ -117,7 +117,7 @@ public:
{
m_totalterms++;
string otrm;
if (!unacmaybefold(itrm, otrm, "UTF-8", true)) {
if (!unacmaybefold(itrm, otrm, "UTF-8", UNACOP_UNACFOLD)) {
LOGDEB(("splitter::takeword: unac [%s] failed\n", itrm.c_str()));
m_unacerrors++;
// We don't generate a fatal error because of a bad term,